socket: Prevent extra logging in socket_rwv

Problem: If the socket is not even connected socket_rwv fails and the error log comes every 3 seconds for every re-connect. Fix: Prevent this error log if the socket is not even connected. Change-Id: I963f1345c4b6779dd5491948f7f5c4357a507b67 BUG: 963088 Signed-off-by: Pranith Kumar K <pkarampu@redhat.com> Reviewed-on: http://review.gluster.org/5008 Reviewed-by: Amar Tumballi <amarts@redhat.com> Tested-by: Gluster Build System <jenkins@build.gluster.com> Reviewed-by: Vijay Bellur <vbellur@redhat.com>
author: Pranith Kumar K <pkarampu@redhat.com> 2013-05-15 12:09:28 +0530
committer: Vijay Bellur <vbellur@redhat.com> 2013-05-31 11:22:24 -0700
commit: 9e974f9311869fb317c9c691ed09e7dd18047248 (patch)
tree: 598cbff63e2a10a3eef4a3b8e4128331f5b5b810 /rpc
parent: 114c02e54abae9b3cbf0f84bc48132bf1535118e (diff)
1 files changed, 19 insertions, 19 deletions
diff --git a/rpc/rpc-transport/socket/src/socket.c b/rpc/rpc-transport/socket/src/socket.c
index d9d215389..97c0dc57d 100644
--- a/rpc/rpc-transport/socket/src/socket.c
+++ b/rpc/rpc-transport/socket/src/socket.c
@@ -147,15 +147,8 @@ typedef int SSL_trinary_func (SSL *, void *, int);
                                       &in->pending_vector,              \
                                       &in->pending_count,               \
                                       &bytes_read);                     \
-                if (ret == -1) {                                        \
-                        if (priv->read_fail_log)                        \
-                                gf_log (this->name, GF_LOG_WARNING,     \
-                                        "reading from socket failed."   \
-                                        "Error (%s), peer (%s)",        \
-                                        strerror (errno),               \
-                                        this->peerinfo.identifier);     \
+                if (ret == -1)                                          \
                         break;                                          \
-                }                                                       \
                 __socket_proto_update_priv_after_read (priv, ret, bytes_read); \
         }
 
@@ -424,6 +417,19 @@ out:
 	return ret;
 }
 
+static gf_boolean_t
+__does_socket_rwv_error_need_logging (socket_private_t *priv, int write)
+{
+        int read = !write;
+
+        if (priv->connected == -1) /* Didn't even connect, of course it fails */
+                return _gf_false;
+
+        if (read && (priv->read_fail_log == _gf_false))
+                return _gf_false;
+
+        return _gf_true;
+}
 
 /*
  * return value:
@@ -507,12 +513,15 @@ __socket_rwv (rpc_transport_t *this, struct iovec *vector, int count,
                         if (errno == EINTR)
                                 continue;
 
-                        if (write || (!write && priv->read_fail_log))
+                        if (__does_socket_rwv_error_need_logging (priv,
+                                                                  write)) {
                                 gf_log (this->name, GF_LOG_WARNING,
                                         "%s on %s failed (%s)",
                                         write ? "writev":"readv",
                                         this->peerinfo.identifier,
                                         strerror (errno));
+                        }
+
 			if (priv->use_ssl) {
 				ssl_dump_error_stack(this->name);
 			}
@@ -1953,17 +1962,8 @@ __socket_proto_state_machine (rpc_transport_t *this,
                                               &in->pending_vector,
                                               &in->pending_count,
                                               NULL);
-                        if (ret == -1) {
-                                if (priv->read_fail_log == 1) {
-                                        gf_log (this->name,
-                                                ((priv->connected == 1) ?
-                                                 GF_LOG_WARNING : GF_LOG_DEBUG),
-                                                "reading from socket failed. Error (%s)"
-                                                ", peer (%s)", strerror (errno),
-                                                this->peerinfo.identifier);
-                                }
+                        if (ret == -1)
                                 goto out;
-                        }
 
                         if (ret > 0) {
                                 gf_log (this->name, GF_LOG_TRACE, "partial "
diff --git a/xlators/Makefile.am b/xlators/Makefile.am
index 4c94f5e44..f60fa85ce 100644
--- a/xlators/Makefile.am
+++ b/xlators/Makefile.am
@@ -1,3 +1,4 @@
-SUBDIRS = cluster storage protocol performance debug features encryption mount nfs mgmt
+SUBDIRS = cluster storage protocol performance debug features encryption mount nfs mgmt system \
+          playground
 
 CLEANFILES =
diff --git a/xlators/bindings/Makefile.am b/xlators/bindings/Makefile.am
deleted file mode 100644
index f77665802..000000000
--- a/xlators/bindings/Makefile.am
+++ /dev/null
@@ -1 +0,0 @@
-SUBDIRS = $(BINDINGS_SUBDIRS)
diff --git a/xlators/bindings/python/src/Makefile.am b/xlators/bindings/python/src/Makefile.am
deleted file mode 100644
index c0b9141c6..000000000
--- a/xlators/bindings/python/src/Makefile.am
+++ /dev/null
@@ -1,19 +0,0 @@
-
-xlator_PROGRAMS = python.so
-
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/bindings
-
-python_PYTHON = gluster.py glustertypes.py glusterstack.py
-
-pythondir = $(xlatordir)/python
-
-python_so_SOURCES = python.c
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles \
-	$(PYTHON_CPPLAGS) -DGLUSTER_PYTHON_PATH=\"$(pythondir)\"
-
-AM_LDFLAGS = $(PYTHON_LDFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/bindings/python/src/gluster.py b/xlators/bindings/python/src/gluster.py
deleted file mode 100644
index ee0eb1310..000000000
--- a/xlators/bindings/python/src/gluster.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
-from ctypes import *
-from glustertypes import *
-from glusterstack import *
-import sys
-import inspect
-
-libglusterfs = CDLL("libglusterfs.so")
-_gf_log = libglusterfs._gf_log
-_gf_log.restype = c_int32
-_gf_log.argtypes = [c_char_p, c_char_p, c_char_p, c_int32, c_int, c_char_p]
-
-gf_log_loglevel = c_int.in_dll(libglusterfs, "gf_log_loglevel")
-
-GF_LOG_NONE = 0
-GF_LOG_CRITICAL = 1
-GF_LOG_ERROR = 2
-GF_LOG_WARNING = 3
-GF_LOG_DEBUG = 4
-
-def gf_log(module, level, fmt, *params):
-    if level <= gf_log_loglevel:
-        frame = sys._getframe(1)
-        _gf_log(module, frame.f_code.co_filename, frame.f_code.co_name,
-                frame.f_lineno, level, fmt, *params)
-
-class ComplexTranslator(object):
-    def __init__(self, xlator):
-        self.xlator = xlator_t.from_address(xlator)
-
-    def __getattr__(self, item):
-        return getattr(self.xlator, item)
diff --git a/xlators/bindings/python/src/glusterstack.py b/xlators/bindings/python/src/glusterstack.py
deleted file mode 100644
index ba24c8165..000000000
--- a/xlators/bindings/python/src/glusterstack.py
+++ /dev/null
@@ -1,55 +0,0 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
-from ctypes import *
-from glustertypes import *
-
-libc = CDLL("libc.so.6")
-calloc = libc.calloc
-calloc.argtypes = [c_int, c_int]
-calloc.restype = c_void_p
-
-# TODO: Can these be done in C somehow?
-def stack_wind(frame, rfn, obj, fn, *params):
-    """Frame is a frame object"""
-    _new = cast(calloc(1, sizeof(call_frame_t)), POINTER(call_frame_t))
-    _new[0].root = frame.root
-    _new[0].next = frame.root[0].frames.next
-    _new[0].prev = pointer(frame.root[0].frames)
-    if frame.root[0].frames.next:
-        frame.root[0].frames.next[0].prev = _new
-    frame.root[0].frames.next = _new
-    _new[0].this = obj
-    # TODO: Type checking like tmp_cbk?
-    _new[0].ret = rfn
-    _new[0].parent = pointer(frame)
-    _new[0].cookie = cast(_new, c_void_p)
-    # TODO: Initialize lock
-    #_new.lock.init()
-    frame.ref_count += 1
-    fn(_new, obj, *params)
-
-def stack_unwind(frame, *params):
-    """Frame is a frame object"""
-    fn = frame[0].ret
-    parent = frame[0].parent[0]
-    parent.ref_count -= 1
-
-    op_ret = params[0]
-    op_err = params[1]
-    params = params[2:]
-    fn(parent, call_frame_t.from_address(frame[0].cookie), parent.this,
-            op_ret, op_err, *params)
diff --git a/xlators/bindings/python/src/glustertypes.py b/xlators/bindings/python/src/glustertypes.py
deleted file mode 100644
index e9069d07c..000000000
--- a/xlators/bindings/python/src/glustertypes.py
+++ /dev/null
@@ -1,167 +0,0 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
-from ctypes import *
-import collections
-
-#
-# Forward declaration of some gluster types
-#
-class call_frame_t(Structure):
-    pass
-
-class call_ctx_t(Structure):
-    pass
-
-class call_pool_t(Structure):
-    pass
-
-class xlator_t(Structure):
-    def _getFirstChild(self):
-        return self.children[0].xlator
-    firstChild = property(_getFirstChild)
-
-class xlator_list_t(Structure):
-    pass
-
-class xlator_fops(Structure):
-    pass
-
-class xlator_mops(Structure):
-    pass
-
-class glusterfs_ctx_t(Structure):
-    pass
-
-class list_head(Structure):
-    pass
-
-class dict_t(Structure):
-    pass
-
-class inode_table_t(Structure):
-    pass
-
-class fd_t(Structure):
-    pass
-
-class iovec(Structure):
-    _fields_ = [
-            ("iov_base", c_void_p),
-            ("iov_len", c_size_t),
-            ]
-
-    def __init__(self, s):
-        self.iov_base = cast(c_char_p(s), c_void_p)
-        self.iov_len = len(s)
-
-    def getBytes(self):
-        return string_at(self.iov_base, self.iov_len)
-
-# This is a pthread_spinlock_t
-# TODO: what happens to volatile-ness?
-gf_lock_t = c_int
-
-uid_t = c_uint32
-gid_t = c_uint32
-pid_t = c_int32
-
-off_t = c_int64
-
-#
-# Function pointer types
-#
-ret_fn_t = CFUNCTYPE(c_int32, POINTER(call_frame_t), POINTER(call_frame_t),
-                              POINTER(xlator_t), c_int32, c_int32)
-
-fini_fn_t = CFUNCTYPE(None, POINTER(xlator_t))
-init_fn_t = CFUNCTYPE(c_int32, POINTER(xlator_t))
-event_notify_fn_t = CFUNCTYPE(c_int32, POINTER(xlator_t), c_int32, c_void_p)
-
-list_head._fields_ = [
-        ("next", POINTER(list_head)),
-        ("prev", POINTER(list_head)),
-        ]
-
-call_frame_t._fields_ = [
-        ("root", POINTER(call_ctx_t)),
-        ("parent", POINTER(call_frame_t)),
-        ("next", POINTER(call_frame_t)),
-        ("prev", POINTER(call_frame_t)),
-        ("local", c_void_p),
-        ("this", POINTER(xlator_t)),
-        ("ret", ret_fn_t),
-        ("ref_count", c_int32),
-        ("lock", gf_lock_t),
-        ("cookie", c_void_p),
-        ("op", c_int32),
-        ("type", c_int8),
-        ]
-
-call_ctx_t._fields_ = [
-        ("all_frames", list_head),
-        ("trans", c_void_p),
-        ("pool", call_pool_t),
-        ("unique", c_uint64),
-        ("state", c_void_p),
-        ("uid", uid_t),
-        ("gid", gid_t),
-        ("pid", pid_t),
-        ("frames", call_frame_t),
-        ("req_refs", POINTER(dict_t)),
-        ("rsp_refs", POINTER(dict_t)),
-        ]
-
-xlator_t._fields_ = [
-        ("name", c_char_p),
-        ("type", c_char_p),
-        ("next", POINTER(xlator_t)),
-        ("prev", POINTER(xlator_t)),
-        ("parent", POINTER(xlator_t)),
-        ("children", POINTER(xlator_list_t)),
-        ("fops", POINTER(xlator_fops)),
-        ("mops", POINTER(xlator_mops)),
-        ("fini", fini_fn_t),
-        ("init", init_fn_t),
-        ("notify", event_notify_fn_t),
-        ("options", POINTER(dict_t)),
-        ("ctx", POINTER(glusterfs_ctx_t)),
-        ("itable", POINTER(inode_table_t)),
-        ("ready", c_char),
-        ("private", c_void_p),
-        ]
-
-xlator_list_t._fields_ = [
-        ("xlator", POINTER(xlator_t)),
-        ("next", POINTER(xlator_list_t)),
-        ]
-
-fop_functions = collections.defaultdict(lambda: c_void_p)
-fop_function_names = ['lookup', 'forget', 'stat', 'fstat', 'chmod', 'fchmod',
-        'chown', 'fchown', 'truncate', 'ftruncate', 'utimens', 'access',
-        'readlink', 'mknod', 'mkdir', 'unlink', 'rmdir', 'symlink',
-        'rename', 'link', 'create', 'open', 'readv', 'writev', 'flush',
-        'close', 'fsync', 'opendir', 'readdir', 'closedir', 'fsyncdir',
-        'statfs', 'setxattr', 'getxattr', 'removexattr', 'lk', 'writedir',
-        # TODO: Call backs?
-        ]
-
-fop_writev_t = CFUNCTYPE(c_int32, POINTER(call_frame_t), POINTER(xlator_t),
-                                  POINTER(fd_t), POINTER(iovec), c_int32,
-                                  off_t)
-
-fop_functions['writev'] = fop_writev_t
-xlator_fops._fields_ = [(f, fop_functions[f]) for f in fop_function_names]
diff --git a/xlators/bindings/python/src/python.c b/xlators/bindings/python/src/python.c
deleted file mode 100644
index c11323cda..000000000
--- a/xlators/bindings/python/src/python.c
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
-   Copyright (c) 2007-2009 Chris AtLee <chris@atlee.ca>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   General Public License for more details.
-
-   You should have received a copy of the GNU General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#include <Python.h>
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "xlator.h"
-#include "logging.h"
-#include "defaults.h"
-
-typedef struct
-{
-    char        *scriptname;
-    PyObject    *pXlator;
-    PyObject    *pScriptModule;
-    PyObject    *pGlusterModule;
-    PyThreadState *pInterp;
-
-    PyObject    *pFrameType, *pVectorType, *pFdType;
-} python_private_t;
-
-int32_t
-python_writev (call_frame_t *frame,
-              xlator_t *this,
-              fd_t *fd,
-              struct iovec *vector,
-              int32_t count, 
-              off_t offset)
-{
-  python_private_t *priv = (python_private_t *)this->private;
-  gf_log("python", GF_LOG_DEBUG, "In writev");
-  if (PyObject_HasAttrString(priv->pXlator, "writev"))
-  {
-
-      PyObject *retval = PyObject_CallMethod(priv->pXlator, "writev",
-              "O O O i l",
-              PyObject_CallMethod(priv->pFrameType, "from_address", "O&", PyLong_FromVoidPtr, frame),
-              PyObject_CallMethod(priv->pFdType, "from_address", "O&", PyLong_FromVoidPtr, fd),
-              PyObject_CallMethod(priv->pVectorType, "from_address", "O&", PyLong_FromVoidPtr, vector),
-              count,
-              offset);
-      if (PyErr_Occurred())
-      {
-          PyErr_Print();
-      }
-      Py_XDECREF(retval);
-  }
-  else
-  {
-      return default_writev(frame, this, fd, vector, count, offset);
-  }
-  return 0;
-}
-
-struct xlator_fops fops = {
-    .writev       = python_writev
-};
-
-static PyObject *
-AnonModule_FromFile (const char* fname)
-{
-    // Get the builtins
-    PyThreadState* pThread = PyThreadState_Get();
-    PyObject *pBuiltins = pThread->interp->builtins;
-
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        return NULL;
-    }
-
-    // Create a new dictionary for running code in
-    PyObject *pModuleDict = PyDict_New();
-    PyDict_SetItemString(pModuleDict, "__builtins__", pBuiltins);
-    Py_INCREF(pBuiltins);
-
-    // Run the file in the new context
-    FILE* fp = fopen(fname, "r");
-    PyRun_File(fp, fname, Py_file_input, pModuleDict, pModuleDict);
-    fclose(fp);
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        Py_DECREF(pModuleDict);
-        Py_DECREF(pBuiltins);
-        return NULL;
-    }
-
-    // Create an object to hold the new context
-    PyRun_String("class ModuleWrapper(object):\n\tpass\n", Py_single_input, pModuleDict, pModuleDict);
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        Py_DECREF(pModuleDict);
-        Py_DECREF(pBuiltins);
-        return NULL;
-    }
-    PyObject *pModule = PyRun_String("ModuleWrapper()", Py_eval_input, pModuleDict, pModuleDict);
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        Py_DECREF(pModuleDict);
-        Py_DECREF(pBuiltins);
-        Py_XDECREF(pModule);
-        return NULL;
-    }
-
-    // Set the new context's dictionary to the one we used to run the code
-    // inside
-    PyObject_SetAttrString(pModule, "__dict__", pModuleDict);
-    if (PyErr_Occurred())
-    {
-        PyErr_Print();
-        Py_DECREF(pModuleDict);
-        Py_DECREF(pBuiltins);
-        Py_DECREF(pModule);
-        return NULL;
-    }
-
-    return pModule;
-}
-
-int32_t
-init (xlator_t *this)
-{
-  // This is ok to call more than once per process
-  Py_InitializeEx(0);
-
-  if (!this->children) {
-    gf_log ("python", GF_LOG_ERROR, 
-            "FATAL: python should have exactly one child");
-    return -1;
-  }
-
-  python_private_t *priv = CALLOC (sizeof (python_private_t), 1);
-  ERR_ABORT (priv);
-
-  data_t *scriptname = dict_get (this->options, "scriptname");
-  if (scriptname) {
-      priv->scriptname = data_to_str(scriptname);
-  } else {
-      gf_log("python", GF_LOG_ERROR,
-              "FATAL: python requires the scriptname parameter");
-      return -1;
-  }
-
-  priv->pInterp = Py_NewInterpreter();
-    
-  // Adjust python's path
-  PyObject *syspath = PySys_GetObject("path");
-  PyObject *path = PyString_FromString(GLUSTER_PYTHON_PATH);
-  PyList_Append(syspath, path);
-  Py_DECREF(path);
-
-  gf_log("python", GF_LOG_DEBUG,
-          "Loading gluster module");
-
-  priv->pGlusterModule = PyImport_ImportModule("gluster");
-  if (PyErr_Occurred())
-  {
-      PyErr_Print();
-      return -1;
-  }
-
-  priv->pFrameType = PyObject_GetAttrString(priv->pGlusterModule, "call_frame_t");
-  priv->pFdType = PyObject_GetAttrString(priv->pGlusterModule, "fd_t");
-  priv->pVectorType = PyObject_GetAttrString(priv->pGlusterModule, "iovec");
-
-  gf_log("python", GF_LOG_DEBUG, "Loading script...%s", priv->scriptname);
-  
-  priv->pScriptModule = AnonModule_FromFile(priv->scriptname);
-  if (!priv->pScriptModule || PyErr_Occurred())
-  {
-      gf_log("python", GF_LOG_ERROR, "Error loading %s", priv->scriptname);
-      PyErr_Print();
-      return -1;
-  }
-
-  if (!PyObject_HasAttrString(priv->pScriptModule, "xlator"))
-  {
-      gf_log("python", GF_LOG_ERROR, "%s does not have a xlator attribute", priv->scriptname);
-      return -1;
-  }
-  gf_log("python", GF_LOG_DEBUG, "Instantiating translator");
-  priv->pXlator = PyObject_CallMethod(priv->pScriptModule, "xlator", "O&",
-          PyLong_FromVoidPtr, this);
-  if (PyErr_Occurred() || !priv->pXlator)
-  {
-      PyErr_Print();
-      return -1;
-  }
-
-  this->private = priv;
-
-  gf_log ("python", GF_LOG_DEBUG, "python xlator loaded");
-  return 0;
-}
-
-void 
-fini (xlator_t *this)
-{
-  python_private_t *priv = (python_private_t*)(this->private);
-  Py_DECREF(priv->pXlator);
-  Py_DECREF(priv->pScriptModule);
-  Py_DECREF(priv->pGlusterModule);
-  Py_DECREF(priv->pFrameType);
-  Py_DECREF(priv->pFdType);
-  Py_DECREF(priv->pVectorType);
-  Py_EndInterpreter(priv->pInterp);
-  return;
-}
diff --git a/xlators/bindings/python/src/testxlator.py b/xlators/bindings/python/src/testxlator.py
deleted file mode 100644
index 507455c85..000000000
--- a/xlators/bindings/python/src/testxlator.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#   Copyright (c) 2007 Chris AtLee <chris@atlee.ca>
-#   This file is part of GlusterFS.
-#
-#   GlusterFS is free software; you can redistribute it and/or modify
-#   it under the terms of the GNU General Public License as published
-#   by the Free Software Foundation; either version 3 of the License,
-#   or (at your option) any later version.
-#
-#   GlusterFS is distributed in the hope that it will be useful, but
-#   WITHOUT ANY WARRANTY; without even the implied warranty of
-#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-#   General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public License
-#   along with this program.  If not, see
-#   <http://www.gnu.org/licenses/>.
-
-"""
-This is a test translator written in python.
-
-Important things to note:
-    This file must be import-able from glusterfsd.  This probably means
-    setting PYTHONPATH to where this file is located.
-
-    This file must have a top-level xlator class object that will be
-    used to instantiate individual translators.
-"""
-from gluster import *
-
-class MyXlator(ComplexTranslator):
-    name = "MyXlator"
-    def writev_cbk(self, frame, cookie, op_ret, op_errno, buf):
-        stack_unwind(frame, op_ret, op_errno, buf)
-        return 0
-
-    def writev(self, frame, fd, vector, count, offset):
-        gf_log(self.name, GF_LOG_WARNING, "writev %i bytes", vector.iov_len)
-        # TODO: Use cookie to pass this to writev_cbk
-        old_count = vector.iov_len
-
-        data = vector.getBytes().encode("zlib")
-
-        vector = iovec(data)
-        gf_log(self.name, GF_LOG_WARNING, "writev %i bytes", vector.iov_len)
-
-        @ret_fn_t
-        def rfn(frame, prev, this, op_ret, op_errno, *params):
-            if len(params) == 0:
-                params = [0]
-            return self.writev_cbk(frame, prev, old_count, op_errno, *params)
-
-        stack_wind(frame, rfn, self.firstChild,
-                self.firstChild[0].fops[0].writev, fd, vector, count, offset)
-        return 0
-
-xlator = MyXlator
diff --git a/xlators/cluster/Makefile.am b/xlators/cluster/Makefile.am
index 0990822a7..6e883e565 100644
--- a/xlators/cluster/Makefile.am
+++ b/xlators/cluster/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = stripe afr dht
+SUBDIRS = stripe afr dht nsr-server nsr-recon nsr-client
 
 CLEANFILES = 
diff --git a/xlators/cluster/afr/src/Makefile.am b/xlators/cluster/afr/src/Makefile.am
index 4e4b4c752..ea5a90abb 100644
--- a/xlators/cluster/afr/src/Makefile.am
+++ b/xlators/cluster/afr/src/Makefile.am
@@ -1,26 +1,38 @@
 xlator_LTLIBRARIES = afr.la pump.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
-afr_common_source = afr-dir-read.c afr-dir-write.c afr-inode-read.c afr-inode-write.c afr-open.c afr-transaction.c afr-self-heal-data.c afr-self-heal-common.c afr-self-heal-metadata.c afr-self-heal-entry.c afr-self-heal-algorithm.c afr-lk-common.c
+afr_common_source = afr-dir-read.c afr-dir-write.c afr-inode-read.c \
+	afr-inode-write.c afr-open.c afr-transaction.c afr-lk-common.c \
+	afr-read-txn.c \
+	$(top_builddir)/xlators/lib/src/libxlator.c
 
-afr_la_LDFLAGS = -module -avoidversion
-afr_la_SOURCES = $(afr_common_source) afr.c
+AFR_SELFHEAL_SOURCES = afr-self-heal-common.c afr-self-heal-data.c \
+	afr-self-heal-entry.c afr-self-heal-metadata.c afr-self-heald.c \
+	afr-self-heal-name.c
+
+afr_la_LDFLAGS = -module -avoid-version
+afr_la_SOURCES = $(afr_common_source) $(AFR_SELFHEAL_SOURCES) afr.c
 afr_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-pump_la_LDFLAGS = -module -avoidversion
-pump_la_SOURCES =  $(afr_common_source) pump.c
+pump_la_LDFLAGS = -module -avoid-version
+pump_la_SOURCES =  $(afr_common_source) $(AFR_SELFHEAL_SOURCES) pump.c
 pump_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = afr.h afr-transaction.h afr-inode-write.h afr-inode-read.h afr-dir-read.h afr-dir-write.h afr-self-heal.h afr-self-heal-common.h afr-self-heal-algorithm.h pump.h afr-mem-types.h afr-common.c
+noinst_HEADERS = afr.h afr-transaction.h afr-inode-write.h afr-inode-read.h \
+	afr-dir-read.h afr-dir-write.h afr-self-heal.h afr-mem-types.h \
+	afr-common.c afr-self-heald.h pump.h \
+	$(top_builddir)/xlators/lib/src/libxlator.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	    -I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/contrib/md5 -shared -nostartfiles $(GF_CFLAGS)
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES =
 
 uninstall-local:
 	rm -f $(DESTDIR)$(xlatordir)/replicate.so
-	rm -f $(DESTDIR)$(xlatordir)/pump.so
 
 install-data-hook:
 	ln -sf afr.so $(DESTDIR)$(xlatordir)/replicate.so
diff --git a/xlators/cluster/afr/src/afr-common.c b/xlators/cluster/afr/src/afr-common.c
index 8d2a4f997..164a651ba 100644
--- a/xlators/cluster/afr/src/afr-common.c
+++ b/xlators/cluster/afr/src/afr-common.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -44,6 +35,7 @@
 #include "compat.h"
 #include "byte-order.h"
 #include "statedump.h"
+#include "inode.h"
 
 #include "fd.h"
 
@@ -53,308 +45,875 @@
 #include "afr-dir-write.h"
 #include "afr-transaction.h"
 #include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-#include "pump.h"
+#include "afr-self-heald.h"
 
-#define AFR_ICTX_OPENDIR_DONE_MASK     0x0000000200000000ULL
-#define AFR_ICTX_SPLIT_BRAIN_MASK      0x0000000100000000ULL
-#define AFR_ICTX_READ_CHILD_MASK       0x00000000FFFFFFFFULL
 
-int32_t
-afr_set_dict_gfid (dict_t *dict, uuid_t gfid)
+call_frame_t *
+afr_copy_frame (call_frame_t *base)
 {
-        int ret = 0;
+	afr_local_t *local = NULL;
+	call_frame_t *frame = NULL;
+	int op_errno = 0;
+
+	frame = copy_frame (base);
+	if (!frame)
+		return NULL;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local) {
+		AFR_STACK_DESTROY (frame);
+		return NULL;
+	}
+
+	return frame;
+}
 
-        GF_ASSERT (gfid);
+/*
+ * INODE CTX 64-bit VALUE FORMAT FOR SMALL (<= 16) SUBVOL COUNTS:
+ *
+ * |<----------   64bit   ------------>|
+ *  63           32 31    16 15       0
+ * |   EVENT_GEN   |  DATA  | METADATA |
+ *
+ *
+ *  METADATA (bit-0 .. bit-15): bitmap representing subvolumes from which
+ *                              metadata can be attempted to be read.
+ *
+ *                              bit-0 => priv->subvolumes[0]
+ *                              bit-1 => priv->subvolumes[1]
+ *                              ... etc. till bit-15
+ *
+ *  DATA (bit-16 .. bit-31): bitmap representing subvolumes from which data
+ *                           can be attempted to be read.
+ *
+ *                           bit-16 => priv->subvolumes[0]
+ *                           bit-17 => priv->subvolumes[1]
+ *                           ... etc. till bit-31
+ *
+ *  EVENT_GEN (bit-32 .. bit-63): event generation (i.e priv->event_generation)
+ *                                when DATA and METADATA was last updated.
+ *
+ *                                If EVENT_GEN is < priv->event_generation,
+ *                                or is 0, it means afr_inode_refresh() needs
+ *                                to be called to recalculate the bitmaps.
+ */
 
-        ret = dict_set_static_bin (dict, "gfid-req", gfid, 16);
-        if (ret)
-                gf_log (THIS->name, GF_LOG_DEBUG, "gfid set failed");
+int
+__afr_inode_read_subvol_get_small (inode_t *inode, xlator_t *this,
+				   unsigned char *data, unsigned char *metadata,
+				   int *event_p)
+{
+	afr_private_t *priv = NULL;
+	int ret = -1;
+	uint16_t datamap = 0;
+	uint16_t metadatamap = 0;
+	uint32_t event = 0;
+	uint64_t val = 0;
+	int i = 0;
+
+	priv = this->private;
+
+	ret = __inode_ctx_get (inode, this, &val);
+	if (ret < 0)
+		return ret;
+
+	metadatamap = (val & 0x000000000000ffff);
+	datamap =     (val & 0x00000000ffff0000) >> 16;
+	event =       (val & 0xffffffff00000000) >> 32;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (metadata)
+			metadata[i] = (metadatamap >> i) & 1;
+		if (data)
+			data[i] = (datamap >> i) & 1;
+	}
+
+	if (event_p)
+		*event_p = event;
+	return ret;
+}
 
-        return ret;
+
+int
+__afr_inode_read_subvol_set_small (inode_t *inode, xlator_t *this,
+				   unsigned char *data, unsigned char *metadata,
+				   int event)
+{
+	afr_private_t *priv = NULL;
+	uint16_t datamap = 0;
+	uint16_t metadatamap = 0;
+	uint64_t val = 0;
+	int i = 0;
+
+	priv = this->private;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (data[i])
+			datamap |= (1 << i);
+		if (metadata[i])
+			metadatamap |= (1 << i);
+	}
+
+	val = ((uint64_t) metadatamap) |
+		(((uint64_t) datamap) << 16) |
+		(((uint64_t) event) << 32);
+
+	return __inode_ctx_set (inode, this, &val);
 }
 
-uint64_t
-afr_is_split_brain (xlator_t *this, inode_t *inode)
+
+int
+__afr_inode_read_subvol_reset_small (inode_t *inode, xlator_t *this)
 {
-        int ret = 0;
+	int ret = -1;
+	uint16_t datamap = 0;
+	uint16_t metadatamap = 0;
+	uint32_t event = 0;
+	uint64_t val = 0;
 
-        uint64_t ctx         = 0;
-        uint64_t split_brain = 0;
+	ret = __inode_ctx_get (inode, this, &val);
+	(void) ret;
 
-        VALIDATE_OR_GOTO (inode, out);
+	metadatamap = (val & 0x000000000000ffff) >> 0;
+	datamap =     (val & 0x00000000ffff0000) >> 16;
+	event = 0;
 
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx);
+	val = ((uint64_t) metadatamap) |
+		(((uint64_t) datamap) << 16) |
+		(((uint64_t) event) << 32);
 
-                if (ret < 0)
-                        goto unlock;
+	return __inode_ctx_set (inode, this, &val);
+}
 
-                split_brain = ctx & AFR_ICTX_SPLIT_BRAIN_MASK;
-        }
-unlock:
-        UNLOCK (&inode->lock);
 
-out:
-        return split_brain;
+int
+__afr_inode_read_subvol_get (inode_t *inode, xlator_t *this,
+			     unsigned char *data, unsigned char *metadata,
+			     int *event_p)
+{
+	afr_private_t *priv = NULL;
+	int ret = -1;
+
+	priv = this->private;
+
+	if (priv->child_count <= 16)
+		ret = __afr_inode_read_subvol_get_small (inode, this, data,
+							 metadata, event_p);
+	else
+		/* TBD: allocate structure with array and read from it */
+		ret = -1;
+
+	return ret;
 }
 
 
-void
-afr_set_split_brain (xlator_t *this, inode_t *inode, gf_boolean_t set)
+int
+__afr_inode_read_subvol_set (inode_t *inode, xlator_t *this, unsigned char *data,
+			     unsigned char *metadata, int event)
 {
-        uint64_t ctx = 0;
-        int      ret = 0;
+	afr_private_t *priv = NULL;
+	int ret = -1;
 
-        VALIDATE_OR_GOTO (inode, out);
+	priv = this->private;
 
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx);
+	if (priv->child_count <= 16)
+		ret = __afr_inode_read_subvol_set_small (inode, this, data,
+							 metadata, event);
+	else
+		ret = -1;
 
-                if (ret < 0) {
-                        ctx = 0;
-                }
+	return ret;
+}
 
-                if (set) {
-                        ctx = (~AFR_ICTX_SPLIT_BRAIN_MASK & ctx)
-                                | (0xFFFFFFFFFFFFFFFFULL & AFR_ICTX_SPLIT_BRAIN_MASK);
-                } else {
-                        ctx = (~AFR_ICTX_SPLIT_BRAIN_MASK & ctx);
-                }
-                __inode_ctx_put (inode, this, ctx);
-        }
-        UNLOCK (&inode->lock);
-out:
-        return;
+
+int
+__afr_inode_read_subvol_reset (inode_t *inode, xlator_t *this)
+{
+	afr_private_t *priv = NULL;
+	int ret = -1;
+
+	priv = this->private;
+
+	if (priv->child_count <= 16)
+		ret = __afr_inode_read_subvol_reset_small (inode, this);
+	else
+		ret = -1;
+
+	return ret;
 }
 
 
-uint64_t
-afr_is_opendir_done (xlator_t *this, inode_t *inode)
+int
+afr_inode_read_subvol_get (inode_t *inode, xlator_t *this, unsigned char *data,
+			   unsigned char *metadata, int *event_p)
 {
-        int ret = 0;
+	int ret = -1;
 
-        uint64_t ctx          = 0;
-        uint64_t opendir_done = 0;
+	LOCK(&inode->lock);
+	{
+		ret = __afr_inode_read_subvol_get (inode, this, data,
+						   metadata, event_p);
+	}
+	UNLOCK(&inode->lock);
 
-        VALIDATE_OR_GOTO (inode, out);
+	return ret;
+}
 
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx);
 
-                if (ret < 0)
-                        goto unlock;
+int
+afr_inode_read_subvol_set (inode_t *inode, xlator_t *this, unsigned char *data,
+			   unsigned char *metadata, int event)
+{
+	int ret = -1;
 
-                opendir_done = ctx & AFR_ICTX_OPENDIR_DONE_MASK;
-        }
-unlock:
-        UNLOCK (&inode->lock);
+	LOCK(&inode->lock);
+	{
+		ret = __afr_inode_read_subvol_set (inode, this, data, metadata,
+						   event);
+	}
+	UNLOCK(&inode->lock);
 
-out:
-        return opendir_done;
+	return ret;
 }
 
 
-void
-afr_set_opendir_done (xlator_t *this, inode_t *inode)
+int
+afr_inode_read_subvol_reset (inode_t *inode, xlator_t *this)
 {
-        uint64_t ctx = 0;
-        int      ret = 0;
+	int ret = -1;
 
-        VALIDATE_OR_GOTO (inode, out);
+	LOCK(&inode->lock);
+	{
+		ret = __afr_inode_read_subvol_reset (inode, this);
+	}
+	UNLOCK(&inode->lock);
 
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx);
+	return ret;
+}
 
-                if (ret < 0) {
-                        ctx = 0;
-                }
 
-                ctx = (~AFR_ICTX_OPENDIR_DONE_MASK & ctx)
-                        | (0xFFFFFFFFFFFFFFFFULL & AFR_ICTX_OPENDIR_DONE_MASK);
+int
+afr_accused_fill (xlator_t *this, dict_t *xdata, unsigned char *accused,
+		  afr_transaction_type type)
+{
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int idx = afr_index_for_transaction_type (type);
+	void *pending_raw = NULL;
+	int pending[3];
+	int ret = 0;
+
+	priv = this->private;
+
+	for (i = 0; i < priv->child_count; i++) {
+		ret = dict_get_ptr (xdata, priv->pending_key[i],
+				    &pending_raw);
+		if (ret) /* no pending flags */
+			continue;
+		memcpy (pending, pending_raw, sizeof(pending));
+
+		if (ntoh32 (pending[idx]))
+			accused[i] = 1;
+	}
+
+	return 0;
+}
 
-                __inode_ctx_put (inode, this, ctx);
-        }
-        UNLOCK (&inode->lock);
-out:
-        return;
+
+int
+afr_accuse_smallfiles (xlator_t *this, struct afr_reply *replies,
+		       unsigned char *data_accused)
+{
+	int i = 0;
+	afr_private_t *priv = NULL;
+	uint64_t maxsize = 0;
+
+	priv = this->private;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (data_accused[i])
+			continue;
+		if (replies[i].poststat.ia_size > maxsize)
+			maxsize = replies[i].poststat.ia_size;
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (data_accused[i])
+			continue;
+		if (replies[i].poststat.ia_size < maxsize)
+			data_accused[i] = 1;
+	}
+
+	return 0;
 }
 
 
-uint64_t
-afr_read_child (xlator_t *this, inode_t *inode)
+int
+afr_replies_interpret (call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-        int ret = 0;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	struct afr_reply *replies = NULL;
+	int event_generation = 0;
+	int i = 0;
+	unsigned char *data_accused = NULL;
+	unsigned char *metadata_accused = NULL;
+	unsigned char *data_readable = NULL;
+	unsigned char *metadata_readable = NULL;
+	int ret = 0;
+
+	local = frame->local;
+	priv = this->private;
+	replies = local->replies;
+	event_generation = local->event_generation;
+
+	data_accused = alloca0 (priv->child_count);
+	data_readable = alloca0 (priv->child_count);
+	metadata_accused = alloca0 (priv->child_count);
+	metadata_readable = alloca0 (priv->child_count);
+
+	for (i = 0; i < priv->child_count; i++) {
+		data_readable[i] = 1;
+		metadata_readable[i] = 1;
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid) {
+			data_readable[i] = 0;
+			metadata_readable[i] = 0;
+			continue;
+		}
+
+		if (replies[i].op_ret == -1) {
+			data_readable[i] = 0;
+			metadata_readable[i] = 0;
+			continue;
+		}
+
+		afr_accused_fill (this, replies[i].xdata, data_accused,
+				  (inode->ia_type == IA_IFDIR) ?
+				   AFR_ENTRY_TRANSACTION : AFR_DATA_TRANSACTION);
+
+		afr_accused_fill (this, replies[i].xdata,
+				  metadata_accused, AFR_METADATA_TRANSACTION);
+
+	}
+
+	if (inode->ia_type != IA_IFDIR)
+		afr_accuse_smallfiles (this, replies, data_accused);
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (data_accused[i]) {
+			data_readable[i] = 0;
+			ret = 1;
+		}
+		if (metadata_accused[i]) {
+			metadata_readable[i] = 0;
+			ret = 1;
+		}
+	}
+
+	afr_inode_read_subvol_set (inode, this, data_readable,
+				   metadata_readable, event_generation);
+	return ret;
+}
 
-        uint64_t ctx         = 0;
-        uint64_t read_child  = 0;
 
-        VALIDATE_OR_GOTO (inode, out);
 
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx);
+int
+afr_refresh_selfheal_done (int ret, call_frame_t *heal, void *opaque)
+{
+	if (heal)
+		STACK_DESTROY (heal->root);
+	return 0;
+}
 
-                if (ret < 0)
-                        goto unlock;
+int
+afr_inode_refresh_err (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int err = 0;
+
+	local = frame->local;
+	priv = this->private;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (local->replies[i].valid && !local->replies[i].op_ret) {
+			err = 0;
+			goto ret;
+		}
+	}
+
+	err = afr_final_errno (local, priv);
+ret:
+	return -err;
+}
 
-                read_child = ctx & AFR_ICTX_READ_CHILD_MASK;
-        }
-unlock:
-        UNLOCK (&inode->lock);
 
-out:
-        return read_child;
+int
+afr_refresh_selfheal_wrap (void *opaque)
+{
+	call_frame_t *frame = opaque;
+	afr_local_t *local = NULL;
+	xlator_t *this = NULL;
+	int err = 0;
+
+	local = frame->local;
+	this = frame->this;
+
+	afr_selfheal (frame->this, local->refreshinode->gfid);
+
+	afr_selfheal_unlocked_discover (frame, local->refreshinode,
+					local->refreshinode->gfid,
+					local->replies);
+
+	afr_replies_interpret (frame, this, local->refreshinode);
+
+	err = afr_inode_refresh_err (frame, this);
+
+	afr_replies_wipe (local, this->private);
+
+	local->refreshfn (frame, this, err);
+
+	return 0;
 }
 
 
-void
-afr_set_read_child (xlator_t *this, inode_t *inode, int32_t read_child)
+gf_boolean_t
+afr_selfheal_enabled (xlator_t *this)
 {
-        uint64_t ctx = 0;
-        int      ret = 0;
+	afr_private_t *priv = NULL;
+	gf_boolean_t data = _gf_false;
 
-        VALIDATE_OR_GOTO (inode, out);
+	priv = this->private;
 
-        LOCK (&inode->lock);
-        {
-                ret = __inode_ctx_get (inode, this, &ctx);
+	gf_string2boolean (priv->data_self_heal, &data);
 
-                if (ret < 0) {
-                        ctx = 0;
-                }
+	return data || priv->metadata_self_heal || priv->entry_self_heal;
+}
 
-                ctx = (~AFR_ICTX_READ_CHILD_MASK & ctx)
-                        | (AFR_ICTX_READ_CHILD_MASK & read_child);
 
-                __inode_ctx_put (inode, this, ctx);
-        }
-        UNLOCK (&inode->lock);
 
-out:
-        return;
+int
+afr_inode_refresh_done (call_frame_t *frame, xlator_t *this)
+{
+	call_frame_t *heal = NULL;
+	afr_local_t *local = NULL;
+	int ret = 0;
+	int err = 0;
+
+	local = frame->local;
+
+	ret = afr_replies_interpret (frame, this, local->refreshinode);
+
+	err = afr_inode_refresh_err (frame, this);
+
+	afr_replies_wipe (local, this->private);
+
+	if (ret && afr_selfheal_enabled (this)) {
+		heal = copy_frame (frame);
+		if (heal)
+			heal->root->pid = -1;
+		ret = synctask_new (this->ctx->env, afr_refresh_selfheal_wrap,
+				    afr_refresh_selfheal_done, heal, frame);
+		if (ret)
+			goto refresh_done;
+	} else {
+	refresh_done:
+		local->refreshfn (frame, this, err);
+	}
+
+	return 0;
 }
 
 
-/**
- * afr_local_cleanup - cleanup everything in frame->local
- */
+int
+afr_inode_refresh_subvol_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			      int op_ret, int op_errno, inode_t *inode,
+			      struct iatt *buf, dict_t *xdata, struct iatt *par)
+{
+	afr_local_t *local = NULL;
+	int call_child = (long) cookie;
+	int call_count = 0;
 
-void
-afr_local_sh_cleanup (afr_local_t *local, xlator_t *this)
+	local = frame->local;
+
+	local->replies[call_child].valid = 1;
+	local->replies[call_child].op_ret = op_ret;
+	local->replies[call_child].op_errno = op_errno;
+	if (op_ret != -1) {
+		local->replies[call_child].poststat = *buf;
+		local->replies[call_child].postparent = *par;
+		local->replies[call_child].xdata = dict_ref (xdata);
+	}
+
+	call_count = afr_frame_return (frame);
+
+	if (call_count == 0)
+		afr_inode_refresh_done (frame, this);
+
+	return 0;
+}
+
+
+int
+afr_inode_refresh_subvol (call_frame_t *frame, xlator_t *this, int i,
+			  inode_t *inode, dict_t *xdata)
 {
-        afr_self_heal_t *sh = NULL;
-        afr_private_t   *priv = NULL;
-        int              i = 0;
+	loc_t loc = {0, };
+	afr_private_t *priv = NULL;
 
+	priv = this->private;
 
-        sh = &local->self_heal;
-        priv = this->private;
+	loc.inode = inode;
+	uuid_copy (loc.gfid, inode->gfid);
 
-        if (sh->buf)
-                GF_FREE (sh->buf);
+	STACK_WIND_COOKIE (frame, afr_inode_refresh_subvol_cbk,
+			   (void *) (long) i, priv->children[i],
+			   priv->children[i]->fops->lookup, &loc, xdata);
+	return 0;
+}
 
-        if (sh->xattr) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (sh->xattr[i]) {
-                                dict_unref (sh->xattr[i]);
-                                sh->xattr[i] = NULL;
-                        }
-                }
-                GF_FREE (sh->xattr);
+
+int
+afr_inode_refresh_do (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int call_count = 0;
+	int i = 0;
+	dict_t *xdata = NULL;
+
+	priv = this->private;
+	local = frame->local;
+
+	afr_replies_wipe (local, priv);
+
+	xdata = dict_new ();
+	if (!xdata) {
+		afr_inode_refresh_done (frame, this);
+		return 0;
+	}
+
+	if (afr_xattr_req_prepare (this, xdata) != 0) {
+		dict_unref (xdata);
+		afr_inode_refresh_done (frame, this);
+		return 0;
+	}
+
+	local->call_count = AFR_COUNT (local->child_up, priv->child_count);
+
+	call_count = local->call_count;
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->child_up[i])
+			continue;
+
+		afr_inode_refresh_subvol (frame, this, i, local->refreshinode,
+					  xdata);
+
+		if (!--call_count)
+			break;
+	}
+
+	dict_unref (xdata);
+
+	return 0;
+}
+
+
+int
+afr_inode_refresh (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		   afr_inode_refresh_cbk_t refreshfn)
+{
+	afr_local_t *local = NULL;
+
+	local = frame->local;
+
+	local->refreshfn = refreshfn;
+
+	if (local->refreshinode) {
+		inode_unref (local->refreshinode);
+		local->refreshinode = NULL;
+	}
+
+	local->refreshinode = inode_ref (inode);
+
+	afr_inode_refresh_do (frame, this);
+
+	return 0;
+}
+
+
+int
+afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req)
+{
+        int             i           = 0;
+        afr_private_t   *priv       = NULL;
+        int             ret         = 0;
+
+        priv   = this->private;
+
+        for (i = 0; i < priv->child_count; i++) {
+                ret = dict_set_uint64 (xattr_req, priv->pending_key[i],
+                                       AFR_NUM_CHANGE_LOGS * sizeof(int));
+                if (ret < 0)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Unable to set dict value for %s",
+                                priv->pending_key[i]);
+                /* 3 = data+metadata+entry */
+        }
+        ret = dict_set_uint64 (xattr_req, AFR_DIRTY,
+			       AFR_NUM_CHANGE_LOGS * sizeof(int));
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "failed to set dirty "
+                        "query flag");
         }
 
-        if (sh->child_errno)
-                GF_FREE (sh->child_errno);
+	return ret;
+}
 
-        if (sh->pending_matrix) {
-                for (i = 0; i < priv->child_count; i++) {
-                        GF_FREE (sh->pending_matrix[i]);
-                }
-                GF_FREE (sh->pending_matrix);
+int
+afr_lookup_xattr_req_prepare (afr_local_t *local, xlator_t *this,
+                              dict_t *xattr_req, loc_t *loc)
+{
+        int     ret = -ENOMEM;
+
+        local->xattr_req = dict_new ();
+        if (!local->xattr_req)
+                goto out;
+        if (xattr_req)
+                dict_copy (xattr_req, local->xattr_req);
+
+        ret = afr_xattr_req_prepare (this, local->xattr_req);
+	if (ret < 0) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"%s: Unable to prepare xattr_req", loc->path);
+	}
+
+        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_INODELK_COUNT, 0);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: Unable to set dict value for %s",
+                        loc->path, GLUSTERFS_INODELK_COUNT);
+        }
+        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_ENTRYLK_COUNT, 0);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: Unable to set dict value for %s",
+                        loc->path, GLUSTERFS_ENTRYLK_COUNT);
         }
 
-        if (sh->delta_matrix) {
-                for (i = 0; i < priv->child_count; i++) {
-                        GF_FREE (sh->delta_matrix[i]);
-                }
-                GF_FREE (sh->delta_matrix);
+        ret = dict_set_uint32 (local->xattr_req, GLUSTERFS_PARENT_ENTRYLK, 0);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: Unable to set dict value for %s",
+                        loc->path, GLUSTERFS_PARENT_ENTRYLK);
         }
 
-        if (sh->sources)
-                GF_FREE (sh->sources);
+        ret = 0;
+out:
+        return ret;
+}
 
-        if (sh->success)
-                GF_FREE (sh->success);
 
-        if (sh->locked_nodes)
-                GF_FREE (sh->locked_nodes);
+int
+afr_hash_child (inode_t *inode, int32_t child_count, int hashmode)
+{
+        uuid_t gfid_copy = {0,};
+        pid_t pid;
 
-        if (sh->healing_fd && !sh->healing_fd_opened) {
-                fd_unref (sh->healing_fd);
-                sh->healing_fd = NULL;
+        if (!hashmode) {
+                return -1;
         }
 
-        if (sh->linkname)
-                GF_FREE ((char *)sh->linkname);
+        if (inode) {
+               uuid_copy (gfid_copy, inode->gfid);
+        }
 
-        loc_wipe (&sh->parent_loc);
+        if (hashmode > 1) {
+                /*
+                 * Why getpid?  Because it's one of the cheapest calls
+                 * available - faster than gethostname etc. - and returns a
+                 * constant-length value that's sure to be shorter than a UUID.
+                 * It's still very unlikely to be the same across clients, so
+                 * it still provides good mixing.  We're not trying for
+                 * perfection here.  All we need is a low probability that
+                 * multiple clients won't converge on the same subvolume.
+                 */
+                pid = getpid();
+                memcpy (gfid_copy, &pid, sizeof(pid));
+        }
+
+        return SuperFastHash((char *)gfid_copy,
+                             sizeof(gfid_copy)) % child_count;
+}
+
+
+int
+afr_read_subvol_select_by_policy (inode_t *inode, xlator_t *this,
+				  unsigned char *readable)
+{
+	afr_private_t *priv = NULL;
+	int read_subvol = -1;
+	int i = 0;
+
+	priv = this->private;
+
+	/* first preference - explicitly specified or local subvolume */
+	if (priv->read_child >= 0 && readable[priv->read_child])
+		return priv->read_child;
+
+	/* second preference - use hashed mode */
+	read_subvol = afr_hash_child (inode, priv->child_count,
+				      priv->hash_mode);
+	if (read_subvol >= 0 && readable[read_subvol])
+		return read_subvol;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (readable[i])
+			return i;
+	}
+
+	/* no readable subvolumes, either split brain or all subvols down */
+
+	return -1;
+}
+
+
+int
+afr_inode_read_subvol_type_get (inode_t *inode, xlator_t *this,
+				unsigned char *readable, int *event_p,
+				int type)
+{
+	int ret = -1;
+
+	if (type == AFR_METADATA_TRANSACTION)
+		ret = afr_inode_read_subvol_get (inode, this, 0, readable,
+						 event_p);
+	else
+		ret = afr_inode_read_subvol_get (inode, this, readable, 0,
+						 event_p);
+	return ret;
+}
+
+
+int
+afr_read_subvol_get (inode_t *inode, xlator_t *this, int *subvol_p,
+		     int *event_p, afr_transaction_type type)
+{
+	afr_private_t *priv = NULL;
+	unsigned char *data_readable = NULL;
+	unsigned char *metadata_readable = NULL;
+	unsigned char *readable = NULL;
+	unsigned char *intersection = NULL;
+	int subvol = -1;
+	int event = 0;
+
+	priv = this->private;
+
+	readable = alloca0 (priv->child_count);
+	data_readable = alloca0 (priv->child_count);
+	metadata_readable = alloca0 (priv->child_count);
+	intersection = alloca0 (priv->child_count);
+
+	afr_inode_read_subvol_type_get (inode, this, readable, &event, type);
+
+	afr_inode_read_subvol_get (inode, this, data_readable, metadata_readable,
+				   &event);
+
+	AFR_INTERSECT (intersection, data_readable, metadata_readable,
+		       priv->child_count);
+
+	if (AFR_COUNT (intersection, priv->child_count) > 0)
+		subvol = afr_read_subvol_select_by_policy (inode, this,
+							   intersection);
+	else
+		subvol = afr_read_subvol_select_by_policy (inode, this,
+							   readable);
+	if (subvol_p)
+		*subvol_p = subvol;
+	if (event_p)
+		*event_p = event;
+	return subvol;
 }
 
 
 void
 afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this)
 {
-        int             i = 0;
-        afr_private_t * priv = NULL;
+        afr_private_t *priv    = NULL;
+        int           i        = 0;
 
         priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->pending && local->pending[i])
-                        GF_FREE (local->pending[i]);
-        }
+        afr_matrix_cleanup (local->pending, priv->child_count);
 
-        GF_FREE (local->pending);
+        GF_FREE (local->internal_lock.locked_nodes);
 
-        if (local->internal_lock.locked_nodes)
-                GF_FREE (local->internal_lock.locked_nodes);
-
-        if (local->internal_lock.inode_locked_nodes)
-                GF_FREE (local->internal_lock.inode_locked_nodes);
-
-        if (local->internal_lock.entry_locked_nodes)
-                GF_FREE (local->internal_lock.entry_locked_nodes);
+        for (i = 0; local->internal_lock.inodelk[i].domain; i++) {
+                GF_FREE (local->internal_lock.inodelk[i].locked_nodes);
+        }
 
-        if (local->internal_lock.lower_locked_nodes)
-                GF_FREE (local->internal_lock.lower_locked_nodes);
+        GF_FREE (local->internal_lock.lower_locked_nodes);
 
+        afr_entry_lockee_cleanup (&local->internal_lock);
 
-        GF_FREE (local->transaction.child_errno);
-        GF_FREE (local->child_errno);
+        GF_FREE (local->transaction.pre_op);
+        GF_FREE (local->transaction.eager_lock);
+        GF_FREE (local->transaction.fop_subvols);
+        GF_FREE (local->transaction.failed_subvols);
 
         GF_FREE (local->transaction.basename);
         GF_FREE (local->transaction.new_basename);
 
         loc_wipe (&local->transaction.parent_loc);
         loc_wipe (&local->transaction.new_parent_loc);
+
 }
 
 
 void
+afr_replies_wipe (afr_local_t *local, afr_private_t *priv)
+{
+	int i;
+
+	if (!local->replies)
+		return;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (local->replies[i].xdata) {
+			dict_unref (local->replies[i].xdata);
+			local->replies[i].xdata = NULL;
+		}
+	}
+
+	memset (local->replies, 0, sizeof(*local->replies) * priv->child_count);
+}
+
+void
+afr_remove_eager_lock_stub (afr_local_t *local)
+{
+        LOCK (&local->fd->lock);
+        {
+                list_del_init (&local->transaction.eager_locked);
+        }
+        UNLOCK (&local->fd->lock);
+}
+
+void
 afr_local_cleanup (afr_local_t *local, xlator_t *this)
 {
-        int i;
         afr_private_t * priv = NULL;
 
         if (!local)
                 return;
 
-        afr_local_sh_cleanup (local, this);
+	syncbarrier_destroy (&local->barrier);
+
+        if (local->transaction.eager_lock_on &&
+            !list_empty (&local->transaction.eager_locked))
+                afr_remove_eager_lock_stub (local);
 
         afr_local_transaction_cleanup (local, this);
 
@@ -369,37 +928,36 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
         if (local->xattr_req)
                 dict_unref (local->xattr_req);
 
+        if (local->dict)
+                dict_unref (local->dict);
+
+	afr_replies_wipe (local, priv);
+	GF_FREE(local->replies);
+
         GF_FREE (local->child_up);
 
-        { /* lookup */
-                if (local->cont.lookup.xattrs) {
-                        for (i = 0; i < priv->child_count; i++) {
-                                if (local->cont.lookup.xattrs[i]) {
-                                        dict_unref (local->cont.lookup.xattrs[i]);
-                                        local->cont.lookup.xattrs[i] = NULL;
-                                }
-                        }
-                        GF_FREE (local->cont.lookup.xattrs);
-                        local->cont.lookup.xattrs = NULL;
-                }
+        GF_FREE (local->read_attempted);
 
-                if (local->cont.lookup.xattr) {
-                        dict_unref (local->cont.lookup.xattr);
-                }
+        GF_FREE (local->readable);
 
-                if (local->cont.lookup.inode) {
-                        inode_unref (local->cont.lookup.inode);
-                }
-        }
+	if (local->inode)
+		inode_unref (local->inode);
+
+	if (local->parent)
+		inode_unref (local->parent);
+
+	if (local->parent2)
+		inode_unref (local->parent2);
+
+	if (local->refreshinode)
+		inode_unref (local->refreshinode);
 
         { /* getxattr */
-                if (local->cont.getxattr.name)
-                        GF_FREE (local->cont.getxattr.name);
+                GF_FREE (local->cont.getxattr.name);
         }
 
         { /* lk */
-                if (local->cont.lk.locked_nodes)
-                        GF_FREE (local->cont.lk.locked_nodes);
+                GF_FREE (local->cont.lk.locked_nodes);
         }
 
         { /* create */
@@ -426,6 +984,8 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
 
         { /* writev */
                 GF_FREE (local->cont.writev.vector);
+                if (local->cont.writev.iobref)
+                        iobref_unref (local->cont.writev.iobref);
         }
 
         { /* setxattr */
@@ -433,18 +993,40 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this)
                         dict_unref (local->cont.setxattr.dict);
         }
 
+        { /* fsetxattr */
+                if (local->cont.fsetxattr.dict)
+                        dict_unref (local->cont.fsetxattr.dict);
+        }
+
         { /* removexattr */
                 GF_FREE (local->cont.removexattr.name);
         }
-
+        { /* xattrop */
+                if (local->cont.xattrop.xattr)
+                        dict_unref (local->cont.xattrop.xattr);
+        }
+        { /* fxattrop */
+                if (local->cont.fxattrop.xattr)
+                        dict_unref (local->cont.fxattrop.xattr);
+        }
         { /* symlink */
                 GF_FREE (local->cont.symlink.linkpath);
         }
 
         { /* opendir */
-                if (local->cont.opendir.checksum)
-                        GF_FREE (local->cont.opendir.checksum);
+                GF_FREE (local->cont.opendir.checksum);
+        }
+
+        { /* readdirp */
+                if (local->cont.readdir.dict)
+                        dict_unref (local->cont.readdir.dict);
         }
+
+        if (local->xdata_req)
+                dict_unref (local->xdata_req);
+
+        if (local->xdata_rsp)
+                dict_unref (local->xdata_rsp);
 }
 
 
@@ -466,949 +1048,1060 @@ afr_frame_return (call_frame_t *frame)
 }
 
 
-/**
- * up_children_count - return the number of children that are up
- */
-
-int
-afr_up_children_count (int child_count, unsigned char *child_up)
+gf_boolean_t
+afr_is_entry_possibly_under_txn (afr_local_t *local, xlator_t *this)
 {
-        int i   = 0;
-        int ret = 0;
+	int i = 0;
+	int tmp = 0;
+	afr_private_t *priv = NULL;
 
-        for (i = 0; i < child_count; i++)
-                if (child_up[i])
-                        ret++;
-        return ret;
-}
+	priv = this->private;
 
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->replies[i].xdata)
+			continue;
+		if (dict_get_int32 (local->replies[i].xdata,
+				    GLUSTERFS_PARENT_ENTRYLK,
+				    &tmp) == 0)
+			if (tmp)
+				return _gf_true;
+	}
 
-ino64_t
-afr_itransform (ino64_t ino, int child_count, int child_index)
-{
-        ino64_t scaled_ino = -1;
+	return _gf_false;
+}
 
-        if (ino == ((uint64_t) -1)) {
-                scaled_ino = ((uint64_t) -1);
-                goto out;
-        }
 
-        scaled_ino = (ino * child_count) + child_index;
+/*
+ * Quota size xattrs are not maintained by afr. There is a
+ * possibility that they differ even when both the directory changelog xattrs
+ * suggest everything is fine. So if there is at least one 'source' check among
+ * the sources which has the maximum quota size. Otherwise check among all the
+ * available ones for maximum quota size. This way if there is a source and
+ * stale copies it always votes for the 'source'.
+ * */
 
-out:
-        return scaled_ino;
+static void
+afr_handle_quota_size (call_frame_t *frame, xlator_t *this)
+{
+	unsigned char *readable = NULL;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	struct afr_reply *replies = NULL;
+	int i = 0;
+	uint64_t size = 0;
+	uint64_t max_size = 0;
+	int readable_cnt = 0;
+
+	local = frame->local;
+	priv = this->private;
+	replies = local->replies;
+
+	readable = alloca0 (priv->child_count);
+
+	afr_inode_read_subvol_get (local->inode, this, readable, 0, 0);
+
+	readable_cnt = AFR_COUNT (readable, priv->child_count);
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid || replies[i].op_ret == -1)
+			continue;
+		if (readable_cnt && !readable[i])
+			continue;
+		if (!replies[i].xdata)
+			continue;
+		if (dict_get_uint64 (replies[i].xdata, QUOTA_SIZE_KEY, &size))
+			continue;
+		if (size > max_size)
+			max_size = size;
+	}
+
+	if (!max_size)
+		return;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid || replies[i].op_ret == -1)
+			continue;
+		if (readable_cnt && !readable[i])
+			continue;
+		if (!replies[i].xdata)
+			continue;
+		if (dict_set_uint64 (replies[i].xdata, QUOTA_SIZE_KEY, max_size))
+			continue;
+	}
 }
 
 
-int
-afr_deitransform_orig (ino64_t ino, int child_count)
+static void
+afr_lookup_done (call_frame_t *frame, xlator_t *this)
 {
-        int index = -1;
+        afr_private_t       *priv  = NULL;
+        afr_local_t         *local = NULL;
+	int                 i = -1;
+	int                 op_errno = 0;
+	int                 read_subvol = 0;
+	unsigned char      *readable = NULL;
+	int                 event = 0;
+	struct afr_reply   *replies = NULL;
+	uuid_t              read_gfid = {0, };
+	gf_boolean_t        locked_entry = _gf_false;
+	gf_boolean_t        can_interpret = _gf_true;
+
+        priv  = this->private;
+        local = frame->local;
+	replies = local->replies;
 
-        index = ino % child_count;
+	locked_entry = afr_is_entry_possibly_under_txn (local, this);
 
-        return index;
+	readable = alloca0 (priv->child_count);
+
+	afr_inode_read_subvol_get (local->loc.parent, this, readable,
+				   NULL, &event);
+
+	/* First, check if we have an ESTALE from somewhere,
+	   If so, propagate that so that a revalidate can be
+	   issued
+	*/
+	op_errno = afr_final_errno (frame->local, this->private);
+	local->op_errno = op_errno;
+        if (op_errno == ESTALE) {
+		local->op_errno = op_errno;
+		local->op_ret = -1;
+                goto unwind;
+	}
+
+	read_subvol = -1;
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (locked_entry && replies[i].op_ret == -1 &&
+		    replies[i].op_errno == ENOENT) {
+			/* Second, check entry is still
+			   "underway" in creation */
+			local->op_ret = -1;
+			local->op_errno = ENOENT;
+			read_subvol = i;
+			goto unwind;
+		}
+
+		if (replies[i].op_ret == -1)
+			continue;
+
+		if (read_subvol == -1 || !readable[read_subvol]) {
+			read_subvol = i;
+			uuid_copy (read_gfid, replies[i].poststat.ia_gfid);
+			local->op_ret = 0;
+		}
+	}
+
+	if (read_subvol == -1)
+		goto unwind;
+	/* We now have a read_subvol, which is readable[] (if there
+	   were any). Next we look for GFID mismatches. We don't
+	   consider a GFID mismatch as an error if read_subvol is
+	   readable[] but the mismatching GFID subvol is not.
+	*/
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid || replies[i].op_ret == -1) {
+			if (priv->child_up[i])
+				can_interpret = _gf_false;
+			continue;
+		}
+
+		if (!uuid_compare (replies[i].poststat.ia_gfid,
+				   read_gfid))
+			continue;
+
+		can_interpret = _gf_false;
+
+		if (locked_entry)
+			continue;
+
+		/* Now GFIDs mismatch. It's OK as long as this subvol
+		   is not readable[] but read_subvol is */
+		if (readable[read_subvol] && !readable[i])
+			continue;
+
+		/* LOG ERROR */
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		goto unwind;
+	}
+
+	/* Forth, for the finalized GFID, pick the best subvolume
+	   to return stats from.
+	*/
+	if (can_interpret) {
+		/* It is safe to call afr_replies_interpret() because we have
+		   a response from all the UP subvolumes and all of them resolved
+		   to the same GFID
+		*/
+		if (afr_replies_interpret (frame, this, local->inode)) {
+			read_subvol = afr_data_subvol_get (local->inode, this,
+							   0, 0);
+			afr_inode_read_subvol_reset (local->inode, this);
+			goto cant_interpret;
+		} else {
+			read_subvol = afr_data_subvol_get (local->inode, this,
+							   0, 0);
+		}
+	} else {
+	cant_interpret:
+		if (read_subvol == -1)
+			dict_del (replies[0].xdata, GF_CONTENT_KEY);
+		else
+			dict_del (replies[read_subvol].xdata, GF_CONTENT_KEY);
+	}
+
+	afr_handle_quota_size (frame, this);
+
+unwind:
+	if (read_subvol == -1)
+		read_subvol = 0;
+
+	AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
+			  local->inode, &local->replies[read_subvol].poststat,
+			  local->replies[read_subvol].xdata,
+			  &local->replies[read_subvol].postparent);
 }
 
+/*
+ * During a lookup, some errors are more "important" than
+ * others in that they must be given higher priority while
+ * returning to the user.
+ *
+ * The hierarchy is ESTALE > ENOENT > others
+ */
 
 int
-afr_deitransform (ino64_t ino, int child_count)
+afr_higher_errno (int32_t old_errno, int32_t new_errno)
 {
-        return 0;
+	if (old_errno == ENODATA || new_errno == ENODATA)
+		return ENODATA;
+	if (old_errno == ESTALE || new_errno == ESTALE)
+		return ESTALE;
+	if (old_errno == ENOENT || new_errno == ENOENT)
+		return ENOENT;
+
+	return new_errno;
 }
 
 
 int
-afr_self_heal_lookup_unwind (call_frame_t *frame, xlator_t *this)
+afr_final_errno (afr_local_t *local, afr_private_t *priv)
 {
-        afr_local_t *local = NULL;
+	int i = 0;
+	int op_errno = 0;
+	int tmp_errno = 0;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->replies[i].valid)
+			continue;
+		if (local->replies[i].op_ret == 0)
+			continue;
+		tmp_errno = local->replies[i].op_errno;
+		op_errno = afr_higher_errno (op_errno, tmp_errno);
+	}
+
+	return op_errno;
+}
 
-        local = frame->local;
+static int
+get_pathinfo_host (char *pathinfo, char *hostname, size_t size)
+{
+        char    *start = NULL;
+        char    *end = NULL;
+        int     ret  = -1;
+        int     i    = 0;
 
-        if (local->govinda_gOvinda) {
-                afr_set_split_brain (this, local->cont.lookup.inode, _gf_true);
-        }
+        if (!pathinfo)
+                goto out;
 
-        AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-                          local->cont.lookup.inode,
-                          &local->cont.lookup.buf,
-                          local->cont.lookup.xattr,
-                          &local->cont.lookup.postparent);
+        start = strchr (pathinfo, ':');
+        if (!start)
+                goto out;
+        end = strrchr (pathinfo, ':');
+        if (start == end)
+                goto out;
 
-        return 0;
+        memset (hostname, 0, size);
+        i = 0;
+        while (++start != end)
+                hostname[i++] = *start;
+        ret = 0;
+out:
+        return ret;
 }
 
-
-static void
-afr_lookup_collect_xattr (afr_local_t *local, xlator_t *this,
-                          int child_index, dict_t *xattr)
+int
+afr_local_pathinfo (char *pathinfo, gf_boolean_t *local)
 {
-        uint32_t        inodelk_count = 0;
-        uint32_t        entrylk_count = 0;
-
-        int ret = 0;
-
-        if (afr_sh_has_metadata_pending (xattr, child_index, this)) {
-                local->self_heal.need_metadata_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "metadata self-heal is pending for %s.",
-                       local->loc.path);
-        }
-
-        if (afr_sh_has_entry_pending (xattr, child_index, this)) {
-                local->self_heal.need_entry_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "entry self-heal is pending for %s.", local->loc.path);
+        int             ret   = 0;
+        char            pathinfohost[1024] = {0};
+        char            localhost[1024] = {0};
+        xlator_t        *this = THIS;
+
+        *local = _gf_false;
+        ret = get_pathinfo_host (pathinfo, pathinfohost, sizeof (pathinfohost));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Invalid pathinfo: %s",
+                        pathinfo);
+                goto out;
         }
 
-        if (afr_sh_has_data_pending (xattr, child_index, this)) {
-                local->self_heal.need_data_self_heal = _gf_true;
-                gf_log(this->name, GF_LOG_DEBUG,
-                       "data self-heal is pending for %s.", local->loc.path);
+        ret = gethostname (localhost, sizeof (localhost));
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "gethostname() failed, "
+                        "reason: %s", strerror (errno));
+                goto out;
         }
 
-        ret = dict_get_uint32 (xattr, GLUSTERFS_INODELK_COUNT,
-                               &inodelk_count);
-        if (ret == 0)
-                local->inodelk_count += inodelk_count;
-
-        ret = dict_get_uint32 (xattr, GLUSTERFS_ENTRYLK_COUNT,
-                               &entrylk_count);
-        if (ret == 0)
-                local->entrylk_count += entrylk_count;
+        if (!strcmp (localhost, pathinfohost))
+                *local = _gf_true;
+out:
+        return ret;
 }
 
-
-static void
-afr_lookup_self_heal_check (xlator_t *this, afr_local_t *local,
-                            struct iatt *buf, struct iatt *lookup_buf)
+static int32_t
+afr_local_discovery_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			 int32_t op_ret, int32_t op_errno, dict_t *dict,
+			 dict_t *xdata)
 {
-        if (FILETYPE_DIFFERS (buf, lookup_buf)) {
-                /* mismatching filetypes with same name
-                */
-
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "filetype differs for %s ", local->loc.path);
+        int              ret            = 0;
+        char            *pathinfo       = NULL;
+        gf_boolean_t     is_local        = _gf_false;
+        afr_private_t   *priv           = NULL;
+        int32_t          child_index    = -1;
 
-                local->govinda_gOvinda = 1;
+        if (op_ret != 0) {
+                goto out;
         }
 
-        if (PERMISSION_DIFFERS (buf, lookup_buf)) {
-                /* mismatching permissions */
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "permissions differ for %s ", local->loc.path);
-                local->self_heal.need_metadata_self_heal = _gf_true;
-        }
+	priv = this->private;
+	child_index = (int32_t)(long)cookie;
 
-        if (OWNERSHIP_DIFFERS (buf, lookup_buf)) {
-                /* mismatching permissions */
-                local->self_heal.need_metadata_self_heal = _gf_true;
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "ownership differs for %s ", local->loc.path);
+        ret = dict_get_str (dict, GF_XATTR_PATHINFO_KEY, &pathinfo);
+        if (ret != 0) {
+                goto out;
         }
 
-        if (SIZE_DIFFERS (buf, lookup_buf)
-            && IA_ISREG (buf->ia_type)) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "size differs for %s ", local->loc.path);
-                local->self_heal.need_data_self_heal = _gf_true;
+        ret = afr_local_pathinfo (pathinfo, &is_local);
+        if (ret) {
+                goto out;
         }
 
+        /*
+         * Note that one local subvolume will override another here.  The only
+         * way to avoid that would be to retain extra information about whether
+         * the previous read_child is local, and it's just not worth it.  Even
+         * the slowest local subvolume is far preferable to a remote one.
+         */
+        if (is_local) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "selecting local read_child %s",
+                        priv->children[child_index]->name);
+                priv->read_child = child_index;
+        }
+out:
+        STACK_DESTROY(frame->root);
+        return 0;
 }
 
-
 static void
-afr_lookup_done (call_frame_t *frame, xlator_t *this, struct iatt *lookup_buf)
+afr_attempt_local_discovery (xlator_t *this, int32_t child_index)
 {
-        int  unwind = 1;
-        int  source = -1;
-        int  up_count = 0;
-        char sh_type_str[256] = {0,};
+        call_frame_t    *newframe = NULL;
+        loc_t            tmploc = {0,};
+        afr_private_t   *priv = this->private;
 
-        afr_private_t *priv  = NULL;
-        afr_local_t   *local = NULL;
-
-        priv  = this->private;
-        local = frame->local;
-
-        local->cont.lookup.postparent.ia_ino  = local->cont.lookup.parent_ino;
-
-        if (local->cont.lookup.ino) {
-                local->cont.lookup.buf.ia_ino = local->cont.lookup.ino;
-        }
-
-        up_count = afr_up_children_count (priv->child_count, priv->child_up);
-        if (up_count == 1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Only 1 child up - do not attempt to detect self heal");
-                goto unwind;
-        }
-
-        if (local->op_ret == 0) {
-                /* KLUDGE: assuming DHT will not itransform in
-                   revalidate */
-                if (local->cont.lookup.inode->ino) {
-                        local->cont.lookup.buf.ia_ino =
-                                local->cont.lookup.inode->ino;
-                }
-        }
-
-        if (local->success_count && local->enoent_count) {
-                local->self_heal.need_metadata_self_heal = _gf_true;
-                local->self_heal.need_data_self_heal     = _gf_true;
-                local->self_heal.need_entry_self_heal    = _gf_true;
-                gf_log(this->name, GF_LOG_NORMAL,
-                       "entries are missing in lookup of %s.",
-                       local->loc.path);
-        }
-
-        if (local->success_count) {
-                /* check for split-brain case in previous lookup */
-                if (afr_is_split_brain (this,
-                                        local->cont.lookup.inode)) {
-                        local->self_heal.need_data_self_heal = _gf_true;
-                        gf_log(this->name, GF_LOG_NORMAL,
-                               "split brain detected during lookup of "
-                               "%s.", local->loc.path);
-                }
+        newframe = create_frame(this,this->ctx->pool);
+        if (!newframe) {
+                return;
         }
 
-        if ((local->self_heal.need_metadata_self_heal
-             || local->self_heal.need_data_self_heal
-             || local->self_heal.need_entry_self_heal)
-            && ((!local->cont.lookup.is_revalidate)
-                || (local->op_ret != -1))) {
-
-                if (local->inodelk_count || local->entrylk_count) {
-
-                        /* Someone else is doing self-heal on this file.
-                           So just make a best effort to set the read-subvolume
-                           and return */
+        tmploc.gfid[sizeof(tmploc.gfid)-1] = 1;
+        STACK_WIND_COOKIE (newframe, afr_local_discovery_cbk,
+                           (void *)(long)child_index,
+                           priv->children[child_index],
+                           priv->children[child_index]->fops->getxattr,
+                           &tmploc, GF_XATTR_PATHINFO_KEY, NULL);
+}
 
-                        if (IA_ISREG (local->cont.lookup.inode->ia_type)) {
-                                source = afr_self_heal_get_source (this, local, local->cont.lookup.xattrs);
 
-                                if (source >= 0) {
-                                        afr_set_read_child (this,
-                                                            local->cont.lookup.inode,
-                                                            source);
-                                }
-                        }
-                } else {
-                        if (!local->cont.lookup.inode->ia_type) {
-                                /* fix for RT #602 */
-                                local->cont.lookup.inode->ia_type =
-                                        lookup_buf->ia_type;
-                        }
-
-                        local->self_heal.background = _gf_true;
-                        local->self_heal.type       = local->cont.lookup.buf.ia_type;
-                        local->self_heal.unwind     = afr_self_heal_lookup_unwind;
+int
+afr_lookup_selfheal_wrap (void *opaque)
+{
+	call_frame_t *frame = opaque;
+	afr_local_t *local = NULL;
+	xlator_t *this = NULL;
+	inode_t *inode = NULL;
 
-                        unwind = 0;
+	local = frame->local;
+	this = frame->this;
 
-                        afr_self_heal_type_str_get(&local->self_heal,
-                                                   sh_type_str,
-                                                   sizeof(sh_type_str));
+	afr_selfheal_name (frame->this, local->loc.pargfid, local->loc.name);
 
-                        gf_log (this->name, GF_LOG_NORMAL, "background %s "
-                                "self-heal triggered. path: %s",
-                                sh_type_str, local->loc.path);
+	afr_replies_wipe (local, this->private);
 
-                        afr_self_heal (frame, this);
-                }
-        }
+	inode = afr_selfheal_unlocked_lookup_on (frame, local->loc.parent,
+						 local->loc.name, local->replies,
+						 local->child_up);
+	if (inode)
+		inode_unref (inode);
+	afr_lookup_done (frame, this);
 
-unwind:
-        if (unwind) {
-                AFR_STACK_UNWIND (lookup, frame, local->op_ret,
-                                  local->op_errno,
-                                  local->cont.lookup.inode,
-                                  &local->cont.lookup.buf,
-                                  local->cont.lookup.xattr,
-                                  &local->cont.lookup.postparent);
-        }
+	return 0;
 }
 
 
-/*
- * During a lookup, some errors are more "important" than
- * others in that they must be given higher priority while
- * returning to the user.
- *
- * The hierarchy is ESTALE > ENOENT > others
- *
- */
-
-static gf_boolean_t
-__error_more_important (int32_t old_errno, int32_t new_errno)
+int
+afr_lookup_entry_heal (call_frame_t *frame, xlator_t *this)
 {
-        gf_boolean_t ret = _gf_true;
-
-        /* Nothing should ever overwrite ESTALE */
-        if (old_errno == ESTALE)
-                ret = _gf_false;
-
-        /* Nothing should overwrite ENOENT, except ESTALE */
-        else if ((old_errno == ENOENT) && (new_errno != ESTALE))
-                ret = _gf_false;
-
-        return ret;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	call_frame_t *heal = NULL;
+	int i = 0, first = -1;
+	gf_boolean_t need_heal = _gf_false;
+	struct afr_reply *replies = NULL;
+	int ret = 0;
+
+	local = frame->local;
+	replies = local->replies;
+	priv = this->private;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (first == -1) {
+			first = i;
+			continue;
+		}
+
+		if (replies[i].op_ret != replies[first].op_ret) {
+			need_heal = _gf_true;
+			break;
+		}
+
+		if (uuid_compare (replies[i].poststat.ia_gfid,
+				  replies[first].poststat.ia_gfid)) {
+			need_heal = _gf_true;
+			break;
+		}
+	}
+
+	if (need_heal) {
+		heal = copy_frame (frame);
+		if (heal)
+			heal->root->pid = -1;
+		ret = synctask_new (this->ctx->env, afr_lookup_selfheal_wrap,
+				    afr_refresh_selfheal_done, heal, frame);
+		if (ret)
+			goto lookup_done;
+	} else {
+	lookup_done:
+		afr_lookup_done (frame, this);
+	}
+
+	return ret;
 }
 
 
 int
-afr_fresh_lookup_cbk (call_frame_t *frame, void *cookie,
-                      xlator_t *this,  int32_t op_ret,  int32_t op_errno,
-                      inode_t *inode,   struct iatt *buf, dict_t *xattr,
-                      struct iatt *postparent)
+afr_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
+		dict_t *xdata, struct iatt *postparent)
 {
         afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        struct iatt *   lookup_buf = NULL;
-
         int             call_count      = -1;
         int             child_index     = -1;
-        int             first_up_child  = -1;
 
-        child_index = (long) cookie;
-        priv = this->private;
+	child_index = (long) cookie;
 
-        LOCK (&frame->lock);
-        {
-                local = frame->local;
+	local = frame->local;
 
-                lookup_buf = &local->cont.lookup.buf;
+	local->replies[child_index].valid = 1;
+	local->replies[child_index].op_ret = op_ret;
+	local->replies[child_index].op_errno = op_errno;
+	if (op_ret != -1) {
+		local->replies[child_index].poststat = *buf;
+		local->replies[child_index].postparent = *postparent;
+		if (xdata)
+			local->replies[child_index].xdata = dict_ref (xdata);
+	}
 
-                if (op_ret == -1) {
-                        if (op_errno == ENOENT)
-                                local->enoent_count++;
+        call_count = afr_frame_return (frame);
+        if (call_count == 0) {
+		afr_lookup_entry_heal (frame, this);
+        }
 
-                        if (__error_more_important (local->op_errno, op_errno))
-                                local->op_errno = op_errno;
+	return 0;
+}
 
-                        if (local->op_errno == ESTALE) {
-                                local->op_ret = -1;
-                        }
 
-                        goto unlock;
-                }
 
-                afr_lookup_collect_xattr (local, this, child_index, xattr);
+static void
+afr_discover_done (call_frame_t *frame, xlator_t *this)
+{
+        afr_private_t       *priv  = NULL;
+        afr_local_t         *local = NULL;
+	int                 i = -1;
+	int                 op_errno = 0;
+	int                 read_subvol = 0;
 
-                first_up_child = afr_first_up_child (priv);
+        priv  = this->private;
+        local = frame->local;
 
-                if (child_index == first_up_child) {
-                        local->cont.lookup.ino =
-                                afr_itransform (buf->ia_ino,
-                                                priv->child_count,
-                                                first_up_child);
-                }
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->replies[i].valid)
+			continue;
+		if (local->replies[i].op_ret == 0)
+			local->op_ret = 0;
+	}
 
-                if (local->success_count == 0) {
-                        if (local->op_errno != ESTALE)
-                                local->op_ret = op_ret;
+	op_errno = afr_final_errno (frame->local, this->private);
 
-                        local->cont.lookup.inode               = inode_ref (inode);
-                        local->cont.lookup.xattr               = dict_ref (xattr);
-                        local->cont.lookup.xattrs[child_index] = dict_ref (xattr);
-                        local->cont.lookup.postparent          = *postparent;
+        if (local->op_ret < 0) {
+		local->op_errno = op_errno;
+		local->op_ret = -1;
+                goto unwind;
+	}
 
-                        if (priv->first_lookup && inode->ino == 1) {
-                                gf_log (this->name, GF_LOG_NORMAL,
-                                        "added root inode");
-                                priv->root_inode = inode_ref (inode);
-                                priv->first_lookup = 0;
-                        }
+	afr_replies_interpret (frame, this, local->inode);
 
-                        *lookup_buf = *buf;
+	read_subvol = afr_data_subvol_get (local->inode, this, 0, 0);
+	if (read_subvol == -1) {
+		gf_log (this->name, GF_LOG_WARNING, "no read subvols for %s",
+			local->loc.path);
 
-                        lookup_buf->ia_ino = afr_itransform (buf->ia_ino,
-                                                             priv->child_count,
-                                                             child_index);
-                        if (priv->read_child >= 0) {
-                                afr_set_read_child (this,
-                                                    local->cont.lookup.inode,
-                                                    priv->read_child);
-                        } else {
-                                afr_set_read_child (this,
-                                                    local->cont.lookup.inode,
-                                                    child_index);
-                        }
+		for (i = 0; i < priv->child_count; i++) {
+			if (!local->replies[i].valid ||
+			    local->replies[i].op_ret == -1)
+				continue;
+			read_subvol = i;
+			break;
+		}
+	}
 
-                } else {
-                        afr_lookup_self_heal_check (this, local, buf, lookup_buf);
-
-                        if (child_index == local->read_child_index) {
-                                /*
-                                   lookup has succeeded on the read child.
-                                   So use its inode number
-                                */
-                                if (local->cont.lookup.xattr)
-                                        dict_unref (local->cont.lookup.xattr);
-
-                                local->cont.lookup.xattr = dict_ref (xattr);
-                                local->cont.lookup.xattrs[child_index] = dict_ref (xattr);
-                                local->cont.lookup.postparent          = *postparent;
-
-                                *lookup_buf = *buf;
-
-                                if (priv->read_child >= 0) {
-                                        afr_set_read_child (this,
-                                                            local->cont.lookup.inode,
-                                                            priv->read_child);
-                                } else {
-                                        afr_set_read_child (this,
-                                                            local->cont.lookup.inode,
-                                                            local->read_child_index);
-                                }
-                        }
+unwind:
+	if (read_subvol == -1)
+		read_subvol = 0;
 
-                }
+	AFR_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
+			  local->inode, &local->replies[read_subvol].poststat,
+			  local->replies[read_subvol].xdata,
+			  &local->replies[read_subvol].postparent);
+}
 
-                local->success_count++;
-        }
-unlock:
-        UNLOCK (&frame->lock);
 
-        call_count = afr_frame_return (frame);
+int
+afr_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		  int op_ret, int op_errno, inode_t *inode, struct iatt *buf,
+		  dict_t *xdata, struct iatt *postparent)
+{
+        afr_local_t *   local = NULL;
+        int             call_count      = -1;
+        int             child_index     = -1;
+
+	child_index = (long) cookie;
+
+	local = frame->local;
 
+	local->replies[child_index].valid = 1;
+	local->replies[child_index].op_ret = op_ret;
+	local->replies[child_index].op_errno = op_errno;
+	if (op_ret != -1) {
+		local->replies[child_index].poststat = *buf;
+		local->replies[child_index].postparent = *postparent;
+		if (xdata)
+			local->replies[child_index].xdata = dict_ref (xdata);
+	}
+
+        if (local->do_discovery && (op_ret == 0))
+                afr_attempt_local_discovery (this, child_index);
+
+        call_count = afr_frame_return (frame);
         if (call_count == 0) {
-                afr_lookup_done (frame, this, lookup_buf);
+               afr_discover_done (frame, this);
         }
 
-        return 0;
+	return 0;
 }
 
 
 int
-afr_revalidate_lookup_cbk (call_frame_t *frame, void *cookie,
-                           xlator_t *this, int32_t op_ret, int32_t op_errno,
-                           inode_t *inode, struct iatt *buf, dict_t *xattr,
-                           struct iatt *postparent)
+afr_discover_do (call_frame_t *frame, xlator_t *this, int err)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-        struct iatt *   lookup_buf = NULL;
+	int ret = 0;
+	int i = 0;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int call_count = 0;
 
-        int             call_count      = -1;
-        int             child_index     = -1;
-        int             first_up_child  = -1;
+	local = frame->local;
+	priv = this->private;
 
-        child_index = (long) cookie;
-        priv = this->private;
+	if (err) {
+		local->op_errno = -err;
+		ret = -1;
+		goto out;
+	}
 
-        LOCK (&frame->lock);
-        {
-                local = frame->local;
+	call_count = local->call_count = AFR_COUNT (local->child_up,
+						    priv->child_count);
 
-                lookup_buf = &local->cont.lookup.buf;
+        ret = afr_lookup_xattr_req_prepare (local, this, local->xattr_req,
+					    &local->loc);
+        if (ret) {
+                local->op_errno = -ret;
+		ret = -1;
+                goto out;
+        }
 
-                if (op_ret == -1) {
-                        if (op_errno == ENOENT)
-                                local->enoent_count++;
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND_COOKIE (frame, afr_discover_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->lookup,
+                                           &local->loc, local->xattr_req);
+                        if (!--call_count)
+                                break;
+                }
+        }
 
-                        if (__error_more_important (local->op_errno, op_errno))
-                            local->op_errno = op_errno;
+	return 0;
+out:
+	AFR_STACK_UNWIND (lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
+	return 0;
+}
 
-                            if (local->op_errno == ESTALE) {
-                                    local->op_ret = -1;
-                            }
 
-                            goto unlock;
-                }
+int
+afr_discover (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
+{
+	int op_errno = ENOMEM;
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
+	int event = 0;
 
-                afr_lookup_collect_xattr (local, this, child_index, xattr);
+	priv = this->private;
 
-                first_up_child = afr_first_up_child (priv);
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-                if (child_index == first_up_child) {
-                        local->cont.lookup.ino =
-                                afr_itransform (buf->ia_ino,
-                                                priv->child_count,
-                                                first_up_child);
-                }
+        if (!local->call_count) {
+                op_errno = ENOTCONN;
+                goto out;
+        }
 
-                /* in case of revalidate, we need to send stat of the
-                 * child whose stat was sent during the first lookup.
-                 * (so that time stamp does not vary with revalidate.
-                 * in case it is down, stat of the fist success will
-                 * be replied */
+	if (__is_root_gfid (loc->inode->gfid)) {
+		if (!this->itable)
+			this->itable = loc->inode->table;
+		if (!priv->root_inode)
+			priv->root_inode = inode_ref (loc->inode);
 
-                /* inode number should be preserved across revalidates */
+		if (priv->choose_local && !priv->did_discovery) {
+			/* Logic to detect which subvolumes of AFR are
+			   local, in order to prefer them for reads
+			*/
+			local->do_discovery = _gf_true;
+                        priv->did_discovery = _gf_true;
+                }
+	}
 
-                if (local->success_count == 0) {
-                        if (local->op_errno != ESTALE)
-                                local->op_ret = op_ret;
+        local->op = GF_FOP_LOOKUP;
 
-                        local->cont.lookup.inode               = inode_ref (inode);
-                        local->cont.lookup.xattr               = dict_ref (xattr);
-                        local->cont.lookup.xattrs[child_index] = dict_ref (xattr);
-                        local->cont.lookup.postparent          = *postparent;
+        loc_copy (&local->loc, loc);
 
-                        *lookup_buf = *buf;
+	local->inode = inode_ref (loc->inode);
 
-                        lookup_buf->ia_ino = afr_itransform (buf->ia_ino,
-                                                             priv->child_count,
-                                                             child_index);
+	if (xattr_req)
+		/* If xattr_req was null, afr_lookup_xattr_req_prepare() will
+		   allocate one for us */
+		local->xattr_req = dict_ref (xattr_req);
 
-                        if (priv->read_child >= 0) {
-                                afr_set_read_child (this,
-                                                    local->cont.lookup.inode,
-                                                    priv->read_child);
-                        } else {
-                                afr_set_read_child (this,
-                                                    local->cont.lookup.inode,
-                                                    child_index);
-                        }
+	if (uuid_is_null (loc->inode->gfid)) {
+		afr_discover_do (frame, this, 0);
+		return 0;
+	}
 
-                } else {
-                        afr_lookup_self_heal_check (this, local, buf, lookup_buf);
+	afr_read_subvol_get (loc->inode, this, NULL, &event,
+			     AFR_DATA_TRANSACTION);
 
-                        if (child_index == local->read_child_index) {
+	if (event != local->event_generation)
+		afr_inode_refresh (frame, this, loc->inode, afr_discover_do);
+	else
+		afr_discover_do (frame, this, 0);
 
-                                /*
-                                   lookup has succeeded on the read child.
-                                   So use its inode number
-                                */
+	return 0;
+out:
+	AFR_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+	return 0;
+}
 
-                                if (local->cont.lookup.xattr)
-                                        dict_unref (local->cont.lookup.xattr);
 
-                                local->cont.lookup.xattr               = dict_ref (xattr);
-                                local->cont.lookup.xattrs[child_index] = dict_ref (xattr);
-                                local->cont.lookup.postparent          = *postparent;
+int
+afr_lookup_do (call_frame_t *frame, xlator_t *this, int err)
+{
+	int ret = 0;
+	int i = 0;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int call_count = 0;
 
-                                *lookup_buf = *buf;
+	local = frame->local;
+	priv = this->private;
 
-                                if (priv->read_child >= 0) {
-                                        afr_set_read_child (this,
-                                                            local->cont.lookup.inode,
-                                                            priv->read_child);
-                                } else {
-                                        afr_set_read_child (this,
-                                                            local->cont.lookup.inode,
-                                                            local->read_child_index);
-                                }
-                        }
+	if (err < 0) {
+		local->op_errno = -err;
+		ret = -1;
+		goto out;
+	}
 
-                }
+	call_count = local->call_count = AFR_COUNT (local->child_up,
+						    priv->child_count);
 
-                local->success_count++;
+        ret = afr_lookup_xattr_req_prepare (local, this, local->xattr_req,
+					    &local->loc);
+        if (ret) {
+                local->op_errno = -ret;
+		ret = -1;
+                goto out;
         }
-unlock:
-        UNLOCK (&frame->lock);
-
-        call_count = afr_frame_return (frame);
 
-        if (call_count == 0) {
-                afr_lookup_done (frame, this, lookup_buf);
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND_COOKIE (frame, afr_lookup_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->lookup,
+                                           &local->loc, local->xattr_req);
+                        if (!--call_count)
+                                break;
+                }
         }
-
-        return 0;
+	return 0;
+out:
+	AFR_STACK_UNWIND (lookup, frame, -1, local->op_errno, 0, 0, 0, 0);
+	return 0;
 }
 
+/*
+ * afr_lookup()
+ *
+ * The goal here is to figure out what the element getting looked up is.
+ * i.e what is the GFID, inode type and a conservative estimate of the
+ * inode attributes are.
+ *
+ * As we lookup, operations may be underway on the entry name and the
+ * inode. In lookup() we are primarily concerned only with the entry
+ * operations. If the entry is getting unlinked or renamed, we detect
+ * what operation is underway by querying for on-going transactions and
+ * pending self-healing on the entry through xdata.
+ *
+ * If the entry is a file/dir, it may need self-heal and/or in a
+ * split-brain condition. Lookup is not the place to worry about these
+ * conditions. Outcast marking will naturally handle them in the read
+ * paths.
+ *
+ * Here is a brief goal of what we are trying to achieve:
+ *
+ * - LOOKUP on all subvolumes concurrently, querying on-going transaction
+ *   and pending self-heal info from the servers.
+ *
+ * - If all servers reply the same inode type and GFID, the overall call
+ *   MUST be a success.
+ *
+ * - If inode types or GFIDs mismatch, and there IS either an on-going
+ *   transaction or pending self-heal, inspect what the nature of the
+ *   transaction or pending heal is, and select the appropriate subvolume's
+ *   reply as the winner.
+ *
+ * - If inode types or GFIDs mismatch, and there are no on-going transactions
+ *   or pending self-heal on the entry name on any of the servers, fail the
+ *   lookup with EIO. Something has gone wrong beyond reasonable action.
+ */
 
 int
-afr_lookup (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, dict_t *xattr_req)
+afr_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xattr_req)
 {
-        afr_private_t *priv = NULL;
         afr_local_t   *local = NULL;
-        int            ret = -1;
-        int            i = 0;
-
-        fop_lookup_cbk_t callback;
-
-        int call_count = 0;
-
-        uint64_t       ctx;
-
         int32_t        op_errno = 0;
+	int            event = 0;
 
-        priv = this->private;
-
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+	if (!loc->parent) {
+		afr_discover (frame, this, loc, xattr_req);
+		return 0;
+	}
 
-        local->op_ret = -1;
+	if (__is_root_gfid (loc->parent->gfid)) {
+		if (!strcmp (loc->name, GF_REPLICATE_TRASH_DIR)) {
+			op_errno = EPERM;
+			goto out;
+		}
+	}
 
-        frame->local = local;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        if (!strcmp (loc->path, "/" GF_REPLICATE_TRASH_DIR)) {
-                op_errno = ENOENT;
+        if (!local->call_count) {
+                op_errno = ENOTCONN;
                 goto out;
         }
 
+        local->op = GF_FOP_LOOKUP;
+
         loc_copy (&local->loc, loc);
 
-        ret = inode_ctx_get (loc->inode, this, &ctx);
-        if (ret == 0) {
-                /* lookup is a revalidate */
+	local->inode = inode_ref (loc->inode);
 
-                callback = afr_revalidate_lookup_cbk;
+	if (xattr_req)
+		/* If xattr_req was null, afr_lookup_xattr_req_prepare() will
+		   allocate one for us */
+		local->xattr_req = dict_ref (xattr_req);
 
-                local->cont.lookup.is_revalidate = _gf_true;
-                local->read_child_index          = afr_read_child (this,
-                                                                   loc->inode);
-        } else {
-                callback = afr_fresh_lookup_cbk;
+	afr_read_subvol_get (loc->parent, this, NULL, &event,
+			     AFR_DATA_TRANSACTION);
 
-                LOCK (&priv->read_child_lock);
-                {
-                        local->read_child_index = (++priv->read_child_rr)
-                                % (priv->child_count);
-                }
-                UNLOCK (&priv->read_child_lock);
-        }
+	if (event != local->event_generation)
+		afr_inode_refresh (frame, this, loc->parent, afr_lookup_do);
+	else
+		afr_lookup_do (frame, this, 0);
 
-        if (loc->parent)
-                local->cont.lookup.parent_ino = loc->parent->ino;
+	return 0;
+out:
+	AFR_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
 
-        local->child_up = memdup (priv->child_up, priv->child_count);
+        return 0;
+}
 
-        local->cont.lookup.xattrs = GF_CALLOC (priv->child_count,
-                                    sizeof (*local->cont.lookup.xattr),
-                                    gf_afr_mt_dict_t);
 
-        local->call_count = afr_up_children_count (priv->child_count,
-                                                   local->child_up);
-        call_count = local->call_count;
+/* {{{ open */
 
-        if (local->call_count == 0) {
-                ret      = -1;
-                op_errno = ENOTCONN;
-                goto out;
-        }
+afr_fd_ctx_t *
+__afr_fd_ctx_get (fd_t *fd, xlator_t *this)
+{
+        uint64_t       ctx = 0;
+        int            ret = 0;
+        afr_fd_ctx_t  *fd_ctx = NULL;
 
-        /* By default assume ENOTCONN. On success it will be set to 0. */
-        local->op_errno = ENOTCONN;
+        ret = __fd_ctx_get (fd, this, &ctx);
 
-        if (xattr_req == NULL)
-                local->xattr_req = dict_new ();
-        else
-                local->xattr_req = dict_ref (xattr_req);
+        if (ret < 0) {
+                ret = __afr_fd_ctx_set (this, fd);
+                if (ret < 0)
+                        goto out;
 
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_set_uint64 (local->xattr_req, priv->pending_key[i],
-                                       3 * sizeof(int32_t));
+                ret = __fd_ctx_get (fd, this, &ctx);
                 if (ret < 0)
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Unable to set dict value.");
-                /* 3 = data+metadata+entry */
+                        goto out;
         }
 
-        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_INODELK_COUNT, 0);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Unable to set dict value.");
-        }
+        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+out:
+        return fd_ctx;
+}
 
-        ret = dict_set_uint64 (local->xattr_req, GLUSTERFS_ENTRYLK_COUNT, 0);
-        if (ret < 0) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Unable to set dict value.");
-        }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, callback, (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->lookup,
-                                           loc, local->xattr_req);
-                        if (!--call_count)
-                                break;
-                }
-        }
+afr_fd_ctx_t *
+afr_fd_ctx_get (fd_t *fd, xlator_t *this)
+{
+        afr_fd_ctx_t  *fd_ctx = NULL;
 
-        ret = 0;
-out:
-        if (ret == -1)
-                AFR_STACK_UNWIND (lookup, frame, -1, op_errno,
-                                  NULL, NULL, NULL, NULL);
+        LOCK(&fd->lock);
+        {
+                fd_ctx = __afr_fd_ctx_get (fd, this);
+        }
+        UNLOCK(&fd->lock);
 
-        return 0;
+        return fd_ctx;
 }
 
 
-/* {{{ open */
-
 int
-afr_fd_ctx_set (xlator_t *this, fd_t *fd)
+__afr_fd_ctx_set (xlator_t *this, fd_t *fd)
 {
-        afr_private_t * priv = NULL;
-
-        int ret    = 0;
-
-        uint64_t       ctx;
-        afr_fd_ctx_t * fd_ctx = NULL;
+        afr_private_t * priv   = NULL;
+        int             ret    = -1;
+        uint64_t        ctx    = 0;
+        afr_fd_ctx_t *  fd_ctx = NULL;
+	int             i = 0;
 
         VALIDATE_OR_GOTO (this->private, out);
         VALIDATE_OR_GOTO (fd, out);
 
         priv = this->private;
 
-        LOCK (&fd->lock);
-        {
-                ret = __fd_ctx_get (fd, this, &ctx);
-
-                if (ret == 0)
-                        goto unlock;
+        ret = __fd_ctx_get (fd, this, &ctx);
 
-                fd_ctx = GF_CALLOC (1, sizeof (afr_fd_ctx_t),
-                                    gf_afr_mt_afr_fd_ctx_t);
-                if (!fd_ctx) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
+        if (ret == 0)
+                goto out;
 
-                        ret = -ENOMEM;
-                        goto unlock;
-                }
+        fd_ctx = GF_CALLOC (1, sizeof (afr_fd_ctx_t),
+                            gf_afr_mt_afr_fd_ctx_t);
+        if (!fd_ctx) {
+                ret = -ENOMEM;
+                goto out;
+        }
 
-                fd_ctx->pre_op_done = GF_CALLOC (sizeof (*fd_ctx->pre_op_done),
-                                                 priv->child_count,
-                                                 gf_afr_mt_char);
-                if (!fd_ctx->pre_op_done) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
-                        ret = -ENOMEM;
-                        goto unlock;
-                }
+	for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++) {
+		fd_ctx->pre_op_done[i] = GF_CALLOC (sizeof (*fd_ctx->pre_op_done[i]),
+						    priv->child_count,
+						    gf_afr_mt_int32_t);
+		if (!fd_ctx->pre_op_done[i]) {
+			ret = -ENOMEM;
+			goto out;
+		}
+	}
+
+        fd_ctx->opened_on = GF_CALLOC (sizeof (*fd_ctx->opened_on),
+                                       priv->child_count,
+                                       gf_afr_mt_int32_t);
+        if (!fd_ctx->opened_on) {
+                ret = -ENOMEM;
+                goto out;
+        }
 
-                fd_ctx->pre_op_piggyback = GF_CALLOC (sizeof (*fd_ctx->pre_op_piggyback),
-                                                      priv->child_count,
-                                                      gf_afr_mt_char);
-                if (!fd_ctx->pre_op_piggyback) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
-                        ret = -ENOMEM;
-                        goto unlock;
-                }
+	for (i = 0; i < priv->child_count; i++) {
+		if (fd_is_anonymous (fd))
+			fd_ctx->opened_on[i] = AFR_FD_OPENED;
+		else
+			fd_ctx->opened_on[i] = AFR_FD_NOT_OPENED;
+	}
 
-                fd_ctx->opened_on = GF_CALLOC (sizeof (*fd_ctx->opened_on),
-                                               priv->child_count,
-                                               gf_afr_mt_char);
-                if (!fd_ctx->opened_on) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
-                        ret = -ENOMEM;
-                        goto unlock;
-                }
+        fd_ctx->lock_piggyback = GF_CALLOC (sizeof (*fd_ctx->lock_piggyback),
+                                            priv->child_count,
+                                            gf_afr_mt_char);
+        if (!fd_ctx->lock_piggyback) {
+                ret = -ENOMEM;
+                goto out;
+        }
 
-                fd_ctx->up_count   = priv->up_count;
-                fd_ctx->down_count = priv->down_count;
+        fd_ctx->lock_acquired = GF_CALLOC (sizeof (*fd_ctx->lock_acquired),
+                                           priv->child_count,
+                                           gf_afr_mt_char);
+        if (!fd_ctx->lock_acquired) {
+                ret = -ENOMEM;
+                goto out;
+        }
 
-                fd_ctx->locked_on = GF_CALLOC (sizeof (*fd_ctx->locked_on),
-                                               priv->child_count,
-                                               gf_afr_mt_char);
-                if (!fd_ctx->locked_on) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
-                        ret = -ENOMEM;
-                        goto unlock;
-                }
+	pthread_mutex_init (&fd_ctx->delay_lock, NULL);
 
-                ret = __fd_ctx_set (fd, this, (uint64_t)(long) fd_ctx);
+        INIT_LIST_HEAD (&fd_ctx->eager_locked);
 
-                INIT_LIST_HEAD (&fd_ctx->entries);
-        }
-unlock:
-        UNLOCK (&fd->lock);
+        ret = __fd_ctx_set (fd, this, (uint64_t)(long) fd_ctx);
+        if (ret)
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to set fd ctx (%p)", fd);
 out:
         return ret;
 }
 
-/* {{{ flush */
 
 int
-afr_flush_unwind (call_frame_t *frame, xlator_t *this)
+afr_fd_ctx_set (xlator_t *this, fd_t *fd)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
-
-        local = frame->local;
+        int ret = -1;
 
-        LOCK (&frame->lock);
+        LOCK (&fd->lock);
         {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                AFR_STACK_UNWIND (flush, main_frame,
-                                  local->op_ret, local->op_errno);
+                ret = __afr_fd_ctx_set (this, fd);
         }
+        UNLOCK (&fd->lock);
 
-        return 0;
+        return ret;
 }
 
+/* {{{ flush */
 
 int
-afr_flush_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+afr_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-
-        int call_count  = -1;
-        int child_index = (long) cookie;
-        int need_unwind = 0;
+        afr_local_t *local = NULL;
+        int call_count = -1;
 
         local = frame->local;
-        priv  = this->private;
 
         LOCK (&frame->lock);
         {
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
                 if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
-                }
-
-                local->op_errno = op_errno;
+			local->op_ret = op_ret;
+			if (!local->xdata_rsp && xdata)
+				local->xdata_rsp = dict_ref (xdata);
+		} else {
+			local->op_errno = op_errno;
+		}
         }
         UNLOCK (&frame->lock);
 
-        if (need_unwind)
-                afr_flush_unwind (frame, this);
+	call_count = afr_frame_return (frame);
 
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+	if (call_count == 0)
+		AFR_STACK_UNWIND (flush, frame, local->op_ret,
+				  local->op_errno, local->xdata_rsp);
 
         return 0;
 }
 
-
-int
-afr_flush_wind (call_frame_t *frame, xlator_t *this)
+static int
+afr_flush_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-
-        int i = 0;
-        int call_count = -1;
+        int           i      = 0;
+        afr_local_t   *local = NULL;
+        afr_private_t *priv  = NULL;
+        int call_count       = -1;
 
-        local = frame->local;
         priv = this->private;
-
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
+        local = frame->local;
+        call_count = local->call_count;
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_flush_wind_cbk,
+                        STACK_WIND_COOKIE (frame, afr_flush_cbk,
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->flush,
-                                           local->fd);
-
+                                           local->fd, xdata);
                         if (!--call_count)
                                 break;
+
                 }
         }
 
         return 0;
 }
 
-
-int
-afr_flush_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
-        return 0;
-}
-
-
 int
-afr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+afr_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-
-        call_frame_t  * transaction_frame = NULL;
-
-        int ret        = -1;
-
-        int op_ret   = -1;
-        int op_errno = 0;
-
-        int call_count = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        afr_local_t   *local = NULL;
+        call_stub_t   *stub = NULL;
+        int            op_errno   = ENOMEM;
 
-        priv = this->private;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+	if (!local->call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+	local->fd = fd_ref(fd);
 
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
+        stub = fop_flush_stub (frame, afr_flush_wrapper, fd, xdata);
+        if (!stub)
                 goto out;
-        }
-
-        transaction_frame->local = local;
 
-        local->op = GF_FOP_FLUSH;
+        afr_delayed_changelog_wake_resume (this, fd, stub);
 
-        local->transaction.fop    = afr_flush_wind;
-        local->transaction.done   = afr_flush_done;
-        local->transaction.unwind = afr_flush_unwind;
-
-        local->fd                 = fd_ref (fd);
-
-        local->transaction.main_frame = frame;
-        local->transaction.start  = 0;
-        local->transaction.len    = 0;
-
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
-
-
-        op_ret = 0;
+	return 0;
 out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-
-                AFR_STACK_UNWIND (flush, frame, op_ret, op_errno);
-        }
-
+	AFR_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
         return 0;
 }
 
@@ -1421,23 +2114,31 @@ afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd)
         uint64_t        ctx = 0;
         afr_fd_ctx_t    *fd_ctx = NULL;
         int             ret = 0;
+	int             i = 0;
 
         ret = fd_ctx_get (fd, this, &ctx);
-
         if (ret < 0)
                 goto out;
 
         fd_ctx = (afr_fd_ctx_t *)(long) ctx;
 
         if (fd_ctx) {
-                if (fd_ctx->pre_op_done)
-                        GF_FREE (fd_ctx->pre_op_done);
+                //no need to take any locks
+                if (!list_empty (&fd_ctx->eager_locked))
+                        gf_log (this->name, GF_LOG_WARNING, "%s: Stale "
+                                "Eager-lock stubs found",
+                                uuid_utoa (fd->inode->gfid));
 
-                if (fd_ctx->opened_on)
-                        GF_FREE (fd_ctx->opened_on);
+		for (i = 0; i < AFR_NUM_CHANGE_LOGS; i++)
+			GF_FREE (fd_ctx->pre_op_done[i]);
 
-                if (fd_ctx->locked_on)
-                        GF_FREE (fd_ctx->locked_on);
+                GF_FREE (fd_ctx->opened_on);
+
+                GF_FREE (fd_ctx->lock_piggyback);
+
+                GF_FREE (fd_ctx->lock_acquired);
+
+		pthread_mutex_destroy (&fd_ctx->delay_lock);
 
                 GF_FREE (fd_ctx);
         }
@@ -1450,24 +2151,8 @@ out:
 int
 afr_release (xlator_t *this, fd_t *fd)
 {
-        afr_locked_fd_t *locked_fd = NULL;
-        afr_locked_fd_t *tmp       = NULL;
-        afr_private_t   *priv      = NULL;
-
-        priv = this->private;
-
         afr_cleanup_fd_ctx (this, fd);
 
-        list_for_each_entry_safe (locked_fd, tmp, &priv->saved_fds,
-                                  list) {
-
-                if (locked_fd->fd == fd) {
-                        list_del_init (&locked_fd->list);
-                        GF_FREE (locked_fd);
-                }
-
-        }
-
         return 0;
 }
 
@@ -1475,56 +2160,87 @@ afr_release (xlator_t *this, fd_t *fd)
 /* {{{ fsync */
 
 int
+afr_fsync_unwind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+        AFR_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
+        return 0;
+}
+
+int
 afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf)
+               struct iatt *postbuf, dict_t *xdata)
 {
         afr_local_t *local = NULL;
-
         int call_count = -1;
-
         int child_index = (long) cookie;
-        int read_child  = 0;
+	int read_subvol = 0;
+	call_stub_t *stub = NULL;
 
         local = frame->local;
 
-        read_child = afr_read_child (this, local->fd->inode);
+	read_subvol = afr_data_subvol_get (local->inode, this, 0, 0);
 
         LOCK (&frame->lock);
         {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
-
                 if (op_ret == 0) {
-                        local->op_ret = 0;
+                        if (local->op_ret == -1) {
+				local->op_ret = 0;
 
-                        if (local->success_count == 0) {
-                                local->cont.fsync.prebuf  = *prebuf;
-                                local->cont.fsync.postbuf = *postbuf;
-                        }
+                                local->cont.inode_wfop.prebuf  = *prebuf;
+                                local->cont.inode_wfop.postbuf = *postbuf;
 
-                        if (child_index == read_child) {
-                                local->cont.fsync.prebuf  = *prebuf;
-                                local->cont.fsync.postbuf = *postbuf;
+				if (xdata)
+					local->xdata_rsp = dict_ref (xdata);
                         }
 
-                        local->success_count++;
-                }
-
-                local->op_errno = op_errno;
+                        if (child_index == read_subvol) {
+                                local->cont.inode_wfop.prebuf  = *prebuf;
+                                local->cont.inode_wfop.postbuf = *postbuf;
+				if (xdata) {
+					if (local->xdata_rsp)
+						dict_unref (local->xdata_rsp);
+					local->xdata_rsp = dict_ref (xdata);
+				}
+                        }
+                } else {
+			local->op_errno = op_errno;
+		}
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
-                local->cont.fsync.prebuf.ia_ino  = local->cont.fsync.ino;
-                local->cont.fsync.postbuf.ia_ino = local->cont.fsync.ino;
-
-                AFR_STACK_UNWIND (fsync, frame, local->op_ret, local->op_errno,
-                                  &local->cont.fsync.prebuf,
-                                  &local->cont.fsync.postbuf);
+		/* Make a stub out of the frame, and register it
+		   with the waking up post-op. When the call-stub resumes,
+		   we are guaranteed that there was no post-op pending
+		   (i.e changelogs were unset in the server). This is an
+		   essential "guarantee", that fsync() returns only after
+		   completely finishing EVERYTHING, including the delayed
+		   post-op. This guarantee is expected by FUSE graph switching
+		   for example.
+		*/
+		stub = fop_fsync_cbk_stub (frame, afr_fsync_unwind_cbk,
+                                           local->op_ret, local->op_errno,
+                                           &local->cont.inode_wfop.prebuf,
+                                           &local->cont.inode_wfop.postbuf,
+                                           local->xdata_rsp);
+		if (!stub) {
+			AFR_STACK_UNWIND (fsync, frame, -1, ENOMEM, 0, 0, 0);
+			return 0;
+		}
+
+		/* If no new unstable writes happened between the
+		   time we cleared the unstable write witness flag in afr_fsync
+		   and now, calling afr_delayed_changelog_wake_up() should
+		   wake up and skip over the fsync phase and go straight to
+		   afr_changelog_post_op_now()
+		*/
+		afr_delayed_changelog_wake_resume (this, local->fd, stub);
         }
 
         return 0;
@@ -1532,38 +2248,34 @@ afr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 
 int
-afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
-           int32_t datasync)
+afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+	   dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
+	afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
-
-        int ret = -1;
-
         int i = 0;
         int32_t call_count = 0;
-        int32_t op_ret   = -1;
-        int32_t op_errno = 0;
+        int32_t op_errno = ENOMEM;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+	priv = this->private;
 
-        priv = this->private;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+        call_count = local->call_count;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+        local->fd = fd_ref (fd);
 
-        call_count = local->call_count;
-        frame->local = local;
+	if (afr_fd_has_witnessed_unstable_write (this, fd)) {
+		/* don't care. we only wanted to CLEAR the bit */
+	}
 
-        local->fd             = fd_ref (fd);
-        local->cont.fsync.ino = fd->inode->ino;
+	local->inode = inode_ref (fd->inode);
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
@@ -1571,17 +2283,16 @@ afr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
                                            (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->fsync,
-                                           fd, datasync);
+                                           fd, datasync, xdata);
                         if (!--call_count)
                                 break;
                 }
         }
 
-        op_ret = 0;
+	return 0;
 out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (fsync, frame, op_ret, op_errno, NULL, NULL);
-        }
+	AFR_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
+
         return 0;
 }
 
@@ -1589,22 +2300,24 @@ out:
 
 /* {{{ fsync */
 
-int32_t
-afr_fsyncdir_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno)
+int
+afr_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t *local = NULL;
-
         int call_count = -1;
 
         local = frame->local;
 
         LOCK (&frame->lock);
         {
-                if (op_ret == 0)
+                if (op_ret == 0) {
                         local->op_ret = 0;
-
-                local->op_errno = op_errno;
+			if (!local->xdata_rsp && xdata)
+				local->xdata_rsp = dict_ref (xdata);
+		} else {
+			local->op_errno = op_errno;
+		}
         }
         UNLOCK (&frame->lock);
 
@@ -1612,59 +2325,49 @@ afr_fsyncdir_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (fsyncdir, frame, local->op_ret,
-                                  local->op_errno);
+				  local->op_errno, local->xdata_rsp);
 
         return 0;
 }
 
 
-int32_t
-afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              int32_t datasync)
+int
+afr_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t datasync,
+	      dict_t *xdata)
 {
-        afr_private_t *priv = NULL;
+	afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
-
-        int ret = -1;
-
         int i = 0;
         int32_t call_count = 0;
-        int32_t op_ret   = -1;
-        int32_t op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int32_t op_errno = ENOMEM;
 
-        priv = this->private;
-
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+	priv = this->private;
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
         call_count = local->call_count;
-        frame->local = local;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND (frame, afr_fsyncdir_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->fsyncdir,
-                                    fd, datasync);
+                                    fd, datasync, xdata);
                         if (!--call_count)
                                 break;
                 }
         }
 
-        op_ret = 0;
+	return 0;
 out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (fsyncdir, frame, op_ret, op_errno);
-        }
+	AFR_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
@@ -1675,18 +2378,24 @@ out:
 int32_t
 afr_xattrop_cbk (call_frame_t *frame, void *cookie,
                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                 dict_t *xattr)
+                 dict_t *xattr, dict_t *xdata)
 {
         afr_local_t *local = NULL;
-
         int call_count = -1;
 
         local = frame->local;
 
         LOCK (&frame->lock);
         {
-                if (op_ret == 0)
+                if (op_ret == 0) {
+                        if (!local->cont.xattrop.xattr)
+                                local->cont.xattrop.xattr = dict_ref (xattr);
+
+			if (!local->xdata_rsp && xdata)
+				local->xdata_rsp = dict_ref (xdata);
+
                         local->op_ret = 0;
+                }
 
                 local->op_errno = op_errno;
         }
@@ -1696,7 +2405,7 @@ afr_xattrop_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (xattrop, frame, local->op_ret, local->op_errno,
-                                  xattr);
+                local->cont.xattrop.xattr, local->xdata_rsp);
 
         return 0;
 }
@@ -1704,51 +2413,41 @@ afr_xattrop_cbk (call_frame_t *frame, void *cookie,
 
 int32_t
 afr_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             gf_xattrop_flags_t optype, dict_t *xattr)
+             gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
-
-        int ret = -1;
-
         int i = 0;
         int32_t call_count = 0;
-        int32_t op_ret   = -1;
-        int32_t op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int32_t op_errno = ENOMEM;
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
-
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
         call_count = local->call_count;
-        frame->local = local;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND (frame, afr_xattrop_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->xattrop,
-                                    loc, optype, xattr);
+                                    loc, optype, xattr, xdata);
                         if (!--call_count)
                                 break;
                 }
         }
 
-        op_ret = 0;
+	return 0;
 out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (xattrop, frame, op_ret, op_errno, NULL);
-        }
+	AFR_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
+
         return 0;
 }
 
@@ -1759,7 +2458,7 @@ out:
 int32_t
 afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
                   xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  dict_t *xattr)
+                  dict_t *xattr, dict_t *xdata)
 {
         afr_local_t *local = NULL;
 
@@ -1769,8 +2468,14 @@ afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
 
         LOCK (&frame->lock);
         {
-                if (op_ret == 0)
+                if (op_ret == 0) {
+                        if (!local->cont.fxattrop.xattr)
+                                local->cont.fxattrop.xattr = dict_ref (xattr);
+
+			if (!local->xdata_rsp && xdata)
+				local->xdata_rsp = dict_ref (xdata);
                         local->op_ret = 0;
+                }
 
                 local->op_errno = op_errno;
         }
@@ -1780,7 +2485,7 @@ afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (fxattrop, frame, local->op_ret, local->op_errno,
-                                  xattr);
+                                  local->cont.fxattrop.xattr, local->xdata_rsp);
 
         return 0;
 }
@@ -1788,51 +2493,41 @@ afr_fxattrop_cbk (call_frame_t *frame, void *cookie,
 
 int32_t
 afr_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              gf_xattrop_flags_t optype, dict_t *xattr)
+              gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
-
-        int ret = -1;
-
         int i = 0;
         int32_t call_count = 0;
-        int32_t op_ret   = -1;
         int32_t op_errno = 0;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
         priv = this->private;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
-
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
                 goto out;
-        }
 
         call_count = local->call_count;
-        frame->local = local;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND (frame, afr_fxattrop_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->fxattrop,
-                                    fd, optype, xattr);
+                                    fd, optype, xattr, xdata);
                         if (!--call_count)
                                 break;
                 }
         }
 
-        op_ret = 0;
+	return 0;
 out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, NULL);
-        }
+	AFR_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
+
         return 0;
 }
 
@@ -1840,12 +2535,11 @@ out:
 
 
 int32_t
-afr_inodelk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno)
+afr_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
         afr_local_t *local = NULL;
-
         int call_count = -1;
 
         local = frame->local;
@@ -1863,7 +2557,7 @@ afr_inodelk_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (inodelk, frame, local->op_ret,
-                                  local->op_errno);
+                                  local->op_errno, xdata);
 
         return 0;
 }
@@ -1871,63 +2565,53 @@ afr_inodelk_cbk (call_frame_t *frame, void *cookie,
 
 int32_t
 afr_inodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock)
+             const char *volume, loc_t *loc, int32_t cmd,
+             struct gf_flock *flock, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
-
-        int ret = -1;
-
         int i = 0;
         int32_t call_count = 0;
-        int32_t op_ret   = -1;
-        int32_t op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int32_t op_errno = ENOMEM;
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
-
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
+        local = AFR_FRAME_INIT (frame, op_errno);
+        if (!local)
                 goto out;
-        }
 
         call_count = local->call_count;
-        frame->local = local;
+	if (!call_count) {
+		op_errno = ENOMEM;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND (frame, afr_inodelk_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->inodelk,
-                                    volume, loc, cmd, flock);
+                                    volume, loc, cmd, flock, xdata);
 
                         if (!--call_count)
                                 break;
                 }
         }
 
-        op_ret = 0;
+	return 0;
 out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (inodelk, frame, op_ret, op_errno);
-        }
+	AFR_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
 
 int32_t
-afr_finodelk_cbk (call_frame_t *frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno)
+afr_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
         afr_local_t *local = NULL;
-
         int call_count = -1;
 
         local = frame->local;
@@ -1945,71 +2629,59 @@ afr_finodelk_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (finodelk, frame, local->op_ret,
-                                  local->op_errno);
+                                  local->op_errno, xdata);
 
         return 0;
 }
 
 
 int32_t
-afr_finodelk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock)
+afr_finodelk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+	      int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
-
-        int ret = -1;
-
         int i = 0;
         int32_t call_count = 0;
-        int32_t op_ret   = -1;
-        int32_t op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int32_t op_errno = ENOMEM;
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
-
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
         call_count = local->call_count;
-        frame->local = local;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND (frame, afr_finodelk_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->finodelk,
-                                    volume, fd, cmd, flock);
+                                    volume, fd, cmd, flock, xdata);
 
                         if (!--call_count)
                                 break;
                 }
         }
 
-        op_ret = 0;
+	return 0;
 out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (finodelk, frame, op_ret, op_errno);
-        }
+	AFR_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
 
 int32_t
-afr_entrylk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno)
-
+afr_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_local_t *local = NULL;
-
         int call_count = -1;
 
         local = frame->local;
@@ -2027,73 +2699,62 @@ afr_entrylk_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (entrylk, frame, local->op_ret,
-                                  local->op_errno);
+                                  local->op_errno, xdata);
 
         return 0;
 }
 
 
-int32_t
-afr_entrylk (call_frame_t *frame, xlator_t *this,
-             const char *volume, loc_t *loc,
-             const char *basename, entrylk_cmd cmd, entrylk_type type)
+int
+afr_entrylk (call_frame_t *frame, xlator_t *this, const char *volume,
+	     loc_t *loc, const char *basename, entrylk_cmd cmd,
+	     entrylk_type type, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
-
-        int ret = -1;
-
         int i = 0;
         int32_t call_count = 0;
-        int32_t op_ret   = -1;
         int32_t op_errno = 0;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-
         priv = this->private;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
-
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
         call_count = local->call_count;
-        frame->local = local;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND (frame, afr_entrylk_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->entrylk,
-                                    volume, loc, basename, cmd, type);
+                                    volume, loc, basename, cmd, type, xdata);
 
                         if (!--call_count)
                                 break;
                 }
         }
 
-        op_ret = 0;
+	return 0;
 out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (entrylk, frame, op_ret, op_errno);
-        }
+	AFR_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
 
 
-int32_t
-afr_fentrylk_cbk (call_frame_t *frame, void *cookie,
-                 xlator_t *this, int32_t op_ret, int32_t op_errno)
+int
+afr_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
         afr_local_t *local = NULL;
-
         int call_count = -1;
 
         local = frame->local;
@@ -2111,163 +2772,150 @@ afr_fentrylk_cbk (call_frame_t *frame, void *cookie,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (fentrylk, frame, local->op_ret,
-                                  local->op_errno);
+                                  local->op_errno, xdata);
 
         return 0;
 }
 
 
-int32_t
-afr_fentrylk (call_frame_t *frame, xlator_t *this,
-              const char *volume, fd_t *fd,
-              const char *basename, entrylk_cmd cmd, entrylk_type type)
+int
+afr_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume, fd_t *fd,
+              const char *basename, entrylk_cmd cmd, entrylk_type type,
+	      dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local  = NULL;
-
-        int ret = -1;
-
         int i = 0;
         int32_t call_count = 0;
-        int32_t op_ret   = -1;
-        int32_t op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int32_t op_errno = ENOMEM;
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
-
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
         call_count = local->call_count;
-        frame->local = local;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND (frame, afr_fentrylk_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->fentrylk,
-                                    volume, fd, basename, cmd, type);
+                                    volume, fd, basename, cmd, type, xdata);
 
                         if (!--call_count)
                                 break;
                 }
         }
 
-        op_ret = 0;
+	return 0;
 out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (fentrylk, frame, op_ret, op_errno);
-        }
+	AFR_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
-int32_t
-afr_statfs_cbk (call_frame_t *frame, void *cookie,
-                xlator_t *this, int32_t op_ret, int32_t op_errno,
-                struct statvfs *statvfs)
+
+int
+afr_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+		int op_errno, struct statvfs *statvfs, dict_t *xdata)
 {
         afr_local_t *local = NULL;
-
         int call_count = 0;
+	struct statvfs *buf = NULL;
 
         LOCK (&frame->lock);
         {
                 local = frame->local;
 
-                if (op_ret == 0) {
-                        local->op_ret   = op_ret;
-
-                        if (local->cont.statfs.buf_set) {
-                                if (statvfs->f_bavail < local->cont.statfs.buf.f_bavail)
-                                        local->cont.statfs.buf = *statvfs;
-                        } else {
-                                local->cont.statfs.buf = *statvfs;
-                                local->cont.statfs.buf_set = 1;
-                        }
-                }
-
-                if (op_ret == -1)
+                if (op_ret != 0) {
                         local->op_errno = op_errno;
-
+			goto unlock;
+		}
+
+		local->op_ret = op_ret;
+
+		buf = &local->cont.statfs.buf;
+		if (local->cont.statfs.buf_set) {
+			if (statvfs->f_bavail < buf->f_bavail) {
+				*buf = *statvfs;
+				if (xdata) {
+					if (local->xdata_rsp)
+						dict_unref (local->xdata_rsp);
+					local->xdata_rsp = dict_ref (xdata);
+				}
+			}
+		} else {
+			*buf = *statvfs;
+			local->cont.statfs.buf_set = 1;
+			if (xdata)
+				local->xdata_rsp = dict_ref (xdata);
+		}
         }
+unlock:
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
-                                  &local->cont.statfs.buf);
+                                  &local->cont.statfs.buf, local->xdata_rsp);
 
         return 0;
 }
 
 
-int32_t
-afr_statfs (call_frame_t *frame, xlator_t *this,
-            loc_t *loc)
+int
+afr_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        afr_private_t *  priv        = NULL;
-        int              child_count = 0;
         afr_local_t   *  local       = NULL;
+	afr_private_t   *priv        = NULL;
         int              i           = 0;
-
-        int ret = -1;
         int              call_count = 0;
-        int32_t          op_ret      = -1;
-        int32_t          op_errno    = 0;
+        int32_t          op_errno    = ENOMEM;
 
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (loc, out);
-
-        priv = this->private;
-        child_count = priv->child_count;
+	priv = this->private;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
-
-        frame->local = local;
         call_count = local->call_count;
+	if (!call_count) {
+		op_errno = ENOTCONN;
+		goto out;
+	}
 
-        for (i = 0; i < child_count; i++) {
+        for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
                         STACK_WIND (frame, afr_statfs_cbk,
                                     priv->children[i],
                                     priv->children[i]->fops->statfs,
-                                    loc);
+                                    loc, xdata);
                         if (!--call_count)
                                 break;
                 }
         }
 
-        op_ret = 0;
+	return 0;
 out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (statfs, frame, op_ret, op_errno, NULL);
-        }
+	AFR_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
+
         return 0;
 }
 
 
 int32_t
 afr_lk_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+                   int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+                   dict_t *xdata)
 {
         afr_local_t * local = NULL;
-
         int call_count = -1;
 
         local = frame->local;
@@ -2275,7 +2923,7 @@ afr_lk_unlock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         if (call_count == 0)
                 AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  lock);
+                                  lock, xdata);
 
         return 0;
 }
@@ -2286,8 +2934,7 @@ afr_lk_unlock (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t   * local = NULL;
         afr_private_t * priv  = NULL;
-
-        int i;
+        int i = 0;
         int call_count = 0;
 
         local = frame->local;
@@ -2298,7 +2945,7 @@ afr_lk_unlock (call_frame_t *frame, xlator_t *this)
 
         if (call_count == 0) {
                 AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  &local->cont.lk.ret_flock);
+                                  &local->cont.lk.ret_flock, NULL);
                 return 0;
         }
 
@@ -2312,7 +2959,7 @@ afr_lk_unlock (call_frame_t *frame, xlator_t *this)
                                     priv->children[i],
                                     priv->children[i]->fops->lk,
                                     local->fd, F_SETLK,
-                                    &local->cont.lk.user_flock);
+                                    &local->cont.lk.user_flock, NULL);
 
                         if (!--call_count)
                                 break;
@@ -2325,13 +2972,13 @@ afr_lk_unlock (call_frame_t *frame, xlator_t *this)
 
 int32_t
 afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-            int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+            int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int child_index = -1;
 /*        int            ret  = 0; */
 
-        int child_index = -1;
 
         local = frame->local;
         priv  = this->private;
@@ -2360,30 +3007,15 @@ afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                    priv->children[child_index],
                                    priv->children[child_index]->fops->lk,
                                    local->fd, local->cont.lk.cmd,
-                                   &local->cont.lk.user_flock);
+                                   &local->cont.lk.user_flock, xdata);
         } else if (local->op_ret == -1) {
                 /* all nodes have gone down */
 
                 AFR_STACK_UNWIND (lk, frame, -1, ENOTCONN,
-                                  &local->cont.lk.ret_flock);
+                                  &local->cont.lk.ret_flock, NULL);
         } else {
-                /* locking has succeeded on all nodes that are up */
-
-                /* temporarily
-                ret = afr_mark_locked_nodes (this, local->fd,
-                                             local->cont.lk.locked_nodes);
-                if (ret)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Could not save locked nodes info in fdctx");
-
-                ret = afr_save_locked_fd (this, local->fd);
-                if (ret)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Could not save locked fd");
-
-                */
-		AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
-                                  &local->cont.lk.ret_flock);
+                AFR_STACK_UNWIND (lk, frame, local->op_ret, local->op_errno,
+                                  &local->cont.lk.ret_flock, NULL);
         }
 
         return 0;
@@ -2392,34 +3024,24 @@ afr_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 afr_lk (call_frame_t *frame, xlator_t *this,
-        fd_t *fd, int32_t cmd,
-        struct gf_flock *flock)
+        fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
         afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
-
         int i = 0;
-
-        int32_t op_ret   = -1;
-        int32_t op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int32_t op_errno = ENOMEM;
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
-        AFR_LOCAL_INIT (local, priv);
-
-        frame->local  = local;
+        local = AFR_FRAME_INIT (frame, op_errno);
+        if (!local)
+                goto out;
 
         local->cont.lk.locked_nodes = GF_CALLOC (priv->child_count,
-                                              sizeof (*local->cont.lk.locked_nodes),
-                                              gf_afr_mt_char);
+                                                 sizeof (*local->cont.lk.locked_nodes),
+                                                 gf_afr_mt_char);
 
         if (!local->cont.lk.locked_nodes) {
-                gf_log (this->name, GF_LOG_ERROR, "Out of memory");
                 op_errno = ENOMEM;
                 goto out;
         }
@@ -2432,13 +3054,18 @@ afr_lk (call_frame_t *frame, xlator_t *this,
         STACK_WIND_COOKIE (frame, afr_lk_cbk, (void *) (long) 0,
                            priv->children[i],
                            priv->children[i]->fops->lk,
-                           fd, cmd, flock);
+                           fd, cmd, flock, xdata);
 
-        op_ret = 0;
+	return 0;
 out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (lk, frame, op_ret, op_errno, NULL);
-        }
+	AFR_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
+
+        return 0;
+}
+
+int
+afr_forget (xlator_t *this, inode_t *inode)
+{
         return 0;
 }
 
@@ -2451,47 +3078,28 @@ afr_priv_dump (xlator_t *this)
         int   i = 0;
 
 
-        assert(this);
+        GF_ASSERT (this);
         priv = this->private;
 
-        assert(priv);
+        GF_ASSERT (priv);
         snprintf(key_prefix, GF_DUMP_MAX_BUF_LEN, "%s.%s", this->type, this->name);
         gf_proc_dump_add_section(key_prefix);
-        gf_proc_dump_build_key(key, key_prefix, "child_count");
-        gf_proc_dump_write(key, "%u", priv->child_count);
-        gf_proc_dump_build_key(key, key_prefix, "read_child_rr");
-        gf_proc_dump_write(key, "%u", priv->read_child_rr);
+        gf_proc_dump_write("child_count", "%u", priv->child_count);
         for (i = 0; i < priv->child_count; i++) {
-                gf_proc_dump_build_key(key, key_prefix, "child_up[%d]", i);
+                sprintf (key, "child_up[%d]", i);
                 gf_proc_dump_write(key, "%d", priv->child_up[i]);
-                gf_proc_dump_build_key(key, key_prefix,
-                                        "pending_key[%d]", i);
+                sprintf (key, "pending_key[%d]", i);
                 gf_proc_dump_write(key, "%s", priv->pending_key[i]);
         }
-        gf_proc_dump_build_key(key, key_prefix, "data_self_heal");
-        gf_proc_dump_write(key, "%d", priv->data_self_heal);
-        gf_proc_dump_build_key(key, key_prefix, "metadata_self_heal");
-        gf_proc_dump_write(key, "%d", priv->metadata_self_heal);
-        gf_proc_dump_build_key(key, key_prefix, "entry_self_heal");
-        gf_proc_dump_write(key, "%d", priv->entry_self_heal);
-        gf_proc_dump_build_key(key, key_prefix, "data_change_log");
-        gf_proc_dump_write(key, "%d", priv->data_change_log);
-        gf_proc_dump_build_key(key, key_prefix, "metadata_change_log");
-        gf_proc_dump_write(key, "%d", priv->metadata_change_log);
-        gf_proc_dump_build_key(key, key_prefix, "entry_change_log");
-        gf_proc_dump_write(key, "%d", priv->entry_change_log);
-        gf_proc_dump_build_key(key, key_prefix, "read_child");
-        gf_proc_dump_write(key, "%d", priv->read_child);
-        gf_proc_dump_build_key(key, key_prefix, "favorite_child");
-        gf_proc_dump_write(key, "%u", priv->favorite_child);
-        gf_proc_dump_build_key(key, key_prefix, "data_lock_server_count");
-        gf_proc_dump_write(key, "%u", priv->data_lock_server_count);
-        gf_proc_dump_build_key(key, key_prefix, "metadata_lock_server_count");
-        gf_proc_dump_write(key, "%u", priv->metadata_lock_server_count);
-        gf_proc_dump_build_key(key, key_prefix, "entry_lock_server_count");
-        gf_proc_dump_write(key, "%u", priv->entry_lock_server_count);
-        gf_proc_dump_build_key(key, key_prefix, "wait_count");
-        gf_proc_dump_write(key, "%u", priv->wait_count);
+        gf_proc_dump_write("data_self_heal", "%s", priv->data_self_heal);
+        gf_proc_dump_write("metadata_self_heal", "%d", priv->metadata_self_heal);
+        gf_proc_dump_write("entry_self_heal", "%d", priv->entry_self_heal);
+        gf_proc_dump_write("data_change_log", "%d", priv->data_change_log);
+        gf_proc_dump_write("metadata_change_log", "%d", priv->metadata_change_log);
+        gf_proc_dump_write("entry-change_log", "%d", priv->entry_change_log);
+        gf_proc_dump_write("read_child", "%d", priv->read_child);
+        gf_proc_dump_write("favorite_child", "%d", priv->favorite_child);
+        gf_proc_dump_write("wait_count", "%u", priv->wait_count);
 
         return 0;
 }
@@ -2507,7 +3115,6 @@ static int
 find_child_index (xlator_t *this, xlator_t *child)
 {
         afr_private_t *priv = NULL;
-
         int i = -1;
 
         priv = this->private;
@@ -2522,90 +3129,501 @@ find_child_index (xlator_t *this, xlator_t *child)
 
 int32_t
 afr_notify (xlator_t *this, int32_t event,
-            void *data, ...)
+            void *data, void *data2)
 {
-        afr_private_t *     priv     = NULL;
-        unsigned char *     child_up = NULL;
-
-        int i           = -1;
-        int up_children = 0;
+        afr_private_t   *priv               = NULL;
+        int             i                   = -1;
+        int             up_children         = 0;
+        int             down_children       = 0;
+        int             propagate           = 0;
+        int             had_heard_from_all  = 0;
+        int             have_heard_from_all = 0;
+        int             idx                 = -1;
+        int             ret                 = -1;
+        int             call_psh            = 0;
+        int             up_child            = -1;
+        dict_t          *input              = NULL;
+        dict_t          *output             = NULL;
 
         priv = this->private;
 
         if (!priv)
                 return 0;
 
-        child_up = priv->child_up;
+        /*
+         * We need to reset this in case children come up in "staggered"
+         * fashion, so that we discover a late-arriving local subvolume.  Note
+         * that we could end up issuing N lookups to the first subvolume, and
+         * O(N^2) overall, but N is small for AFR so it shouldn't be an issue.
+         */
+        priv->did_discovery = _gf_false;
+
+        had_heard_from_all = 1;
+        for (i = 0; i < priv->child_count; i++) {
+                if (!priv->last_event[i]) {
+                        had_heard_from_all = 0;
+                }
+        }
+
+        /* parent xlators dont need to know about every child_up, child_down
+         * because of afr ha. If all subvolumes go down, child_down has
+         * to be triggered. In that state when 1 subvolume comes up child_up
+         * needs to be triggered. dht optimizes revalidate lookup by sending
+         * it only to one of its subvolumes. When child up/down happens
+         * for afr's subvolumes dht should be notified by child_modified. The
+         * subsequent revalidate lookup happens on all the dht's subvolumes
+         * which triggers afr self-heals if any.
+         */
+        idx = find_child_index (this, data);
+        if (idx < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Received child_up "
+                        "from invalid subvolume");
+                goto out;
+        }
 
         switch (event) {
         case GF_EVENT_CHILD_UP:
-                i = find_child_index (this, data);
+                LOCK (&priv->lock);
+                {
+                        /*
+                         * This only really counts if the child was never up
+                         * (value = -1) or had been down (value = 0).  See
+                         * comment at GF_EVENT_CHILD_DOWN for a more detailed
+                         * explanation.
+                         */
+                        if (priv->child_up[idx] != 1) {
+                                priv->up_count++;
+				priv->event_generation++;
+                        }
+                        priv->child_up[idx] = 1;
+
+                        call_psh = 1;
+                        up_child = idx;
+                        for (i = 0; i < priv->child_count; i++)
+                                if (priv->child_up[i] == 1)
+                                        up_children++;
+                        if (up_children == 1) {
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "Subvolume '%s' came back up; "
+                                        "going online.", ((xlator_t *)data)->name);
+                        } else {
+                                event = GF_EVENT_CHILD_MODIFIED;
+                        }
 
-                /* temporarily
-                afr_attempt_lock_recovery (this, i);
-                */
+                        priv->last_event[idx] = event;
+                }
+                UNLOCK (&priv->lock);
 
-                child_up[i] = 1;
+                break;
 
+        case GF_EVENT_CHILD_DOWN:
                 LOCK (&priv->lock);
                 {
-                        priv->up_count++;
+                        /*
+                         * If a brick is down when we start, we'll get a
+                         * CHILD_DOWN to indicate its initial state.  There
+                         * was never a CHILD_UP in this case, so if we
+                         * increment "down_count" the difference between than
+                         * and "up_count" will no longer be the number of
+                         * children that are currently up.  This has serious
+                         * implications e.g. for quorum enforcement, so we
+                         * don't increment these values unless the event
+                         * represents an actual state transition between "up"
+                         * (value = 1) and anything else.
+                         */
+                        if (priv->child_up[idx] == 1) {
+                                priv->down_count++;
+				priv->event_generation++;
+                        }
+                        priv->child_up[idx] = 0;
+
+                        for (i = 0; i < priv->child_count; i++)
+                                if (priv->child_up[i] == 0)
+                                        down_children++;
+                        if (down_children == priv->child_count) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "All subvolumes are down. Going offline "
+                                        "until atleast one of them comes back up.");
+                        } else {
+                                event = GF_EVENT_CHILD_MODIFIED;
+                        }
+
+                        priv->last_event[idx] = event;
                 }
                 UNLOCK (&priv->lock);
 
-                /*
-                   if all the children were down, and one child came up,
-                   send notify to parent
-                */
+                break;
 
-                for (i = 0; i < priv->child_count; i++)
-                        if (child_up[i])
-                                up_children++;
+        case GF_EVENT_CHILD_CONNECTING:
+                LOCK (&priv->lock);
+                {
+                        priv->last_event[idx] = event;
+                }
+                UNLOCK (&priv->lock);
 
-                if (up_children == 1) {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "Subvolume '%s' came back up; "
-                                "going online.", ((xlator_t *)data)->name);
+                break;
 
-                        default_notify (this, event, data);
+        case GF_EVENT_TRANSLATOR_OP:
+                input = data;
+                output = data2;
+                if (!had_heard_from_all) {
+                        ret = -1;
+                        goto out;
                 }
+                ret = afr_xl_op (this, input, output);
+                goto out;
+                break;
 
+        default:
+                propagate = 1;
                 break;
+        }
 
-        case GF_EVENT_CHILD_DOWN:
-                i = find_child_index (this, data);
+        /* have all subvolumes reported status once by now? */
+        have_heard_from_all = 1;
+        for (i = 0; i < priv->child_count; i++) {
+                if (!priv->last_event[i])
+                        have_heard_from_all = 0;
+        }
+
+        /* if all subvols have reported status, no need to hide anything
+           or wait for anything else. Just propagate blindly */
+        if (have_heard_from_all)
+                propagate = 1;
 
-                child_up[i] = 0;
+        if (!had_heard_from_all && have_heard_from_all) {
+                /* This is the first event which completes aggregation
+                   of events from all subvolumes. If at least one subvol
+                   had come up, propagate CHILD_UP, but only this time
+                */
+                event = GF_EVENT_CHILD_DOWN;
 
                 LOCK (&priv->lock);
                 {
-                        priv->down_count++;
+                        up_children = AFR_COUNT (priv->child_up, priv->child_count);
+                        for (i = 0; i < priv->child_count; i++) {
+                                if (priv->last_event[i] == GF_EVENT_CHILD_UP) {
+                                        event = GF_EVENT_CHILD_UP;
+                                        break;
+                                }
+
+                                if (priv->last_event[i] ==
+                                                GF_EVENT_CHILD_CONNECTING) {
+                                        event = GF_EVENT_CHILD_CONNECTING;
+                                        /* continue to check other events for CHILD_UP */
+                                }
+                        }
                 }
                 UNLOCK (&priv->lock);
+        }
 
-                /*
-                   if all children are down, and this was the last to go down,
-                   send notify to parent
-                */
+        ret = 0;
+        if (propagate)
+                ret = default_notify (this, event, data);
 
-                for (i = 0; i < priv->child_count; i++)
-                        if (child_up[i])
-                                up_children++;
+        if (call_psh && priv->shd.iamshd) {
+                afr_selfheal_childup (this, up_child);
+	}
+out:
+        return ret;
+}
+
+
+int
+afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno)
+{
+        local->op_ret = -1;
+        local->op_errno = EUCLEAN;
+
+	syncbarrier_init (&local->barrier);
+
+        local->child_up = GF_CALLOC (priv->child_count,
+                                     sizeof (*local->child_up),
+                                     gf_afr_mt_char);
+        if (!local->child_up) {
+                if (op_errno)
+                        *op_errno = ENOMEM;
+                goto out;
+        }
+
+        memcpy (local->child_up, priv->child_up,
+                sizeof (*local->child_up) * priv->child_count);
+        local->call_count = AFR_COUNT (local->child_up, priv->child_count);
+        if (local->call_count == 0) {
+                gf_log (THIS->name, GF_LOG_INFO, "no subvolumes up");
+                if (op_errno)
+                        *op_errno = ENOTCONN;
+                goto out;
+        }
+	local->event_generation = priv->event_generation;
+
+	local->read_attempted = GF_CALLOC (priv->child_count, sizeof (char),
+					   gf_afr_mt_char);
+	if (!local->read_attempted) {
+		if (op_errno)
+			*op_errno = ENOMEM;
+		goto out;
+	}
+
+	local->readable = GF_CALLOC (priv->child_count, sizeof (char),
+				     gf_afr_mt_char);
+	if (!local->readable) {
+		if (op_errno)
+			*op_errno = ENOMEM;
+		goto out;
+	}
+
+	local->replies = GF_CALLOC(priv->child_count, sizeof(*local->replies),
+				   gf_afr_mt_reply_t);
+	if (!local->replies) {
+		if (op_errno)
+			*op_errno = ENOMEM;
+		goto out;
+	}
+
+	return 0;
+out:
+        return -1;
+}
+
+int
+afr_internal_lock_init (afr_internal_lock_t *lk, size_t child_count,
+                        transaction_lk_type_t lk_type)
+{
+        int             ret = -ENOMEM;
+
+        lk->locked_nodes = GF_CALLOC (sizeof (*lk->locked_nodes),
+                                      child_count, gf_afr_mt_char);
+        if (NULL == lk->locked_nodes)
+                goto out;
+
+        lk->lower_locked_nodes = GF_CALLOC (sizeof (*lk->lower_locked_nodes),
+                                            child_count, gf_afr_mt_char);
+        if (NULL == lk->lower_locked_nodes)
+                goto out;
+
+        lk->lock_op_ret   = -1;
+        lk->lock_op_errno = EUCLEAN;
+        lk->transaction_lk_type = lk_type;
+
+        ret = 0;
+out:
+        return ret;
+}
+
+void
+afr_matrix_cleanup (int32_t **matrix, unsigned int m)
+{
+        int             i         = 0;
+
+        if (!matrix)
+                goto out;
+        for (i = 0; i < m; i++) {
+                GF_FREE (matrix[i]);
+        }
+
+        GF_FREE (matrix);
+out:
+        return;
+}
+
+int32_t**
+afr_matrix_create (unsigned int m, unsigned int n)
+{
+        int32_t         **matrix = NULL;
+        int             i       = 0;
+
+        matrix = GF_CALLOC (sizeof (*matrix), m, gf_afr_mt_int32_t);
+        if (!matrix)
+                goto out;
+
+        for (i = 0; i < m; i++) {
+                matrix[i] = GF_CALLOC (sizeof (*matrix[i]), n,
+                                       gf_afr_mt_int32_t);
+                if (!matrix[i])
+                        goto out;
+        }
+        return matrix;
+out:
+        afr_matrix_cleanup (matrix, m);
+        return NULL;
+}
+
+int
+afr_inodelk_init (afr_inodelk_t *lk, char *dom, size_t child_count)
+{
+        int             ret = -ENOMEM;
+
+        lk->domain = dom;
+        lk->locked_nodes = GF_CALLOC (sizeof (*lk->locked_nodes),
+                                      child_count, gf_afr_mt_char);
+        if (NULL == lk->locked_nodes)
+                goto out;
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+afr_transaction_local_init (afr_local_t *local, xlator_t *this)
+{
+        int            child_up_count = 0;
+        int            ret = -ENOMEM;
+        afr_private_t *priv = NULL;
+
+        priv = this->private;
+        ret = afr_internal_lock_init (&local->internal_lock, priv->child_count,
+                                      AFR_TRANSACTION_LK);
+        if (ret < 0)
+                goto out;
+
+        if ((local->transaction.type == AFR_DATA_TRANSACTION) ||
+            (local->transaction.type == AFR_METADATA_TRANSACTION)) {
+                ret = afr_inodelk_init (&local->internal_lock.inodelk[0],
+                                        this->name, priv->child_count);
+                if (ret < 0)
+                        goto out;
+        }
+
+        ret = -ENOMEM;
+        child_up_count = AFR_COUNT (local->child_up, priv->child_count);
+        if (priv->optimistic_change_log && child_up_count == priv->child_count)
+                local->optimistic_change_log = 1;
+
+	local->pre_op_compat = priv->pre_op_compat;
+
+        local->transaction.eager_lock =
+                GF_CALLOC (sizeof (*local->transaction.eager_lock),
+                           priv->child_count,
+                           gf_afr_mt_int32_t);
+
+        if (!local->transaction.eager_lock)
+                goto out;
 
-                if (up_children == 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "All subvolumes are down. Going offline "
-                                "until atleast one of them comes back up.");
+        local->transaction.pre_op = GF_CALLOC (sizeof (*local->transaction.pre_op),
+                                               priv->child_count,
+                                               gf_afr_mt_char);
+        if (!local->transaction.pre_op)
+                goto out;
 
-                        default_notify (this, event, data);
+        local->transaction.fop_subvols = GF_CALLOC (sizeof (*local->transaction.fop_subvols),
+						    priv->child_count,
+						    gf_afr_mt_char);
+        if (!local->transaction.fop_subvols)
+                goto out;
+
+        local->transaction.failed_subvols = GF_CALLOC (sizeof (*local->transaction.failed_subvols),
+						       priv->child_count,
+						       gf_afr_mt_char);
+        if (!local->transaction.failed_subvols)
+                goto out;
+
+        local->pending = afr_matrix_create (priv->child_count,
+                                            AFR_NUM_CHANGE_LOGS);
+        if (!local->pending)
+                goto out;
+
+	INIT_LIST_HEAD (&local->transaction.eager_locked);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+void
+afr_set_low_priority (call_frame_t *frame)
+{
+        frame->root->pid = LOW_PRIO_PROC_PID;
+}
+
+
+gf_boolean_t
+afr_have_quorum (char *logname, afr_private_t *priv)
+{
+        unsigned int        quorum = 0;
+
+        GF_VALIDATE_OR_GOTO(logname,priv,out);
+
+        quorum = priv->quorum_count;
+        if (quorum != AFR_QUORUM_AUTO) {
+                return (priv->up_count >= (priv->down_count + quorum));
+        }
+
+        quorum = priv->child_count / 2 + 1;
+        if (priv->up_count >= (priv->down_count + quorum)) {
+                return _gf_true;
+        }
+
+        /*
+         * Special case for even numbers of nodes: if we have exactly half
+         * and that includes the first ("senior-most") node, then that counts
+         * as quorum even if it wouldn't otherwise.  This supports e.g. N=2
+         * while preserving the critical property that there can only be one
+         * such group.
+         */
+        if ((priv->child_count % 2) == 0) {
+                quorum = priv->child_count / 2;
+                if (priv->up_count >= (priv->down_count + quorum)) {
+                        if (priv->child_up[0]) {
+                                return _gf_true;
+                        }
                 }
+        }
 
-                break;
+out:
+        return _gf_false;
+}
 
-        default:
-                default_notify (this, event, data);
+void
+afr_priv_destroy (afr_private_t *priv)
+{
+        int            i           = 0;
+
+        if (!priv)
+                goto out;
+        inode_unref (priv->root_inode);
+        GF_FREE (priv->last_event);
+        if (priv->pending_key) {
+                for (i = 0; i < priv->child_count; i++)
+                        GF_FREE (priv->pending_key[i]);
         }
+        GF_FREE (priv->pending_key);
+        GF_FREE (priv->children);
+        GF_FREE (priv->child_up);
+        LOCK_DESTROY (&priv->lock);
 
-        return 0;
+        GF_FREE (priv);
+out:
+        return;
+}
+
+int
+xlator_subvolume_count (xlator_t *this)
+{
+        int i = 0;
+        xlator_list_t *list = NULL;
+
+        for (list = this->children; list; list = list->next)
+                i++;
+        return i;
+}
+
+
+void
+afr_handle_open_fd_count (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t     *local = NULL;
+        afr_fd_ctx_t    *fd_ctx   = NULL;
+
+        local = frame->local;
+
+        if (!local->fd)
+		return;
+
+	fd_ctx = afr_fd_ctx_get (local->fd, this);
+	if (!fd_ctx)
+		return;
 
+	fd_ctx->open_fd_count = local->open_fd_count;
 }
diff --git a/xlators/cluster/afr/src/afr-dir-read.c b/xlators/cluster/afr/src/afr-dir-read.c
index 005f62c57..fa1da3958 100644
--- a/xlators/cluster/afr/src/afr-dir-read.c
+++ b/xlators/cluster/afr/src/afr-dir-read.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -46,699 +37,384 @@
 #include "checksum.h"
 
 #include "afr.h"
-#include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-
-
-int
-afr_examine_dir_sh_unwind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local  = NULL;
-
-        local = frame->local;
-
-        afr_set_opendir_done (this, local->fd->inode);
-
-        AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                          local->op_errno, local->fd);
-
-        return 0;
-}
-
-
-gf_boolean_t
-__checksums_differ (uint32_t *checksum, int child_count,
-                    unsigned char *child_up)
-{
-        int ret = _gf_false;
-        int i = 0;
-
-        uint32_t cksum;
-
-        cksum = checksum[0];
-
-        for (i = 0; i < child_count; i++) {
-                if (!child_up[i])
-                        continue;
-
-                if (cksum != checksum[i]) {
-                        ret = _gf_true;
-                        break;
-                }
-
-                cksum = checksum[i];
-        }
-
-        return ret;
-}
+#include "afr-transaction.h"
 
 
 int32_t
-afr_examine_dir_readdir_cbk (call_frame_t *frame, void *cookie,
-                             xlator_t *this, int32_t op_ret, int32_t op_errno,
-                             gf_dirent_t *entries)
+afr_opendir_cbk (call_frame_t *frame, void *cookie,
+                 xlator_t *this, int32_t op_ret, int32_t op_errno,
+                 fd_t *fd, dict_t *xdata)
 {
-	afr_private_t *    priv  = NULL;
-	afr_local_t *      local = NULL;
-        afr_self_heal_t *  sh    = NULL;
+        afr_local_t   *local             = NULL;
+        int            call_count        = -1;
+        int32_t        child_index       = 0;
+	afr_fd_ctx_t  *fd_ctx = NULL;
 
-        gf_dirent_t * entry = NULL;
-        gf_dirent_t * tmp   = NULL;
-
-        int child_index = 0;
-
-        uint32_t entry_cksum;
-
-        int   call_count    = 0;
-        off_t last_offset = 0;
-        char  sh_type_str[256] = {0,};
-
-        priv  = this->private;
         local = frame->local;
-        sh    = &local->self_heal;
-
+	fd_ctx = local->fd_ctx;
         child_index = (long) cookie;
 
-        if (op_ret == -1) {
-                local->op_ret = -1;
-                local->op_ret = op_errno;
-                goto out;
-        }
-
-        if (op_ret == 0)
-                goto out;
-
-        list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                entry_cksum = gf_rsync_weak_checksum (entry->d_name,
-                                                      strlen (entry->d_name));
-                local->cont.opendir.checksum[child_index] ^= entry_cksum;
-        }
-
-        list_for_each_entry (entry, &entries->list, list) {
-                last_offset = entry->d_off;
-        }
-
-        /* read more entries */
-
-        STACK_WIND_COOKIE (frame, afr_examine_dir_readdir_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->readdir,
-                           local->fd, 131072, last_offset);
-
-out:
-        if ((op_ret == 0) || (op_ret == -1)) {
-                call_count = afr_frame_return (frame);
-
-                if (call_count == 0) {
-                        if (__checksums_differ (local->cont.opendir.checksum,
-                                                priv->child_count,
-                                                local->child_up)) {
-
-                                sh->need_entry_self_heal  = _gf_true;
-                                sh->forced_merge          = _gf_true;
-                                sh->type                  = local->fd->inode->ia_type;
-                                sh->background            = _gf_false;
-                                sh->unwind                = afr_examine_dir_sh_unwind;
-
-                                afr_self_heal_type_str_get(&local->self_heal,
-                                                           sh_type_str,
-                                                           sizeof(sh_type_str));
-                                gf_log (this->name, GF_LOG_NORMAL,
-                                        "%s self-heal triggered. path: %s, "
-                                         "reason: checksums of directory differ,"
-                                        " forced merge option set",
-                                        sh_type_str, local->loc.path);
-
-                                afr_self_heal (frame, this);
-                        } else {
-                                afr_set_opendir_done (this, local->fd->inode);
-
-                                AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                                                  local->op_errno, local->fd);
-                        }
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+			fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+                } else {
+                        local->op_ret = op_ret;
+			fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+			if (!local->xdata_rsp && xdata)
+				local->xdata_rsp = dict_ref (xdata);
                 }
         }
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
 
+        if (call_count == 0)
+		AFR_STACK_UNWIND (opendir, frame, local->op_ret,
+				  local->op_errno, local->fd, NULL);
         return 0;
 }
 
 
 int
-afr_examine_dir (call_frame_t *frame, xlator_t *this)
+afr_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t *   local = NULL;
+        afr_private_t * priv        = NULL;
+        afr_local_t   * local       = NULL;
+        int             i           = 0;
+        int             call_count  = -1;
+        int32_t         op_errno    = ENOMEM;
+	afr_fd_ctx_t *fd_ctx = NULL;
 
-        int i;
-        int call_count = 0;
+        priv = this->private;
 
-        local = frame->local;
-        priv  = this->private;
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
+
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+		goto out;
 
-        local->cont.opendir.checksum = GF_CALLOC (priv->child_count,
-                                        sizeof (*local->cont.opendir.checksum),
-                                        gf_afr_mt_int32_t);
+        loc_copy (&local->loc, loc);
 
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
+        local->fd    = fd_ref (fd);
+	local->fd_ctx = fd_ctx;
 
-        local->call_count = call_count;
+        call_count = local->call_count;
 
         for (i = 0; i < priv->child_count; i++) {
                 if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_examine_dir_readdir_cbk,
-                                           (void *) (long) i,
+                        STACK_WIND_COOKIE (frame, afr_opendir_cbk,
+                                           (void*) (long) i,
                                            priv->children[i],
-                                           priv->children[i]->fops->readdir,
-                                           local->fd, 131072, 0);
+                                           priv->children[i]->fops->opendir,
+                                           loc, fd, NULL);
 
                         if (!--call_count)
                                 break;
                 }
         }
 
+	return 0;
+out:
+	AFR_STACK_UNWIND (opendir, frame, -1, op_errno, fd, NULL);
         return 0;
 }
 
 
-int32_t
-afr_opendir_cbk (call_frame_t *frame, void *cookie,
-		 xlator_t *this, int32_t op_ret, int32_t op_errno,
-		 fd_t *fd)
-{
-        afr_private_t *priv              = NULL;
-	afr_local_t   *local             = NULL;
-        int32_t        up_children_count = 0;
-        int           ret                = -1;
+#define BACKEND_D_OFF_BITS 63
+#define PRESENT_D_OFF_BITS 63
 
-	int call_count = -1;
+#define ONE 1ULL
+#define MASK (~0ULL)
+#define PRESENT_MASK (MASK >> (64 - PRESENT_D_OFF_BITS))
+#define BACKEND_MASK (MASK >> (64 - BACKEND_D_OFF_BITS))
 
-        priv  = this->private;
-        local = frame->local;
-
-        up_children_count = afr_up_children_count (priv->child_count,
-                                                   local->child_up);
+#define TOP_BIT (ONE << (PRESENT_D_OFF_BITS - 1))
+#define SHIFT_BITS (max (0, (BACKEND_D_OFF_BITS - PRESENT_D_OFF_BITS + 1)))
 
-	LOCK (&frame->lock);
-	{
-		if (op_ret >= 0)
-			local->op_ret = op_ret;
+static uint64_t
+afr_bits_for (uint64_t num)
+{
+	uint64_t bits = 0, ctrl = 1;
 
-		local->op_errno = op_errno;
+	while (ctrl < num) {
+		ctrl *= 2;
+		bits ++;
 	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-                if (local->op_ret == 0) {
-
-                        ret = afr_fd_ctx_set (this, local->fd);
-
-                        if (ret) {
-                                local->op_ret = -1;
-                                local->op_errno = -1;
-                                gf_log (this->name, GF_LOG_ERROR, " failed to "
-                                        "set fd ctx for fd %d", local->fd);
-                                goto out;
-                        }
-                        if (!afr_is_opendir_done (this, local->fd->inode) &&
-                            up_children_count > 1) {
-
-                                /*
-                                 * This is the first opendir on this inode. We need
-                                 * to check if the directory's entries are the same
-                                 * on all subvolumes. This is needed in addition
-                                 * to regular entry self-heal because the readdir
-                                 * call is sent only to the first subvolume, and
-                                 * thus files that exist only there will never be healed
-                                 * otherwise (assuming changelog shows no anamolies).
-                                 */
-
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "reading contents of directory %s looking for mismatch",
-                                        local->loc.path);
-
-                                afr_examine_dir (frame, this);
-
-                        } else {
-                                AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                                                  local->op_errno, local->fd);
-                        }
-                } else {
-out:
-                        AFR_STACK_UNWIND (opendir, frame, local->op_ret,
-                                          local->op_errno, local->fd);
-                }
-        }
 
-	return 0;
+	return bits;
 }
 
-
-int32_t 
-afr_opendir (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, fd_t *fd)
+int
+afr_itransform (xlator_t *this, int subvol, uint64_t x, uint64_t *y_p)
 {
-	afr_private_t * priv        = NULL;
-	afr_local_t   * local       = NULL;
-
-	int             child_count = 0;
-	int             i           = 0;
-
-	int ret = -1;
-	int call_count = -1;
-
-	int32_t         op_ret   = -1;
-	int32_t         op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        afr_private_t *conf = NULL;
+        int         cnt = 0;
+        int         max = 0;
+        uint64_t    y = 0;
+        uint64_t    hi_mask = 0;
+        uint64_t    off_mask = 0;
+        int         max_bits = 0;
+
+        if (x == ((uint64_t) -1)) {
+                y = (uint64_t) -1;
+                goto out;
+        }
 
-	priv = this->private;
+        conf = this->private;
+        if (!conf)
+                goto out;
 
-	child_count = priv->child_count;
+        max = conf->child_count;
+        cnt = subvol;
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
+	if (max == 1) {
+		y = x;
 		goto out;
 	}
 
-        loc_copy (&local->loc, loc);
-
-	frame->local = local;
-	local->fd    = fd_ref (fd);
-
-	call_count = local->call_count;
-	
-	for (i = 0; i < child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND (frame, afr_opendir_cbk, 
-				    priv->children[i],
-				    priv->children[i]->fops->opendir,
-				    loc, fd);
-
-			if (!--call_count)
-				break;
-		}
-	}
-
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (opendir, frame, op_ret, op_errno, fd);
-	}
-
-	return 0;
-}
-
-
-/**
- * Common algorithm for directory read calls:
- * 
- * - Try the fop on the first child that is up
- * - if we have failed due to ENOTCONN:
- *     try the next child
- *
- * Applicable to: readdir
- */
-
+        max_bits = afr_bits_for (max);
 
-struct entry_name {
-        char *name;
-        struct list_head list;
-};
+        hi_mask = ~(PRESENT_MASK >> (max_bits + 1));
 
-
-static gf_boolean_t
-remembered_name (const char *name, struct list_head *entries)
-{
-        struct entry_name *e;
-        gf_boolean_t ret = _gf_false;
-
-        list_for_each_entry (e, entries, list) {
-                if (!strcmp (name, e->name)) {
-                        ret = _gf_true;
-                        goto out;
-                }
+        if (x & hi_mask) {
+                /* HUGE d_off */
+                off_mask = MASK << max_bits;
+                y = TOP_BIT | ((x >> SHIFT_BITS) & off_mask) | cnt;
+        } else {
+                /* small d_off */
+                y = ((x * max) + cnt);
         }
 
 out:
-        return ret;
-}
-
-
-static void
-afr_remember_entries (gf_dirent_t *entries, fd_t *fd)
-{
-	struct entry_name *n       = NULL;
-	gf_dirent_t *      entry   = NULL;
-
-	int ret = 0;
-
-	uint64_t      ctx;
-	afr_fd_ctx_t *fd_ctx;
-
-	ret = fd_ctx_get (fd, THIS, &ctx);
-	if (ret < 0) {
-		gf_log (THIS->name, GF_LOG_DEBUG,
-			"could not get fd ctx for fd=%p", fd);
-		return;
-	}
-
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+        if (y_p)
+                *y_p = y;
 
-	list_for_each_entry (entry, &entries->list, list) {
-		n = GF_CALLOC (1, sizeof (*n), gf_afr_mt_entry_name);
-		n->name = gf_strdup (entry->d_name);
-		INIT_LIST_HEAD (&n->list);
-
-		list_add (&n->list, &fd_ctx->entries);
-	}
+        return 0;
 }
 
 
-static off_t
-afr_filter_entries (gf_dirent_t *entries, fd_t *fd)
+int
+afr_deitransform (xlator_t *this, uint64_t y, int *subvol_p,
+                  uint64_t *x_p)
 {
-	gf_dirent_t *entry, *tmp;
-	int ret = 0;
-
-	uint64_t      ctx;
-	afr_fd_ctx_t *fd_ctx;
-
-	off_t offset = 0;
-
-	ret = fd_ctx_get (fd, THIS, &ctx);
-	if (ret < 0) {
-		gf_log (THIS->name, GF_LOG_DEBUG,
-			"could not get fd ctx for fd=%p", fd);
-		return -1;
-	}
-
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-	list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-		offset = entry->d_off;
-
-		if (remembered_name (entry->d_name, &fd_ctx->entries)) {
-			list_del (&entry->list);
-			GF_FREE (entry);
-		}
+        afr_private_t *conf = NULL;
+        int         cnt = 0;
+        int         max = 0;
+        uint64_t    x = 0;
+        int         subvol = 0;
+        int         max_bits = 0;
+        uint64_t    off_mask = 0;
+        uint64_t    host_mask = 0;
+
+        if (!this->private)
+                return -1;
+
+        conf = this->private;
+        max = conf->child_count;
+
+	if (max == 1) {
+		x = y;
+		cnt = 0;
+		goto out;
 	}
 
-	return offset;
-}
+        if (y & TOP_BIT) {
+                /* HUGE d_off */
+                max_bits = afr_bits_for (max);
+                off_mask = (MASK << max_bits);
+                host_mask = ~(off_mask);
 
+                x = ((y & ~TOP_BIT) & off_mask) << SHIFT_BITS;
 
-static void
-afr_forget_entries (fd_t *fd)
-{
-	struct entry_name *entry, *tmp;
-	int ret = 0;
+                cnt = y & host_mask;
+	} else {
+                /* small d_off */
+                cnt = y % max;
+                x = y / max;
+        }
 
-	uint64_t      ctx;
-	afr_fd_ctx_t *fd_ctx;
+out:
+        subvol = cnt;
 
-	ret = fd_ctx_get (fd, THIS, &ctx);
-	if (ret < 0) {
-		gf_log (THIS->name, GF_LOG_DEBUG,
-			"could not get fd ctx for fd=%p", fd);
-		return;
-	}
+        if (subvol_p)
+                *subvol_p = subvol;
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+        if (x_p)
+                *x_p = x;
 
-	list_for_each_entry_safe (entry, tmp, &fd_ctx->entries, list) {
-		GF_FREE (entry->name);
-		list_del (&entry->list);
-		GF_FREE (entry);
-	}
+        return 0;
 }
 
 
-int32_t
-afr_readdir_cbk (call_frame_t *frame, void *cookie,
-		 xlator_t *this, int32_t op_ret, int32_t op_errno,
-		 gf_dirent_t *entries)
+static void
+afr_readdir_transform_entries (gf_dirent_t *subvol_entries, int subvol,
+			       gf_dirent_t *entries, fd_t *fd)
 {
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
+	afr_private_t *priv = NULL;
+        gf_dirent_t *entry = NULL;
+        gf_dirent_t *tmp = NULL;
+	unsigned char *data_readable = NULL;
+	unsigned char *metadata_readable = NULL;
+	int gen = 0;
+
+	priv = THIS->private;
+
+	data_readable = alloca0 (priv->child_count);
+	metadata_readable = alloca0 (priv->child_count);
+
+        list_for_each_entry_safe (entry, tmp, &subvol_entries->list, list) {
+                if (__is_root_gfid (fd->inode->gfid) &&
+                    !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
+			continue;
+                }
 
-        gf_dirent_t * entry = NULL;
-        gf_dirent_t * tmp   = NULL;
+		list_del_init (&entry->list);
+		afr_itransform (THIS, subvol, entry->d_off, &entry->d_off);
+		list_add_tail (&entry->list, &entries->list);
 
-        int child_index = -1;
+		if (entry->inode) {
+			gen = 0;
+			afr_inode_read_subvol_get (entry->inode, THIS,
+						   data_readable,
+						   metadata_readable, &gen);
 
-	priv     = this->private;
-	local = frame->local;
-        child_index = (long) cookie;
+			if (gen != priv->event_generation ||
+				!data_readable[subvol] ||
+				!metadata_readable[subvol]) {
 
-	if (op_ret != -1) {
-                list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-                        entry->d_ino = afr_itransform (entry->d_ino,
-                                                       priv->child_count,
-                                                       child_index);
-
-                        if ((local->fd->inode == local->fd->inode->table->root)
-                            && !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
-                                list_del_init (&entry->list);
-                                GF_FREE (entry);
-                        }
-                }
-    	}
-
-        AFR_STACK_UNWIND (readdir, frame, op_ret, op_errno, entries);
-
-	return 0;
+				inode_unref (entry->inode);
+				entry->inode = NULL;
+			}
+		}
+        }
 }
 
 
 int32_t
-afr_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+afr_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		 int32_t op_ret, int32_t op_errno, gf_dirent_t *subvol_entries,
+		 dict_t *xdata)
 {
-        afr_private_t * priv     = NULL;
-        afr_local_t *   local    = NULL;
-        xlator_t **     children = NULL;
-        ino_t           inum = 0;
-
-        int call_child = 0;
-        int ret        = 0;
-
-        gf_dirent_t * entry = NULL;
-        gf_dirent_t * tmp   = NULL;
-
-        int child_index = -1;
+        afr_local_t *local = NULL;
+	gf_dirent_t  entries;
 
-        uint64_t      ctx    = 0;
-        afr_fd_ctx_t *fd_ctx = NULL;
-
-	off_t offset = 0;
-
-        priv     = this->private;
-        children = priv->children;
+	INIT_LIST_HEAD (&entries.list);
 
         local = frame->local;
 
-        child_index = (long) cookie;
-
-	if (priv->strict_readdir) {
-		ret = fd_ctx_get (local->fd, this, &ctx);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"could not get fd ctx for fd=%p", local->fd);
-			op_ret   = -1;
-			op_errno = -ret;
-			goto out;
-		}
-
-		fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-		if (child_went_down (op_ret, op_errno)) {
-			if (all_tried (child_index, priv->child_count)) {
-				goto out;
-			}
-
-			call_child = ++child_index;
-
-			gf_log (this->name, GF_LOG_TRACE,
-				"starting readdir afresh on child %d, offset %"PRId64,
-				call_child, (uint64_t) 0);
-
-			fd_ctx->failed_over = _gf_true;
-
-			STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
-					   (void *) (long) call_child,
-					   children[call_child],
-					   children[call_child]->fops->readdirp, local->fd,
-					   local->cont.readdir.size, 0);
-			return 0;
-		}
-	}
+        if (op_ret < 0 && !local->cont.readdir.offset) {
+		/* failover only if this was first readdir, detected
+		   by offset == 0 */
+		local->op_ret = op_ret;
+		local->op_errno = op_errno;
 
-	if (op_ret != -1) {
-		list_for_each_entry_safe (entry, tmp, &entries->list, list) {
-			inum = afr_itransform (entry->d_ino, priv->child_count,
-					       child_index);
-			entry->d_ino = inum;
-			inum  = afr_itransform (entry->d_stat.ia_ino,
-						priv->child_count, child_index);
-			entry->d_stat.ia_ino = inum;
-
-			if ((local->fd->inode == local->fd->inode->table->root)
-			    && !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR)) {
-				list_del_init (&entry->list);
-				GF_FREE (entry);
-			}
-		}
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
 	}
 
-	if (priv->strict_readdir) {
-		if (fd_ctx->failed_over) {
-			if (list_empty (&entries->list)) {
-				goto out;
-			}
-
-			offset = afr_filter_entries (entries, local->fd);
+	if (op_ret >= 0)
+		afr_readdir_transform_entries (subvol_entries, (long) cookie,
+					       &entries, local->fd);
 
-			afr_remember_entries (entries, local->fd);
-
-			if (list_empty (&entries->list)) {
-				/* All the entries we got were duplicate. We
-				   shouldn't send an empty list now, because
-				   that'll make the application stop reading. So
-				   try to get more entries */
-
-				gf_log (this->name, GF_LOG_TRACE,
-					"trying to fetch non-duplicate entries from offset %"PRId64", child %s",
-					offset, children[child_index]->name);
-
-				STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
-						   (void *) (long) child_index,
-						   children[child_index],
-						   children[child_index]->fops->readdirp,
-						   local->fd, local->cont.readdir.size, offset);
-				return 0;
-			}
-		} else {
-			afr_remember_entries (entries, local->fd);
-		}
-	}
-
-out:
-        AFR_STACK_UNWIND (readdirp, frame, op_ret, op_errno, entries);
+        AFR_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries, xdata);
 
         return 0;
 }
 
 
-int32_t
-afr_do_readdir (call_frame_t *frame, xlator_t *this,
-	        fd_t *fd, size_t size, off_t offset, int whichop)
+int
+afr_readdir_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_private_t * priv       = NULL;
-	xlator_t **     children   = NULL;
-	int             call_child = 0;
-	afr_local_t     *local     = NULL;
-
-        uint64_t      ctx;
-        afr_fd_ctx_t *fd_ctx;
-
-	int ret = -1;
-
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv     = this->private;
-	children = priv->children;
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
 
-	frame->local = local;
+	priv = this->private;
+	local = frame->local;
 
-	call_child = afr_first_up_child (priv);
-	if (call_child == -1) {
-		op_errno = ENOTCONN;
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no child is up");
-		goto out;
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (readdir, frame, local->op_ret,
+				  local->op_errno, 0, 0);
+		return 0;
 	}
 
-        local->fd                  = fd_ref (fd);
-        local->cont.readdir.size   = size;
-
-	if (priv->strict_readdir) {
-		ret = fd_ctx_get (fd, this, &ctx);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"could not get fd ctx for fd=%p", fd);
-			op_errno = -ret;
-			goto out;
-		}
+        if (local->op == GF_FOP_READDIR)
+                STACK_WIND_COOKIE (frame, afr_readdir_cbk,
+                                   (void *) (long) subvol,
+                                   priv->children[subvol],
+                                   priv->children[subvol]->fops->readdir,
+				   local->fd, local->cont.readdir.size,
+                                   local->cont.readdir.offset,
+				   local->xdata_req);
+        else
+                STACK_WIND_COOKIE (frame, afr_readdir_cbk,
+                                   (void *) (long) subvol,
+                                   priv->children[subvol],
+                                   priv->children[subvol]->fops->readdirp,
+				   local->fd, local->cont.readdir.size,
+				   local->cont.readdir.offset,
+				   local->xdata_req);
+	return 0;
+}
 
-		fd_ctx = (afr_fd_ctx_t *)(long) ctx;
 
-		if (fd_ctx->last_tried != call_child) {
-			gf_log (this->name, GF_LOG_TRACE,
-				"first up child has changed from %d to %d, restarting readdir from offset 0",
-				fd_ctx->last_tried, call_child);
+int
+afr_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+		off_t offset, int whichop, dict_t *dict)
+{
+        afr_local_t   *local     = NULL;
+        int32_t       op_errno   = 0;
+	int           subvol = -1;
 
-			fd_ctx->failed_over = _gf_true;
-			offset = 0;
-		}
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-		fd_ctx->last_tried = call_child;
+	local->op = whichop;
+        local->fd = fd_ref (fd);
+        local->cont.readdir.size = size;
+	local->cont.readdir.offset = offset;
+        local->xdata_req = (dict)? dict_ref (dict) : NULL;
+
+	if (offset == 0) {
+		/* First readdir has option of failing over and selecting
+		   an appropriate read subvolume */
+		afr_read_txn (frame, this, fd->inode, afr_readdir_wind,
+			      AFR_DATA_TRANSACTION);
+	} else {
+		/* But continued readdirs MUST stick to the same subvolume
+		   without an option to failover */
+		afr_deitransform (this, offset, &subvol,
+				  (uint64_t *)&local->cont.readdir.offset);
+		afr_readdir_wind (frame, this, subvol);
 	}
 
-        if (whichop == GF_FOP_READDIR)
-                STACK_WIND_COOKIE (frame, afr_readdir_cbk,
-                                   (void *) (long) call_child,
-                                   children[call_child],
-                                   children[call_child]->fops->readdir, fd,
-                                   size, offset);
-        else
-                STACK_WIND_COOKIE (frame, afr_readdirp_cbk,
-                                   (void *) (long) call_child,
-                                   children[call_child],
-                                   children[call_child]->fops->readdirp, fd,
-                                   size, offset);
-
-	op_ret = 0;
+        return 0;
 out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (readdir, frame, op_ret, op_errno, NULL);
-	}
-	return 0;
+        AFR_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
+        return 0;
 }
 
 
 int32_t
 afr_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t offset)
+             off_t offset, dict_t *xdata)
 {
-        afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIR);
+	afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIR, xdata);
+
         return 0;
 }
 
 
 int32_t
 afr_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t offset)
+              off_t offset, dict_t *dict)
 {
-        afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIRP);
+        afr_do_readdir (frame, this, fd, size, offset, GF_FOP_READDIRP, dict);
+
         return 0;
 }
 
@@ -746,8 +422,7 @@ afr_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 int32_t
 afr_releasedir (xlator_t *this, fd_t *fd)
 {
-	afr_forget_entries (fd);
         afr_cleanup_fd_ctx (this, fd);
 
-	return 0;
+        return 0;
 }
diff --git a/xlators/cluster/afr/src/afr-dir-read.h b/xlators/cluster/afr/src/afr-dir-read.h
index e071107e6..09456d159 100644
--- a/xlators/cluster/afr/src/afr-dir-read.h
+++ b/xlators/cluster/afr/src/afr-dir-read.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __DIR_READ_H__
@@ -23,28 +14,23 @@
 
 int32_t
 afr_opendir (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, fd_t *fd);
+	     loc_t *loc, fd_t *fd, dict_t *xdata);
 
 int32_t
 afr_releasedir (xlator_t *this, fd_t *fd);
 
 int32_t
 afr_readdir (call_frame_t *frame, xlator_t *this,
-	     fd_t *fd, size_t size, off_t offset);
+	     fd_t *fd, size_t size, off_t offset, dict_t *xdata);
 
 
 int32_t
 afr_readdirp (call_frame_t *frame, xlator_t *this,
-	     fd_t *fd, size_t size, off_t offset);
-
-int32_t
-afr_getdents (call_frame_t *frame, xlator_t *this,
-	      fd_t *fd, size_t size, off_t offset, int32_t flag);
-
+              fd_t *fd, size_t size, off_t offset, dict_t *dict);
 
 int32_t
 afr_checksum (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, int32_t flags);
+	      loc_t *loc, int32_t flags, dict_t *xdata);
 
 
 #endif /* __DIR_READ_H__ */
diff --git a/xlators/cluster/afr/src/afr-dir-write.c b/xlators/cluster/afr/src/afr-dir-write.c
index 9f77324a3..465dde54f 100644
--- a/xlators/cluster/afr/src/afr-dir-write.c
+++ b/xlators/cluster/afr/src/afr-dir-write.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2009 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -43,820 +34,753 @@
 #include "common-utils.h"
 #include "compat-errno.h"
 #include "compat.h"
+#include "byte-order.h"
 
 #include "afr.h"
 #include "afr-transaction.h"
 
-
 void
-afr_build_parent_loc (loc_t *parent, loc_t *child)
-{
-	char *tmp = NULL;
-
-	if (!child->parent) {
-		loc_copy (parent, child);
-		return;
-	}
-
-	tmp = gf_strdup (child->path);
-	parent->path   = gf_strdup (dirname (tmp));
-	GF_FREE (tmp);
-
-        parent->name   = strrchr (parent->path, '/');
-	if (parent->name)
-		parent->name++;
-
-	parent->inode  = inode_ref (child->parent);
-	parent->parent = inode_parent (parent->inode, 0, NULL);
-	parent->ino    = parent->inode->ino;
-}
-
-/* {{{ create */
+afr_mark_entry_pending_changelog (call_frame_t *frame, xlator_t *this);
 
 int
-afr_create_unwind (call_frame_t *frame, xlator_t *this)
+afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno)
 {
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-        struct iatt  *unwind_buf = NULL;
+        int     ret = -1;
+        char    *child_path = NULL;
 
-	local = frame->local;
+        if (!child->parent) {
+                if (op_errno)
+                        *op_errno = EINVAL;
+                goto out;
+        }
 
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
+        child_path = gf_strdup (child->path);
+        if (!child_path) {
+                if (op_errno)
+                        *op_errno = ENOMEM;
+                goto out;
+        }
 
-	if (main_frame) {
-                if (local->cont.create.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.create.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.create.buf;
-                }
+        parent->path = gf_strdup (dirname (child_path));
+        if (!parent->path) {
+                if (op_errno)
+                        *op_errno = ENOMEM;
+                goto out;
+        }
 
-                unwind_buf->ia_ino = local->cont.create.ino;
+        parent->inode = inode_ref (child->parent);
+	uuid_copy (parent->gfid, child->pargfid);
 
-                local->cont.create.preparent.ia_ino  = local->cont.create.parent_ino;
-                local->cont.create.postparent.ia_ino = local->cont.create.parent_ino;
+        ret = 0;
+out:
+        GF_FREE (child_path);
 
-		AFR_STACK_UNWIND (create, main_frame,
-                                  local->op_ret, local->op_errno,
-				  local->cont.create.fd,
-				  local->cont.create.inode,
-				  unwind_buf, &local->cont.create.preparent,
-                                  &local->cont.create.postparent);
-        }
-        
-	return 0;
+        return ret;
 }
 
 
-int
-afr_create_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		     int32_t op_ret, int32_t op_errno, 
-		     fd_t *fd, inode_t *inode, struct iatt *buf,
-                     struct iatt *preparent, struct iatt *postparent)
+static void
+__afr_dir_write_finalize (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-        uint64_t      ctx;
-        afr_fd_ctx_t *fd_ctx;
-
-        int ret = 0;
-
-	int call_count = -1;
-	int child_index = -1;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int inode_read_subvol = -1;
+	int parent_read_subvol = -1;
+	int parent2_read_subvol = -1;
+	int i = 0;
 
 	local = frame->local;
-	priv  = this->private;
-
-	child_index = (long) cookie;
+	priv = this->private;
 
-	LOCK (&frame->lock);
-	{
-		if (afr_fop_failed (op_ret, op_errno))
-			afr_transaction_fop_failed (frame, this, child_index);
-                
-		if (op_ret != -1) {
-			local->op_ret = op_ret;
-
-                        ret = afr_fd_ctx_set (this, fd);
-
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "could not set ctx on fd=%p", fd);
-
-                                local->op_ret   = -1;
-                                local->op_errno = -ret;
-                                goto unlock;
-                        }
-
-                        ret = fd_ctx_get (fd, this, &ctx);
-
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "could not get fd ctx for fd=%p", fd);
-                                local->op_ret   = -1;
-                                local->op_errno = -ret;
-                                goto unlock;
-                        }
-
-                        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-                        fd_ctx->opened_on[child_index] = 1;
-                        fd_ctx->flags                  = local->cont.create.flags;
-
-                        if (local->success_count == 0) {
-				local->cont.create.buf        = *buf;
-
-                                local->cont.create.ino =
-                                        afr_itransform (buf->ia_ino,
-                                                        priv->child_count,
-                                                        child_index);
-
-                                if (priv->read_child >= 0) {
-                                        afr_set_read_child (this, inode, 
-                                                            priv->read_child);
-                                } else {
-                                        afr_set_read_child (this, inode, 
-                                                            local->read_child_index);
-                                }
-			}
-                        
-                        if (child_index == local->first_up_child) {
-                                local->cont.create.ino =
-                                        afr_itransform (buf->ia_ino,
-                                                        priv->child_count,
-                                                        local->first_up_child);
-                        }
-                        
-                        if (child_index == local->read_child_index) {
-                                local->cont.create.read_child_buf = *buf;
-                                local->cont.create.preparent      = *preparent;
-                                local->cont.create.postparent     = *postparent;
-                        }
-
-			local->cont.create.inode = inode;
-
-			local->success_count++;
+	if (local->inode) {
+		afr_replies_interpret (frame, this, local->inode);
+		inode_read_subvol = afr_data_subvol_get (local->inode, this,
+							 NULL, NULL);
+	}
+	if (local->parent)
+		parent_read_subvol = afr_data_subvol_get (local->parent, this,
+							  NULL, NULL);
+	if (local->parent2)
+		parent2_read_subvol = afr_data_subvol_get (local->parent2, this,
+							   NULL, NULL);
+
+	local->op_ret = -1;
+	local->op_errno = afr_final_errno (local, priv);
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->replies[i].valid)
+			continue;
+		if (local->replies[i].op_ret < 0) {
+			if (local->inode)
+				afr_inode_read_subvol_reset (local->inode,
+							     this);
+			if (local->parent)
+				afr_inode_read_subvol_reset (local->parent,
+							     this);
+			if (local->parent2)
+				afr_inode_read_subvol_reset (local->parent2,
+							     this);
+			continue;
 		}
 
-		local->op_errno = op_errno;
-	}
-
-unlock:
-	UNLOCK (&frame->lock);
+		if (local->op_ret == -1) {
+			local->op_ret = local->replies[i].op_ret;
+			local->op_errno = local->replies[i].op_errno;
+
+			local->cont.dir_fop.buf =
+				local->replies[i].poststat;
+			local->cont.dir_fop.preparent =
+				local->replies[i].preparent;
+			local->cont.dir_fop.postparent =
+				local->replies[i].postparent;
+			local->cont.dir_fop.prenewparent =
+				local->replies[i].preparent2;
+			local->cont.dir_fop.postnewparent =
+				local->replies[i].postparent2;
+			if (local->replies[i].xdata)
+				local->xdata_rsp =
+					dict_ref (local->replies[i].xdata);
+			continue;
+		}
 
-	call_count = afr_frame_return (frame);
+		if (i == inode_read_subvol) {
+			local->cont.dir_fop.buf =
+				local->replies[i].poststat;
+			if (local->replies[i].xdata) {
+				if (local->xdata_rsp)
+					dict_unref (local->xdata_rsp);
+				local->xdata_rsp =
+					dict_ref (local->replies[i].xdata);
+			}
+		}
 
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
+		if (i == parent_read_subvol) {
+			local->cont.dir_fop.preparent =
+				local->replies[i].preparent;
+			local->cont.dir_fop.postparent =
+				local->replies[i].postparent;
+		}
 
-		local->transaction.resume (frame, this);
+		if (i == parent2_read_subvol) {
+			local->cont.dir_fop.prenewparent =
+				local->replies[i].preparent2;
+			local->cont.dir_fop.postnewparent =
+				local->replies[i].postparent2;
+		}
 	}
-	
-	return 0;
 }
 
 
-int
-afr_create_wind (call_frame_t *frame, xlator_t *this)
+static void
+__afr_dir_write_fill (call_frame_t *frame, xlator_t *this, int child_index,
+		      int op_ret, int op_errno, struct iatt *poststat,
+		      struct iatt *preparent, struct iatt *postparent,
+		      struct iatt *preparent2, struct iatt *postparent2,
+		      dict_t *xdata)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+	afr_fd_ctx_t *fd_ctx = NULL;
+
+        local = frame->local;
+	fd_ctx = local->fd_ctx;
+
+	local->replies[child_index].valid = 1;
+	local->replies[child_index].op_ret = op_ret;
+	local->replies[child_index].op_errno = op_errno;
+
+	if (op_ret >= 0) {
+		if (poststat)
+			local->replies[child_index].poststat = *poststat;
+		if (preparent)
+			local->replies[child_index].preparent = *preparent;
+		if (postparent)
+			local->replies[child_index].postparent = *postparent;
+		if (preparent2)
+			local->replies[child_index].preparent2 = *preparent2;
+		if (postparent2)
+			local->replies[child_index].postparent2 = *postparent2;
+		if (xdata)
+			local->replies[child_index].xdata = dict_ref (xdata);
+
+		if (fd_ctx)
+			fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+	} else {
+		if (op_errno != ENOTEMPTY)
+			afr_transaction_fop_failed (frame, this, child_index);
+		if (fd_ctx)
+			fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+	}
 
-	int call_count = -1;
-	int i = 0;
+        return;
+}
 
-	local = frame->local;
-	priv = this->private;
 
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
+static int
+__afr_dir_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		     int op_ret, int op_errno, struct iatt *buf,
+		     struct iatt *preparent, struct iatt *postparent,
+		     struct iatt *preparent2, struct iatt *postparent2,
+                     dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+        int child_index = (long) cookie;
+        int call_count = -1;
 
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
+        local = frame->local;
 
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_create_wind_cbk,
-					   (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->create,
-					   &local->loc, 
-					   local->cont.create.flags, 
-					   local->cont.create.mode, 
-					   local->cont.create.fd,
-                                           local->cont.create.params);
-			if (!--call_count)
-				break;
-		}
+	LOCK (&frame->lock);
+	{
+		__afr_dir_write_fill (frame, this, child_index, op_ret,
+				      op_errno, buf, preparent, postparent,
+				      preparent2, postparent2, xdata);
 	}
-	
-	return 0;
-}
-
+	UNLOCK (&frame->lock);
+        call_count = afr_frame_return (frame);
 
-int
-afr_create_done (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t * local = NULL;
+        if (call_count == 0) {
+		__afr_dir_write_finalize (frame, this);
 
-	local = frame->local;
+		if (afr_txn_nothing_failed (frame, this))
+			local->transaction.unwind (frame, this);
 
-	local->transaction.unwind (frame, this);
+		afr_mark_entry_pending_changelog (frame, this);
 
-	AFR_STACK_DESTROY (frame);
+                local->transaction.resume (frame, this);
+        }
 
-	return 0;
+        return 0;
 }
 
 
 int
-afr_create (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t flags, mode_t mode,
-            fd_t *fd, dict_t *params)
+afr_mark_new_entry_changelog_cbk (call_frame_t *frame, void *cookie,
+				  xlator_t *this, int op_ret, int op_errno,
+                                  dict_t *xattr, dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
+        int call_count = 0;
 
-	int ret = -1;
+        call_count = afr_frame_return (frame);
 
-	int op_ret   = -1;
-	int op_errno = 0;
+        if (call_count == 0)
+                AFR_STACK_DESTROY (frame);
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        return 0;
+}
 
-	priv = this->private;
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
+void
+afr_mark_new_entry_changelog (call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t  *new_frame  = NULL;
+        afr_local_t   *local      = NULL;
+        afr_local_t   *new_local  = NULL;
+        afr_private_t *priv       = NULL;
+        dict_t        *xattr      = NULL;
+        int32_t       **changelog = NULL;
+        int           i           = 0;
+	int           idx         = 0;
+        int           op_errno    = ENOMEM;
+	unsigned char *pending    = NULL;
+	int           call_count   = 0;
+
+        local = frame->local;
+        priv = this->private;
+
+        new_frame = copy_frame (frame);
+        if (!new_frame)
+                goto out;
+
+	new_local = AFR_FRAME_INIT (new_frame, op_errno);
+	if (!new_local)
 		goto out;
-	}
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+        changelog = afr_matrix_create (priv->child_count, AFR_NUM_CHANGE_LOGS);
+        if (!changelog)
+                goto out;
 
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
+        xattr = dict_new ();
+	if (!xattr)
 		goto out;
-	}
-
-	transaction_frame->local = local;
-
-	loc_copy (&local->loc, loc);
-
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr) 
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
-
-	local->cont.create.flags = flags;
-	local->cont.create.mode  = mode;
-	local->cont.create.fd    = fd_ref (fd);
-        if (params)
-                local->cont.create.params = dict_ref (params);
-
-        if (loc->parent)
-                local->cont.create.parent_ino = loc->parent->ino;
 
-	local->transaction.fop    = afr_create_wind;
-	local->transaction.done   = afr_create_done;
-	local->transaction.unwind = afr_create_unwind;
+	idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
 
-	afr_build_parent_loc (&local->transaction.parent_loc, loc);
+	pending = alloca0 (priv->child_count);
 
-	local->transaction.main_frame = frame;
-	local->transaction.basename = AFR_BASENAME (loc->path);
-
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+	for (i = 0; i < priv->child_count; i++) {
+		if (local->transaction.pre_op[i] &&
+		    !local->transaction.failed_subvols[i]) {
+			call_count ++;
+			continue;
+		}
 
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (create, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
+		changelog[i][idx] = hton32(1);
+		pending[i] = 1;
 	}
 
-	return 0;
-}
-
-/* }}} */
-
-/* {{{ mknod */
-
-int
-afr_mknod_unwind (call_frame_t *frame, xlator_t *this)
-{
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
+        new_local->pending = changelog;
+        uuid_copy (new_local->loc.gfid, local->cont.dir_fop.buf.ia_gfid);
+        new_local->loc.inode = inode_ref (local->inode);
 
-        struct iatt *unwind_buf = NULL;
 
-	local = frame->local;
+	afr_set_pending_dict (priv, xattr, changelog);
 
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
+        new_local->call_count = call_count;
 
-	if (main_frame) {
-                if (local->cont.mknod.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.mknod.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.mknod.buf;
-                }
+        for (i = 0; i < priv->child_count; i++) {
+		if (pending[i])
+                        continue;
 
-                unwind_buf->ia_ino = local->cont.mknod.ino;
-
-                local->cont.mknod.preparent.ia_ino  = local->cont.mknod.parent_ino;
-                local->cont.mknod.postparent.ia_ino = local->cont.mknod.parent_ino;
-
-		AFR_STACK_UNWIND (mknod, main_frame,
-                                  local->op_ret, local->op_errno,
-				  local->cont.mknod.inode,
-				  unwind_buf, &local->cont.mknod.preparent,
-                                  &local->cont.mknod.postparent);
+                STACK_WIND_COOKIE (new_frame, afr_mark_new_entry_changelog_cbk,
+                                   (void *) (long) i, priv->children[i],
+                                   priv->children[i]->fops->xattrop,
+                                   &new_local->loc, GF_XATTROP_ADD_ARRAY,
+                                   xattr, NULL);
+		if (!--call_count)
+			break;
         }
 
-	return 0;
+        new_frame = NULL;
+out:
+        if (new_frame)
+                AFR_STACK_DESTROY (new_frame);
+	if (xattr)
+		dict_unref (xattr);
+        return;
 }
 
 
-int
-afr_mknod_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                    struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+void
+afr_mark_entry_pending_changelog (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+	int pre_op_count = 0;
+	int failed_count = 0;
 
-	int call_count = -1;
-	int child_index = -1;
+        local = frame->local;
+        priv  = this->private;
 
-	local = frame->local;
-	priv = this->private;
-
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (afr_fop_failed (op_ret, op_errno))
-			afr_transaction_fop_failed (frame, this, child_index);
-		
-		if (op_ret != -1) {
-			local->op_ret = op_ret;
-
-			if (local->success_count == 0){
-				local->cont.mknod.buf   = *buf;
-				local->cont.mknod.ino   = 
-					afr_itransform (buf->ia_ino,
-							priv->child_count,
-							child_index);
-
-                                if (priv->read_child >= 0) {
-                                        afr_set_read_child (this, inode,
-                                                            priv->read_child);
-                                } else {
-                                        afr_set_read_child (this, inode,
-                                                            local->read_child_index);
-                                }
-			}
+        if (local->op_ret < 0)
+		return;
 
-                        if (child_index == local->first_up_child) {
-                                local->cont.mknod.ino =
-                                        afr_itransform (buf->ia_ino,
-                                                        priv->child_count,
-                                                        local->first_up_child);
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.mknod.read_child_buf = *buf;
-                                local->cont.mknod.preparent      = *preparent;
-                                local->cont.mknod.postparent     = *postparent;
-                        }
-                        
-			local->cont.mknod.inode = inode;
-
-			local->success_count++;
-		}
+	if (local->op != GF_FOP_CREATE && local->op != GF_FOP_MKNOD)
+		return;
 
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
+	pre_op_count = AFR_COUNT (local->transaction.pre_op, priv->child_count);
+	failed_count = AFR_COUNT (local->transaction.failed_subvols,
+				  priv->child_count);
 
-	call_count = afr_frame_return (frame);
+	if (pre_op_count == priv->child_count && !failed_count)
+		return;
 
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
+        afr_mark_new_entry_changelog (frame, this);
 
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+        return;
 }
 
 
-int32_t
-afr_mknod_wind (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
+/* {{{ create */
 
-	int call_count = -1;
-	int i = 0;
+int
+afr_create_unwind (call_frame_t *frame, xlator_t *this)
+{
+        call_frame_t *main_frame = NULL;
+        afr_local_t  *local = NULL;
 
-	local = frame->local;
-	priv  = this->private;
+        local = frame->local;
 
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
+	main_frame = afr_transaction_detach_fop_frame (frame);
 
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
+        if (!main_frame)
 		return 0;
-	}
 
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_mknod_wind_cbk, (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->mknod,
-					   &local->loc, local->cont.mknod.mode,
-					   local->cont.mknod.dev,
-                                           local->cont.mknod.params);
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+	AFR_STACK_UNWIND (create, main_frame, local->op_ret, local->op_errno,
+			  local->cont.create.fd, local->inode,
+			  &local->cont.dir_fop.buf,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+        return 0;
 }
 
 
 int
-afr_mknod_done (call_frame_t *frame, xlator_t *this)
+afr_create_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno,
+                     fd_t *fd, inode_t *inode, struct iatt *buf,
+                     struct iatt *preparent, struct iatt *postparent,
+                     dict_t *xdata)
 {
-	afr_local_t * local = NULL;
-
-	local = frame->local;
+        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
+				    preparent, postparent, NULL, NULL, xdata);
+}
 
-	local->transaction.unwind (frame, this);
-	AFR_STACK_DESTROY (frame);
 
-	return 0;
+int
+afr_create_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+
+        local = frame->local;
+        priv = this->private;
+
+	STACK_WIND_COOKIE (frame, afr_create_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->create,
+			   &local->loc, local->cont.create.flags,
+			   local->cont.create.mode, local->umask,
+			   local->cont.create.fd, local->xdata_req);
+        return 0;
 }
 
 
 int
-afr_mknod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dev_t dev, dict_t *params)
+afr_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+	    mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
 
-	int ret = -1;
+        priv = this->private;
 
-	int op_ret   = -1;
-	int op_errno = 0;
+        QUORUM_CHECK(create,out);
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv = this->private;
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
+                goto out;
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
 		goto out;
-	}
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+        loc_copy (&local->loc, loc);
 
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
+	local->fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!local->fd_ctx)
 		goto out;
-	}
-
-	transaction_frame->local = local;
-
-	loc_copy (&local->loc, loc);
 
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr) 
-                        % (priv->child_count);
-        }
-        UNLOCK (&priv->read_child_lock);
-
-	local->cont.mknod.mode  = mode;
-	local->cont.mknod.dev   = dev;
-        if (params)
-                local->cont.mknod.params = dict_ref (params);
-
-        if (loc->parent)
-                local->cont.mknod.parent_ino = loc->parent->ino;
+	local->inode = inode_ref (loc->inode);
+	local->parent = inode_ref (loc->parent);
 
-	local->transaction.fop    = afr_mknod_wind;
-	local->transaction.done   = afr_mknod_done;
-	local->transaction.unwind = afr_mknod_unwind;
+        local->op                = GF_FOP_CREATE;
+        local->cont.create.flags = flags;
+        local->cont.create.mode  = mode;
+        local->cont.create.fd    = fd_ref (fd);
+        local->umask  = umask;
 
-	afr_build_parent_loc (&local->transaction.parent_loc, loc);
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-	local->transaction.main_frame = frame;
-	local->transaction.basename = AFR_BASENAME (loc->path);
+	if (!local->xdata_req)
+		goto out;
 
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        local->transaction.wind   = afr_create_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_create_unwind;
+
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
+                                    &op_errno);
+        if (ret)
+                goto out;
+
+        local->transaction.main_frame = frame;
+        local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
-	op_ret = 0;
+	return 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (mknod, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL, NULL);
-	}
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
 
-	return 0;
+	AFR_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+			  NULL, NULL);
+        return 0;
 }
 
 /* }}} */
 
-/* {{{ mkdir */
-
+/* {{{ mknod */
 
 int
-afr_mkdir_unwind (call_frame_t *frame, xlator_t *this)
+afr_mknod_unwind (call_frame_t *frame, xlator_t *this)
 {
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-
-        struct iatt *unwind_buf = NULL;
-        
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-                if (local->cont.mkdir.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.mkdir.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.mkdir.buf;
-                }
+        call_frame_t *main_frame = NULL;
+        afr_local_t  *local = NULL;
 
-                unwind_buf->ia_ino = local->cont.mkdir.ino;
+        local = frame->local;
 
-                local->cont.mkdir.preparent.ia_ino  = local->cont.mkdir.parent_ino;
-                local->cont.mkdir.postparent.ia_ino = local->cont.mkdir.parent_ino;
-
-		AFR_STACK_UNWIND (mkdir, main_frame,
-                                  local->op_ret, local->op_errno,
-				  local->cont.mkdir.inode,
-				  unwind_buf, &local->cont.mkdir.preparent,
-                        &local->cont.mkdir.postparent);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
-	return 0;
+	AFR_STACK_UNWIND (mknod, main_frame, local->op_ret, local->op_errno,
+			  local->inode, &local->cont.dir_fop.buf,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+        return 0;
 }
 
 
 int
-afr_mkdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+afr_mknod_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
+	return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
+				    preparent, postparent, NULL, NULL, xdata);
+}
 
-	int call_count = -1;
-	int child_index = -1;
 
-	local = frame->local;
-	priv = this->private;
+int
+afr_mknod_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+
+        local = frame->local;
+        priv  = this->private;
+
+	STACK_WIND_COOKIE (frame, afr_mknod_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->mknod,
+			   &local->loc, local->cont.mknod.mode,
+			   local->cont.mknod.dev, local->umask,
+			   local->xdata_req);
+        return 0;
+}
 
-	child_index = (long) cookie;
+int
+afr_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+           dev_t dev, mode_t umask, dict_t *xdata)
+{
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
 
-	LOCK (&frame->lock);
-	{
-		if (afr_fop_failed (op_ret, op_errno))
-			afr_transaction_fop_failed (frame, this, child_index);
+        priv = this->private;
 
-		if (op_ret != -1) {
-			local->op_ret           = op_ret;
+        QUORUM_CHECK(mknod,out);
 
-			if (local->success_count == 0) {
-				local->cont.mkdir.buf   = *buf;
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
+                goto out;
 
-                                local->cont.mkdir.ino =
-                                        afr_itransform (buf->ia_ino,
-                                                        priv->child_count,
-                                                        child_index);
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
-                                if (priv->read_child >= 0) {
-                                        afr_set_read_child (this, inode,
-                                                            priv->read_child);
-                                } else {
-                                        afr_set_read_child (this, inode,
-                                                            local->read_child_index);
-                                }
-			}
+        loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
+	local->parent = inode_ref (loc->parent);
 
-                        if (child_index == local->first_up_child) {
-                                local->cont.mkdir.ino =
-                                        afr_itransform (buf->ia_ino,
-                                                        priv->child_count,
-                                                        local->first_up_child);
-                        }
-                        
-                        if (child_index == local->read_child_index) {
-                                local->cont.mkdir.read_child_buf = *buf;
-                                local->cont.mkdir.preparent      = *preparent;
-                                local->cont.mkdir.postparent     = *postparent;
-                        }
-
-			local->cont.mkdir.inode = inode;
-
-			local->success_count++;
-		}
+        local->op               = GF_FOP_MKNOD;
+        local->cont.mknod.mode  = mode;
+        local->cont.mknod.dev   = dev;
+        local->umask = umask;
 
-		local->op_errno         = op_errno;
-	}
-	UNLOCK (&frame->lock);
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-	call_count = afr_frame_return (frame);
+	if (!local->xdata_req)
+		goto out;
 
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
+        local->transaction.wind   = afr_mknod_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_mknod_unwind;
+
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
+                                    &op_errno);
+        if (ret)
+                goto out;
+
+        local->transaction.main_frame = frame;
+        local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
+        }
 
-		local->transaction.resume (frame, this);
-	}
-	
 	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+			  NULL);
+        return 0;
 }
 
+/* }}} */
+
+/* {{{ mkdir */
+
 
 int
-afr_mkdir_wind (call_frame_t *frame, xlator_t *this)
+afr_mkdir_unwind (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-	
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv  = this->private;
+        call_frame_t *main_frame = NULL;
+        afr_local_t  *local = NULL;
 
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
+        local = frame->local;
 
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
 		return 0;
-	}
 
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_mkdir_wind_cbk,
-					   (void *) (long) i,	
-					   priv->children[i], 
-					   priv->children[i]->fops->mkdir,
-					   &local->loc, local->cont.mkdir.mode,
-                                           local->cont.mkdir.params);
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+	AFR_STACK_UNWIND (mkdir, main_frame, local->op_ret, local->op_errno,
+			  local->inode, &local->cont.dir_fop.buf,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+        return 0;
 }
 
 
 int
-afr_mkdir_done (call_frame_t *frame, xlator_t *this)
+afr_mkdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-	afr_local_t * local = NULL;
-
-	local = frame->local;
-
-	local->transaction.unwind (frame, this);
+	return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
+				    preparent, postparent, NULL, NULL, xdata);
+}
 
-	AFR_STACK_DESTROY (frame);
 
-	return 0;
+int
+afr_mkdir_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+
+        local = frame->local;
+        priv  = this->private;
+
+	STACK_WIND_COOKIE (frame, afr_mkdir_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->mkdir, &local->loc,
+			   local->cont.mkdir.mode, local->umask,
+			   local->xdata_req);
+        return 0;
 }
 
 
 int
-afr_mkdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dict_t *params)
+afr_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+	   mode_t umask, dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
 
-	int ret = -1;
+        priv = this->private;
 
-	int op_ret   = -1;
-	int op_errno = 0;
+        QUORUM_CHECK(mkdir,out);
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
+                goto out;
 
-	priv = this->private;
-
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
 		goto out;
-	}
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+        loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
+	local->parent = inode_ref (loc->parent);
 
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
+        local->cont.mkdir.mode  = mode;
+        local->umask = umask;
 
-	transaction_frame->local = local;
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-	loc_copy (&local->loc, loc);
+	if (!local->xdata_req)
+		goto out;
 
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr) 
-                        % (priv->child_count);
+        local->op = GF_FOP_MKDIR;
+        local->transaction.wind   = afr_mkdir_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_mkdir_unwind;
+
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
+                                    &op_errno);
+        if (ret)
+                goto out;
+
+        local->transaction.main_frame = frame;
+        local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
-        UNLOCK (&priv->read_child_lock);
-
-	local->cont.mkdir.mode  = mode;
-        if (params)
-                local->cont.mkdir.params = dict_ref (params);
-
-        if (loc->parent)
-                local->cont.mkdir.parent_ino = loc->parent->ino;
-
-	local->transaction.fop    = afr_mkdir_wind;
-	local->transaction.done   = afr_mkdir_done;
-	local->transaction.unwind = afr_mkdir_unwind;
-
-	afr_build_parent_loc (&local->transaction.parent_loc, loc);
-
-	local->transaction.main_frame = frame;
-	local->transaction.basename = AFR_BASENAME (loc->path);
 
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
-	op_ret = 0;
+	return 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
 
-		AFR_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL, NULL);
-	}
-
-	return 0;
+	AFR_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+			  NULL);
+        return 0;
 }
 
 /* }}} */
@@ -867,233 +791,127 @@ out:
 int
 afr_link_unwind (call_frame_t *frame, xlator_t *this)
 {
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
+        call_frame_t *main_frame = NULL;
+        afr_local_t  *local = NULL;
 
-        struct iatt *unwind_buf = NULL;
+        local = frame->local;
 
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-                if (local->cont.link.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.link.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.link.buf;
-                }
-
-		unwind_buf->ia_ino = local->cont.link.ino;
-
-                local->cont.link.preparent.ia_ino  = local->cont.link.parent_ino;
-                local->cont.link.postparent.ia_ino = local->cont.link.parent_ino;
-
-		AFR_STACK_UNWIND (link, main_frame,
-                                  local->op_ret, local->op_errno, 
-				  local->cont.link.inode,
-				  unwind_buf, &local->cont.link.preparent,
-                                  &local->cont.link.postparent);
-	}
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
-	return 0;
+	AFR_STACK_UNWIND (link, main_frame, local->op_ret, local->op_errno,
+			  local->inode, &local->cont.dir_fop.buf,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+        return 0;
 }
 
 
 int
-afr_link_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		   int32_t op_ret, int32_t op_errno, inode_t *inode,
+afr_link_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
                    struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count = -1;
-	int child_index = -1;
-
-	local = frame->local;
-	priv = this->private;
-
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (afr_fop_failed (op_ret, op_errno))
-			afr_transaction_fop_failed (frame, this, child_index);
-
-		if (op_ret != -1) {
-			local->op_ret   = op_ret;
-
-			if (local->success_count == 0) {
-				local->cont.link.buf        = *buf;
-
-                                if (priv->read_child >= 0) {
-                                        afr_set_read_child (this, inode,
-                                                            priv->read_child);
-                                } else {
-                                        afr_set_read_child (this, inode,
-                                                            local->read_child_index);
-                                }
-			}
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.link.read_child_buf = *buf;
-                                local->cont.link.preparent      = *preparent;
-                                local->cont.link.postparent     = *postparent;
-                        }
-
-			local->cont.link.inode    = inode;
-
-			local->success_count++;
-		}
-
-		local->op_errno = op_errno;		
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
-
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
+				    preparent, postparent, NULL, NULL, xdata);
 }
 
 
 int
-afr_link_wind (call_frame_t *frame, xlator_t *this)
+afr_link_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv  = this->private;
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
+        local = frame->local;
+        priv  = this->private;
 
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_link_wind_cbk, (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->link,
-					   &local->loc,
-					   &local->newloc);
-			
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+	STACK_WIND_COOKIE (frame, afr_link_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->link,
+			   &local->loc, &local->newloc, local->xdata_req);
+        return 0;
 }
 
 
 int
-afr_link_done (call_frame_t *frame, xlator_t *this)
+afr_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+	  dict_t *xdata)
 {
-	afr_local_t * local = frame->local;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
 
-	local->transaction.unwind (frame, this);
+        priv = this->private;
 
-	AFR_STACK_DESTROY (frame);
+        QUORUM_CHECK(link,out);
 
-	return 0;
-}
-
-
-int
-afr_link (call_frame_t *frame, xlator_t *this,
-	  loc_t *oldloc, loc_t *newloc)
-{
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
+                goto out;
 
-	int ret = -1;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
-	int op_ret   = -1;
-	int op_errno = 0;
+        loc_copy (&local->loc,    oldloc);
+        loc_copy (&local->newloc, newloc);
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+	local->inode = inode_ref (oldloc->inode);
+	local->parent = inode_ref (newloc->parent);
 
-	priv = this->private;
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
+	if (!local->xdata_req)
 		goto out;
-	}
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
 
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
+        local->op = GF_FOP_LINK;
+
+        local->transaction.wind   = afr_link_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_link_unwind;
+
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, newloc,
+                                    &op_errno);
+        if (ret)
+                goto out;
+
+        local->transaction.main_frame   = frame;
+        local->transaction.basename     = AFR_BASENAME (newloc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
 		op_errno = -ret;
 		goto out;
-	}
-
-	transaction_frame->local = local;
-
-	loc_copy (&local->loc,    oldloc);
-	loc_copy (&local->newloc, newloc);
-
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr) 
-                        % (priv->child_count);
         }
-        UNLOCK (&priv->read_child_lock);
-
-	local->cont.link.ino = oldloc->inode->ino;
-
-        if (oldloc->parent)
-                local->cont.link.parent_ino = newloc->parent->ino;
-
-	local->transaction.fop    = afr_link_wind;
-	local->transaction.done   = afr_link_done;
-	local->transaction.unwind = afr_link_unwind;
-
-	afr_build_parent_loc (&local->transaction.parent_loc, oldloc);
-
-	local->transaction.main_frame   = frame;
-	local->transaction.basename     = AFR_BASENAME (oldloc->path);
-	local->transaction.new_basename = AFR_BASENAME (newloc->path);
 
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
-
-	op_ret = 0;
+	return 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (link, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL, NULL);
-	}
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
 
-	return 0;
+	AFR_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+			  NULL);
+        return 0;
 }
 
 /* }}} */
@@ -1104,246 +922,128 @@ out:
 int
 afr_symlink_unwind (call_frame_t *frame, xlator_t *this)
 {
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-        
-        struct iatt *unwind_buf = NULL;
-        
-	local = frame->local;
+        call_frame_t *main_frame = NULL;
+        afr_local_t  *local = NULL;
 
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
+        local = frame->local;
 
-	if (main_frame) {
-                if (local->cont.symlink.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.symlink.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.symlink.buf;
-                }
-
-                unwind_buf->ia_ino = local->cont.symlink.ino;
-
-                local->cont.symlink.preparent.ia_ino  = local->cont.symlink.parent_ino;
-                local->cont.symlink.postparent.ia_ino = local->cont.symlink.parent_ino;
-
-		AFR_STACK_UNWIND (symlink, main_frame,
-                                  local->op_ret, local->op_errno,
-				  local->cont.symlink.inode,
-				  unwind_buf, &local->cont.symlink.preparent,
-                                  &local->cont.symlink.postparent);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
-	return 0;
+	AFR_STACK_UNWIND (symlink, main_frame, local->op_ret, local->op_errno,
+			  local->inode, &local->cont.dir_fop.buf,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+        return 0;
 }
 
 
 int
-afr_symlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		      int32_t op_ret, int32_t op_errno, inode_t *inode,
+afr_symlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, inode_t *inode,
                       struct iatt *buf, struct iatt *preparent,
-                      struct iatt *postparent)
+                      struct iatt *postparent, dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count = -1;
-	int child_index = -1;
-
-	local = frame->local;
-	priv = this->private;
-
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (afr_fop_failed (op_ret, op_errno))
-			afr_transaction_fop_failed (frame, this, child_index);
-		
-		if (op_ret != -1) {
-			local->op_ret   = op_ret;
-
-			if (local->success_count == 0) {
-				local->cont.symlink.buf        = *buf;
-				local->cont.symlink.ino = 
-					afr_itransform (buf->ia_ino, priv->child_count,
-							child_index);
-
-                                if (priv->read_child >= 0) {
-                                        afr_set_read_child (this, inode,
-                                                            priv->read_child);
-                                } else {
-                                        afr_set_read_child (this, inode,
-                                                            local->read_child_index);
-                                }
-			}
-
-                        if (child_index == local->first_up_child) {
-                                local->cont.symlink.ino =
-                                        afr_itransform (buf->ia_ino,
-                                                        priv->child_count,
-                                                        local->first_up_child);
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.symlink.read_child_buf = *buf;
-                                local->cont.symlink.preparent      = *preparent;
-                                local->cont.symlink.postparent     = *postparent;
-                        }
-
-			local->cont.symlink.inode    = inode;
-
-			local->success_count++;
-		}
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
-
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
+				    preparent, postparent, NULL, NULL, xdata);
 }
 
 
 int
-afr_symlink_wind (call_frame_t *frame, xlator_t *this)
+afr_symlink_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_symlink_wind_cbk,
-					   (void *) (long) i,	
-					   priv->children[i], 
-					   priv->children[i]->fops->symlink,
-					   local->cont.symlink.linkpath,
-					   &local->loc,
-                                           local->cont.symlink.params);
-
-			if (!--call_count)
-				break;
-
-		}
-	}
-	
-	return 0;
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+
+        local = frame->local;
+        priv = this->private;
+
+	STACK_WIND_COOKIE (frame, afr_symlink_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->symlink,
+			   local->cont.symlink.linkpath, &local->loc,
+			   local->umask, local->xdata_req);
+        return 0;
 }
 
 
 int
-afr_symlink_done (call_frame_t *frame, xlator_t *this)
+afr_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
+	     loc_t *loc, mode_t umask, dict_t *xdata)
 {
-	afr_local_t * local = frame->local;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
 
-	local->transaction.unwind (frame, this);
+        priv = this->private;
 
-	AFR_STACK_DESTROY (frame);
-	
-	return 0;
-}
+        QUORUM_CHECK(symlink,out);
 
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
+                goto out;
 
-int
-afr_symlink (call_frame_t *frame, xlator_t *this,
-	     const char *linkpath, loc_t *loc, dict_t *params)
-{
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
-
-	int ret = -1;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
-	int op_ret   = -1;
-	int op_errno = 0;
+        loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
+	local->parent = inode_ref (loc->parent);
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        local->cont.symlink.linkpath = gf_strdup (linkpath);
+        local->umask = umask;
 
-	priv = this->private;
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
+	if (!local->xdata_req)
 		goto out;
-	}
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
 
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
+        local->op = GF_FOP_SYMLINK;
+        local->transaction.wind   = afr_symlink_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_symlink_unwind;
+
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
+                                    &op_errno);
+        if (ret)
+                goto out;
+
+        local->transaction.main_frame   = frame;
+        local->transaction.basename     = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
 		op_errno = -ret;
 		goto out;
-	}
-
-	transaction_frame->local = local;
-	
-	loc_copy (&local->loc, loc);
-
-        LOCK (&priv->read_child_lock);
-        {
-                local->read_child_index = (++priv->read_child_rr) 
-                        % (priv->child_count);
         }
-        UNLOCK (&priv->read_child_lock);
-
-	local->cont.symlink.linkpath = gf_strdup (linkpath);
-        if (params)
-                local->cont.symlink.params = dict_ref (params);
-
-        if (loc->parent)
-                local->cont.symlink.parent_ino = loc->parent->ino;
-
-	local->transaction.fop    = afr_symlink_wind;
-	local->transaction.done   = afr_symlink_done;
-	local->transaction.unwind = afr_symlink_unwind;
-
-	afr_build_parent_loc (&local->transaction.parent_loc, loc);
-
-	local->transaction.main_frame   = frame;
-	local->transaction.basename     = AFR_BASENAME (loc->path);
-
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
 
-	op_ret = 0;
+	return 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (symlink, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL, NULL);
-	}
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
 
-	return 0;
+	AFR_STACK_UNWIND (symlink, frame, -1, op_errno, NULL, NULL, NULL,
+			  NULL, NULL);
+        return 0;
 }
 
 /* }}} */
@@ -1353,228 +1053,159 @@ out:
 int
 afr_rename_unwind (call_frame_t *frame, xlator_t *this)
 {
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
+        call_frame_t *main_frame = NULL;
+        afr_local_t  *local = NULL;
 
-        struct iatt *unwind_buf = NULL;
-
-	local = frame->local;
+        local = frame->local;
 
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-                if (local->cont.rename.read_child_buf.ia_ino) {
-                        unwind_buf = &local->cont.rename.read_child_buf;
-                } else {
-                        unwind_buf = &local->cont.rename.buf;
-                }
-
-		unwind_buf->ia_ino = local->cont.rename.ino;
-
-                local->cont.rename.preoldparent.ia_ino  = local->cont.rename.oldparent_ino;
-                local->cont.rename.postoldparent.ia_ino = local->cont.rename.oldparent_ino;
-                local->cont.rename.prenewparent.ia_ino  = local->cont.rename.newparent_ino;
-                local->cont.rename.postnewparent.ia_ino = local->cont.rename.newparent_ino;
-
-		AFR_STACK_UNWIND (rename, main_frame,
-                                  local->op_ret, local->op_errno, 
-				  unwind_buf,
-                                  &local->cont.rename.preoldparent,
-                                  &local->cont.rename.postoldparent,
-                                  &local->cont.rename.prenewparent,
-                                  &local->cont.rename.postnewparent);
-	}
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
-	return 0;
+	AFR_STACK_UNWIND (rename, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.dir_fop.buf,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent,
+			  &local->cont.dir_fop.prenewparent,
+			  &local->cont.dir_fop.postnewparent, local->xdata_rsp);
+        return 0;
 }
 
 
 int
-afr_rename_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		     int32_t op_ret, int32_t op_errno, struct iatt *buf,
+afr_rename_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *buf,
                      struct iatt *preoldparent, struct iatt *postoldparent,
-                     struct iatt *prenewparent, struct iatt *postnewparent)
+                     struct iatt *prenewparent, struct iatt *postnewparent,
+                     dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-
-	int call_count = -1;
-	int child_index = -1;
-
-	local = frame->local;
-
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (afr_fop_failed (op_ret, op_errno) && op_errno != ENOTEMPTY)
-			afr_transaction_fop_failed (frame, this, child_index);
-
-		if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-		
-                                if (buf) {
-                                        local->cont.rename.buf = *buf;
-                                }
-
-                                local->success_count++;
-                        }
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.rename.read_child_buf = *buf;
-
-                                local->cont.rename.preoldparent  = *preoldparent;
-                                local->cont.rename.postoldparent = *postoldparent;
-                                local->cont.rename.prenewparent  = *prenewparent;
-                                local->cont.rename.postnewparent = *postnewparent;
-                        }
-                }
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, buf,
+				    preoldparent, postoldparent, prenewparent,
+				    postnewparent, xdata);
 }
 
 
-int32_t
-afr_rename_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_rename_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
 	local = frame->local;
 	priv = this->private;
 
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_rename_wind_cbk, 
-					   (void *) (long) i,	
-					   priv->children[i], 
-					   priv->children[i]->fops->rename,
-					   &local->loc,
-					   &local->newloc);
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+	STACK_WIND_COOKIE (frame, afr_rename_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->rename,
+			   &local->loc, &local->newloc, local->xdata_req);
+        return 0;
 }
 
 
 int
-afr_rename_done (call_frame_t *frame, xlator_t *this)
+afr_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+	    dict_t *xdata)
 {
-	afr_local_t * local = frame->local;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
+        int                     nlockee                 = 0;
 
-	local->transaction.unwind (frame, this);
-
-	AFR_STACK_DESTROY (frame);
-	
-	return 0;
-}
+        priv = this->private;
 
+        QUORUM_CHECK(rename,out);
 
-int
-afr_rename (call_frame_t *frame, xlator_t *this,
-	    loc_t *oldloc, loc_t *newloc)
-{
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
+                op_errno = ENOMEM;
 
-	int ret = -1;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
-	int op_ret   = -1;
-	int op_errno = 0;
+        loc_copy (&local->loc,    oldloc);
+        loc_copy (&local->newloc, newloc);
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+	local->inode = inode_ref (oldloc->inode);
+	local->parent = inode_ref (oldloc->parent);
+	local->parent2 = inode_ref (newloc->parent);
 
-	priv = this->private;
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
+	if (!local->xdata_req)
 		goto out;
-	}
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+        local->op = GF_FOP_RENAME;
+        local->transaction.wind   = afr_rename_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_rename_unwind;
+
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, oldloc,
+                                    &op_errno);
+        if (ret)
+                goto out;
+        ret = afr_build_parent_loc (&local->transaction.new_parent_loc, newloc,
+                                    &op_errno);
+        if (ret)
+                goto out;
+
+        local->transaction.main_frame   = frame;
+        local->transaction.basename     = AFR_BASENAME (oldloc->path);
+        local->transaction.new_basename = AFR_BASENAME (newloc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = nlockee = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->transaction.new_parent_loc,
+                                     local->transaction.new_basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        nlockee++;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        nlockee++;
+        if (local->newloc.inode && IA_ISDIR (local->newloc.inode->ia_type)) {
+                ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                             &local->newloc,
+                                             NULL,
+                                             priv->child_count);
+                if (ret)
+                        goto out;
+
+                nlockee++;
+        }
+        qsort (int_lock->lockee, nlockee, sizeof (*int_lock->lockee),
+               afr_entry_lockee_cmp);
+        int_lock->lockee_count = nlockee;
 
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_RENAME_TRANSACTION);
+        if (ret < 0) {
 		op_errno = -ret;
 		goto out;
-	}
-
-	transaction_frame->local = local;
-
-	loc_copy (&local->loc,    oldloc);
-	loc_copy (&local->newloc, newloc);
-
-        local->read_child_index = afr_read_child (this, oldloc->inode);
-
-	local->cont.rename.ino = oldloc->inode->ino;
-
-        if (oldloc->parent)
-                local->cont.rename.oldparent_ino = oldloc->parent->ino;
-        if (newloc->parent)
-                local->cont.rename.newparent_ino = newloc->parent->ino;
-
-	local->transaction.fop    = afr_rename_wind;
-	local->transaction.done   = afr_rename_done;
-	local->transaction.unwind = afr_rename_unwind;
-
-	afr_build_parent_loc (&local->transaction.parent_loc, oldloc);
-	afr_build_parent_loc (&local->transaction.new_parent_loc, newloc);
-
-	local->transaction.main_frame   = frame;
-	local->transaction.basename     = AFR_BASENAME (oldloc->path);
-	local->transaction.new_basename = AFR_BASENAME (newloc->path);
-
-	afr_transaction (transaction_frame, this, AFR_ENTRY_RENAME_TRANSACTION);
+        }
 
-	op_ret = 0;
+	return 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
 
-		AFR_STACK_UNWIND (rename, frame, op_ret, op_errno,
-                                  NULL, NULL, NULL, NULL, NULL);
-	}
-
-	return 0;
+	AFR_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+			  NULL, NULL);
+        return 0;
 }
 
 /* }}} */
@@ -1584,201 +1215,123 @@ out:
 int
 afr_unlink_unwind (call_frame_t *frame, xlator_t *this)
 {
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
+        call_frame_t *main_frame = NULL;
+        afr_local_t  *local = NULL;
 
-	if (main_frame) {
-                local->cont.unlink.preparent.ia_ino  = local->cont.unlink.parent_ino;
-                local->cont.unlink.postparent.ia_ino = local->cont.unlink.parent_ino;
+        local = frame->local;
 
-		AFR_STACK_UNWIND (unlink, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  &local->cont.unlink.preparent,
-                                  &local->cont.unlink.postparent);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
-	return 0;
+	AFR_STACK_UNWIND (unlink, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+        return 0;
 }
 
 
 int
-afr_unlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                     struct iatt *postparent)
+afr_unlink_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count  = -1;
-	int child_index = (long) cookie;
-
-	local = frame->local;
-	priv  = this->private;
-
-	LOCK (&frame->lock);
-	{
-                if (child_index == local->read_child_index) {
-                        local->read_child_returned = _gf_true;
-                }
-
-		if (afr_fop_failed (op_ret, op_errno))
-			afr_transaction_fop_failed (frame, this, child_index);
-
-		if (op_ret != -1) {
-			if (local->success_count == 0) {
-				local->op_ret   = op_ret;
-                                local->cont.unlink.preparent  = *preparent;
-                                local->cont.unlink.postparent = *postparent;
-			}
-
-                        if (child_index == local->read_child_index) {
-                                local->cont.unlink.preparent  = *preparent;
-                                local->cont.unlink.postparent = *postparent;
-                        }
-
-			local->success_count++;
-		}
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
-
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, NULL,
+				    preparent, postparent, NULL, NULL, xdata);
 }
 
 
-int32_t
-afr_unlink_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_unlink_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv  = this->private;
+        local = frame->local;
+        priv  = this->private;
 
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_unlink_wind_cbk,	
-					   (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->unlink,
-					   &local->loc);
-			
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+	STACK_WIND_COOKIE (frame, afr_unlink_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->unlink,
+			   &local->loc, local->xflag, local->xdata_req);
+        return 0;
 }
 
 
-int32_t
-afr_unlink_done (call_frame_t *frame, xlator_t *this)
+int
+afr_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+	    dict_t *xdata)
 {
-	afr_local_t * local = frame->local;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
 
-	local->transaction.unwind (frame, this);
+        priv = this->private;
 
-	AFR_STACK_DESTROY (frame);
-	
-	return 0;
-}
+        QUORUM_CHECK(unlink,out);
 
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
+                goto out;
 
-int32_t
-afr_unlink (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc)
-{
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
-
-	int ret = -1;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
-	int op_ret   = -1;
-	int op_errno = 0;
+        loc_copy (&local->loc, loc);
+        local->xflag = xflag;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+	local->inode = inode_ref (loc->inode);
+	local->parent = inode_ref (loc->parent);
 
-	priv = this->private;
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
+	if (!local->xdata_req)
 		goto out;
-	}
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
+        local->op = GF_FOP_UNLINK;
+        local->transaction.wind   = afr_unlink_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_unlink_unwind;
+
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
+                                    &op_errno);
+        if (ret)
+                goto out;
+
+        local->transaction.main_frame = frame;
+        local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[0], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        int_lock->lockee_count++;
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
 		op_errno = -ret;
 		goto out;
-	}
-
-	transaction_frame->local = local;
-
-	loc_copy (&local->loc, loc);
-
-        if (loc->parent)
-                local->cont.unlink.parent_ino = loc->parent->ino;
-
-	local->transaction.fop    = afr_unlink_wind;
-	local->transaction.done   = afr_unlink_done;
-	local->transaction.unwind = afr_unlink_unwind;
-
-	afr_build_parent_loc (&local->transaction.parent_loc, loc);
-
-	local->transaction.main_frame = frame;
-	local->transaction.basename = AFR_BASENAME (loc->path);
-
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        }
 
-	op_ret = 0;
+	return 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (unlink, frame, op_ret, op_errno,
-                                  NULL, NULL);
-	}
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
 
-	return 0;
+	AFR_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
 }
 
 /* }}} */
@@ -1790,204 +1343,137 @@ out:
 int
 afr_rmdir_unwind (call_frame_t *frame, xlator_t *this)
 {
-	call_frame_t *main_frame = NULL;
-	afr_local_t  *local = NULL;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame) {
-			main_frame = local->transaction.main_frame;
-		}
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-                local->cont.rmdir.preparent.ia_ino  = local->cont.rmdir.parent_ino;
-                local->cont.rmdir.postparent.ia_ino = local->cont.rmdir.parent_ino;
-
-		AFR_STACK_UNWIND (rmdir, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  &local->cont.rmdir.preparent,
-                                  &local->cont.rmdir.postparent);
-        }
-
-	return 0;
-}
-
-
-int
-afr_rmdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-		    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                    struct iatt *postparent)
-{
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count  = -1;
-	int child_index = (long) cookie;
-        int read_child  = 0;
-
-	local = frame->local;
-	priv = this->private;
+        call_frame_t *main_frame = NULL;
+        afr_local_t  *local = NULL;
 
-	LOCK (&frame->lock);
-	{
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
+        local = frame->local;
 
-		if (afr_fop_failed (op_ret, op_errno) && (op_errno != ENOTEMPTY))
-			afr_transaction_fop_failed (frame, this, child_index);
-
-		if (op_ret != -1) {
-			if (local->success_count == 0) {
-				local->op_ret = op_ret;
-                                local->cont.rmdir.preparent  = *preparent;
-                                local->cont.rmdir.postparent = *postparent;
-
-			}
-
-                        if (child_index == read_child) {
-                                local->cont.rmdir.preparent  = *preparent;
-                                local->cont.rmdir.postparent = *postparent;
-                        }
-
-			local->success_count++;
-		}
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
-	if (call_count == 0) {
-		local->transaction.unwind (frame, this);
-		local->transaction.resume (frame, this);
-	}
-	
-	return 0;
+	AFR_STACK_UNWIND (rmdir, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.dir_fop.preparent,
+			  &local->cont.dir_fop.postparent, local->xdata_rsp);
+        return 0;
 }
 
 
 int
-afr_rmdir_wind (call_frame_t *frame, xlator_t *this)
+afr_rmdir_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
 {
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv  = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {				
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_rmdir_wind_cbk,	
-					   (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->rmdir,
-					   &local->loc, local->cont.rmdir.flags);
-
-			if (!--call_count)
-				break;
-		}
-	}
-	
-	return 0;
+        return __afr_dir_write_cbk (frame, cookie, this, op_ret, op_errno, NULL,
+				    preparent, postparent, NULL, NULL, xdata);
 }
 
 
 int
-afr_rmdir_done (call_frame_t *frame, xlator_t *this)
+afr_rmdir_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
-	afr_local_t * local = frame->local;
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
-	local->transaction.unwind (frame, this);
+        local = frame->local;
+        priv  = this->private;
 
-	AFR_STACK_DESTROY (frame);
-	
-	return 0;
+	STACK_WIND_COOKIE (frame, afr_rmdir_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->rmdir,
+			   &local->loc, local->cont.rmdir.flags, local->xdata_req);
+        return 0;
 }
 
 
 int
-afr_rmdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, int flags)
+afr_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+	   dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t  * transaction_frame = NULL;
-	
-	int ret = -1;
+        afr_private_t           *priv                   = NULL;
+        afr_local_t             *local                  = NULL;
+        afr_internal_lock_t     *int_lock               = NULL;
+        call_frame_t            *transaction_frame      = NULL;
+        int                     ret                     = -1;
+        int                     op_errno                = ENOMEM;
+        int                     nlockee                 = 0;
 
-	int op_ret   = -1;
-	int op_errno = 0;
+        priv = this->private;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        QUORUM_CHECK(rmdir,out);
 
-	priv = this->private;
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
+                goto out;
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
 		goto out;
-	}
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
 
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	transaction_frame->local = local;
+        loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
+	local->parent = inode_ref (loc->parent);
 
         local->cont.rmdir.flags = flags;
-	loc_copy (&local->loc, loc);
-
-        if (loc->parent)
-                local->cont.rmdir.parent_ino = loc->parent->ino;
 
-	local->transaction.fop    = afr_rmdir_wind;
-	local->transaction.done   = afr_rmdir_done;
-	local->transaction.unwind = afr_rmdir_unwind;
+        if (xdata)
+                local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-	afr_build_parent_loc (&local->transaction.parent_loc, loc);
-
-	local->transaction.main_frame = frame;
-	local->transaction.basename = AFR_BASENAME (loc->path);
+	if (!local->xdata_req)
+		goto out;
 
-	afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        local->op = GF_FOP_RMDIR;
+        local->transaction.wind   = afr_rmdir_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_rmdir_unwind;
+
+        ret = afr_build_parent_loc (&local->transaction.parent_loc, loc,
+                                    &op_errno);
+        if (ret)
+                goto out;
+
+        local->transaction.main_frame = frame;
+        local->transaction.basename = AFR_BASENAME (loc->path);
+        int_lock = &local->internal_lock;
+
+        int_lock->lockee_count = nlockee = 0;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->transaction.parent_loc,
+                                     local->transaction.basename,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        nlockee++;
+        ret = afr_init_entry_lockee (&int_lock->lockee[nlockee], local,
+                                     &local->loc,
+                                     NULL,
+                                     priv->child_count);
+        if (ret)
+                goto out;
+
+        nlockee++;
+        qsort (int_lock->lockee, nlockee, sizeof (*int_lock->lockee),
+               afr_entry_lockee_cmp);
+        int_lock->lockee_count = nlockee;
+
+        ret = afr_transaction (transaction_frame, this, AFR_ENTRY_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
+        }
 
-	op_ret = 0;
+	return 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
-                                  NULL, NULL);
-	}
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
 
-	return 0;
+	AFR_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
 }
 
 /* }}} */
-
diff --git a/xlators/cluster/afr/src/afr-dir-write.h b/xlators/cluster/afr/src/afr-dir-write.h
index 4f180857e..02f0a3682 100644
--- a/xlators/cluster/afr/src/afr-dir-write.h
+++ b/xlators/cluster/afr/src/afr-dir-write.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __DIR_WRITE_H__
@@ -23,38 +14,34 @@
 int32_t
 afr_create (call_frame_t *frame, xlator_t *this,
             loc_t *loc, int32_t flags, mode_t mode,
-            fd_t *fd, dict_t *params);
+            mode_t umask, fd_t *fd, dict_t *xdata);
 
 int32_t
 afr_mknod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dev_t dev, dict_t *params);
+	   loc_t *loc, mode_t mode, dev_t dev, mode_t umask, dict_t *xdata);
 
 int32_t
 afr_mkdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dict_t *params);
+	   loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
 
 int32_t
 afr_unlink (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc);
+	    loc_t *loc, int xflag, dict_t *xdata);
 
 int32_t
 afr_rmdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, int flags);
+	   loc_t *loc, int flags, dict_t *xdata);
 
 int32_t
 afr_link (call_frame_t *frame, xlator_t *this,
-	  loc_t *oldloc, loc_t *newloc);
+	  loc_t *oldloc, loc_t *newloc, dict_t *xdata);
 
 int32_t
 afr_rename (call_frame_t *frame, xlator_t *this,
-	    loc_t *oldloc, loc_t *newloc);
+	    loc_t *oldloc, loc_t *newloc, dict_t *xdata);
 
 int
 afr_symlink (call_frame_t *frame, xlator_t *this,
-	     const char *linkpath, loc_t *oldloc, dict_t *params);
-
-int32_t
-afr_setdents (call_frame_t *frame, xlator_t *this,
-	      fd_t *fd, int32_t flags, dir_entry_t *entries, int32_t count);
+	     const char *linkpath, loc_t *oldloc, mode_t umask, dict_t *params);
 
 #endif /* __DIR_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-inode-read.c b/xlators/cluster/afr/src/afr-inode-read.c
index 68ac78731..4cb219246 100644
--- a/xlators/cluster/afr/src/afr-inode-read.c
+++ b/xlators/cluster/afr/src/afr-inode-read.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -44,843 +35,1630 @@
 #include "compat-errno.h"
 #include "compat.h"
 
-#include "afr.h"
+#include "afr-transaction.h"
 
 
-/**
- * Common algorithm for inode read calls:
- *
- * - Try the fop on the first child that is up
- * - if we have failed due to ENOTCONN:
- *     try the next child
- *
- * Applicable to: access, stat, fstat, readlink, getxattr
- */
-
 /* {{{ access */
 
-int32_t
-afr_access_cbk (call_frame_t *frame, void *cookie,
-		xlator_t *this, int32_t op_ret, int32_t op_errno)
+int
+afr_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		int op_ret, int op_errno, dict_t *xdata)
 {
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
+        afr_local_t     *local = NULL;
 
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try = -1;
-        int read_child = -1;
+        local = frame->local;
 
-	priv     = this->private;
-	children = priv->children;
+	if (op_ret < 0) {
+		local->op_ret = op_ret;
+		local->op_errno = op_errno;
 
-	local = frame->local;
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-        read_child = (long) cookie;
+	AFR_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
 
-	if (op_ret == -1) {
-        retry:
-		last_tried = local->cont.access.last_tried;
+        return 0;
+}
 
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
-		this_try    = ++local->cont.access.last_tried;
 
-                if (this_try == read_child) {
-                        goto retry;
-                }
+int
+afr_access_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
 
-		unwind = 0;
+	priv = this->private;
+	local = frame->local;
 
-		STACK_WIND_COOKIE (frame, afr_access_cbk,
-				   (void *) (long) read_child,
-				   children[this_try],
-				   children[this_try]->fops->access,
-				   &local->loc, local->cont.access.mask);
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (access, frame, local->op_ret,
+				  local->op_errno, 0);
+		return 0;
 	}
 
-out:
-	if (unwind) {
-		AFR_STACK_UNWIND (access, frame, op_ret, op_errno);
-	}
+        STACK_WIND_COOKIE (frame, afr_access_cbk, (void *) (long) subvol,
+                           priv->children[subvol],
+                           priv->children[subvol]->fops->access,
+                           &local->loc, local->cont.access.mask,
+			   local->xdata_req);
+	return 0;
+}
+
+int
+afr_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
+	    int mask, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+	int op_errno = 0;
+
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
+
+	local->op = GF_FOP_ACCESS;
+	loc_copy (&local->loc, loc);
+	local->cont.access.mask = mask;
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
+
+	afr_read_txn (frame, this, loc->inode, afr_access_wind,
+		      AFR_METADATA_TRANSACTION);
 
 	return 0;
+out:
+	AFR_STACK_UNWIND (access, frame, -1, op_errno, NULL);
+
+        return 0;
 }
 
+/* }}} */
 
-int32_t
-afr_access (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t mask)
+/* {{{ stat */
+
+int
+afr_stat_cbk (call_frame_t *frame, void *cookie,
+              xlator_t *this, int32_t op_ret, int32_t op_errno,
+              struct iatt *buf, dict_t *xdata)
 {
-	afr_private_t * priv       = NULL;
-	xlator_t **     children   = NULL;
-	int             call_child = 0;
-	afr_local_t     *local     = NULL;
+        afr_local_t     *local = NULL;
 
-        int32_t         read_child = -1;
+        local = frame->local;
 
+	if (op_ret < 0) {
+		local->op_ret = op_ret;
+		local->op_errno = op_errno;
 
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+	AFR_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
 
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
+        return 0;
+}
 
-	children = priv->children;
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+int
+afr_stat_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
 
-        read_child = afr_read_child (this, loc->inode);
+	priv = this->private;
+	local = frame->local;
 
-        if ((read_child >= 0) && (priv->child_up[read_child])) {
-                call_child = read_child;
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (stat, frame, local->op_ret, local->op_errno,
+				  0, 0);
+		return 0;
+	}
 
-                local->cont.access.last_tried = -1;
+        STACK_WIND_COOKIE (frame, afr_stat_cbk, (void *) (long) subvol,
+                           priv->children[subvol],
+                           priv->children[subvol]->fops->stat,
+                           &local->loc, local->xdata_req);
+	return 0;
+}
 
-        } else {
-                call_child = afr_first_up_child (priv);
-                if (call_child == -1) {
-                        op_errno = ENOTCONN;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "no child is up");
-                        goto out;
-                }
+int
+afr_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
+	int op_errno = 0;
 
-                local->cont.access.last_tried = call_child;
-        }
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
+	local->op = GF_FOP_STAT;
 	loc_copy (&local->loc, loc);
-	local->cont.access.mask       = mask;
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
 
-	STACK_WIND_COOKIE (frame, afr_access_cbk,
-			   (void *) (long) call_child,
-			   children[call_child], children[call_child]->fops->access,
-			   loc, mask);
+	afr_read_txn (frame, this, loc->inode, afr_stat_wind,
+		      AFR_DATA_TRANSACTION);
 
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (access, frame, op_ret, op_errno);
-	}
 	return 0;
+out:
+	AFR_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+
+        return 0;
 }
 
 
 /* }}} */
 
-/* {{{ stat */
+/* {{{ fstat */
 
-int32_t
-afr_stat_cbk (call_frame_t *frame, void *cookie,
-	      xlator_t *this, int32_t op_ret, int32_t op_errno,
-	      struct iatt *buf)
+int
+afr_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *buf,
+               dict_t *xdata)
 {
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
-
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try = -1;
-	int read_child = -1;
+        afr_local_t     *local = NULL;
 
-	priv     = this->private;
-	children = priv->children;
+        local = frame->local;
 
-	read_child = (long) cookie;
-
-	local = frame->local;
+	if (op_ret < 0) {
+		local->op_ret = op_ret;
+		local->op_errno = op_errno;
 
-	if (op_ret == -1) {
-	retry:
-		last_tried = local->cont.stat.last_tried;
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
-		this_try = ++local->cont.stat.last_tried;
+	AFR_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
 
-		if (this_try == read_child) {
-			goto retry;
-		}
+        return 0;
+}
 
-		unwind = 0;
 
-		STACK_WIND_COOKIE (frame, afr_stat_cbk,
-				   (void *) (long) read_child,
-				   children[this_try],
-				   children[this_try]->fops->stat,
-				   &local->loc);
-	}
+int
+afr_fstat_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
 
-out:
-	if (unwind) {
-		if (buf)
-			buf->ia_ino = local->cont.stat.ino;
+	priv = this->private;
+	local = frame->local;
 
-		AFR_STACK_UNWIND (stat, frame, op_ret, op_errno, buf);
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (fstat, frame, local->op_ret, local->op_errno,
+				  0, 0);
+		return 0;
 	}
 
+        STACK_WIND_COOKIE (frame, afr_fstat_cbk, (void *) (long) subvol,
+                           priv->children[subvol],
+                           priv->children[subvol]->fops->fstat,
+                           local->fd, local->xdata_req);
 	return 0;
 }
 
 
 int32_t
-afr_stat (call_frame_t *frame, xlator_t *this,
-	  loc_t *loc)
+afr_fstat (call_frame_t *frame, xlator_t *this,
+           fd_t *fd, dict_t *xdata)
 {
-	afr_private_t * priv       = NULL;
-	afr_local_t   * local      = NULL;
-	xlator_t **     children   = NULL;
+        afr_local_t *local = NULL;
+	int op_errno = 0;
 
-        int32_t         read_child = -1;
-	int             call_child = 0;
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-	int32_t         op_ret     = -1;
-	int32_t         op_errno   = 0;
+	local->op = GF_FOP_FSTAT;
+	local->fd = fd_ref (fd);
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        afr_fix_open (fd, this);
 
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
+	afr_read_txn (frame, this, fd->inode, afr_fstat_wind,
+		      AFR_DATA_TRANSACTION);
 
-	children = priv->children;
+	return 0;
+out:
+	AFR_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+        return 0;
+}
 
-	frame->local = local;
+/* }}} */
 
-        read_child = afr_read_child (this, loc->inode);
+/* {{{ readlink */
 
-        if ((read_child >= 0) && (priv->child_up[read_child])) {
-                call_child = read_child;
+int
+afr_readlink_cbk (call_frame_t *frame, void *cookie,
+                  xlator_t *this, int32_t op_ret, int32_t op_errno,
+                  const char *buf, struct iatt *sbuf, dict_t *xdata)
+{
+	afr_local_t *local = NULL;
 
-                local->cont.stat.last_tried = -1;
+	local = frame->local;
 
-        } else {
-		call_child = afr_first_up_child (priv);
-		if (call_child == -1) {
-			op_errno = ENOTCONN;
-			gf_log (this->name, GF_LOG_DEBUG,
-				"no child is up");
-			goto out;
-		}
+	if (op_ret < 0) {
+		local->op_ret = -1;
+		local->op_errno = op_errno;
 
-		local->cont.stat.last_tried = call_child;
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
 	}
 
-	loc_copy (&local->loc, loc);
+	AFR_STACK_UNWIND (readlink, frame, op_ret, op_errno,
+			  buf, sbuf, xdata);
+	return 0;
+}
 
-	local->cont.stat.ino = loc->inode->ino;
+int
+afr_readlink_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
 
-	STACK_WIND_COOKIE (frame, afr_stat_cbk, (void *) (long) call_child,
-			   children[call_child],
-			   children[call_child]->fops->stat,
-			   loc);
+	local = frame->local;
+	priv = this->private;
 
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (stat, frame, op_ret, op_errno, NULL);
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (readlink, frame, local->op_ret,
+				  local->op_errno, 0, 0, 0);
+		return 0;
 	}
 
+	STACK_WIND_COOKIE (frame, afr_readlink_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->readlink,
+			   &local->loc, local->cont.readlink.size,
+			   local->xdata_req);
 	return 0;
 }
 
 
-/* }}} */
+int
+afr_readlink (call_frame_t *frame, xlator_t *this,
+              loc_t *loc, size_t size, dict_t *xdata)
+{
+        afr_local_t   * local      = NULL;
+        int32_t         op_errno   = 0;
 
-/* {{{ fstat */
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-int32_t
-afr_fstat_cbk (call_frame_t *frame, void *cookie,
-	       xlator_t *this, int32_t op_ret, int32_t op_errno,
-	       struct iatt *buf)
-{
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
+	local->op = GF_FOP_READLINK;
+        loc_copy (&local->loc, loc);
+        local->cont.readlink.size = size;
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
 
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try = -1;
-        int read_child = -1;
+	afr_read_txn (frame, this, loc->inode, afr_readlink_wind,
+		      AFR_DATA_TRANSACTION);
 
-	priv     = this->private;
-	children = priv->children;
+        return 0;
+out:
+	AFR_STACK_UNWIND(readlink, frame, -1, op_errno, 0, 0, 0);
 
-	local = frame->local;
+	return 0;
+}
 
-	read_child = (long) cookie;
 
-	if (op_ret == -1) {
-	retry:
-		last_tried = local->cont.fstat.last_tried;
+/* }}} */
 
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
-		this_try   = ++local->cont.fstat.last_tried;
+/* {{{ getxattr */
 
-		if (this_try == read_child) {
-			goto retry;
-		}
+struct _xattr_key {
+        char *key;
+        struct list_head list;
+};
 
-		unwind = 0;
 
-		STACK_WIND_COOKIE (frame, afr_fstat_cbk,
-				   (void *) (long) read_child,
-				   children[this_try],
-				   children[this_try]->fops->fstat,
-				   local->fd);
-	}
+int
+__gather_xattr_keys (dict_t *dict, char *key, data_t *value,
+                     void *data)
+{
+        struct list_head *  list  = data;
+        struct _xattr_key * xkey  = NULL;
 
-out:
-	if (unwind) {
-		if (buf)
-			buf->ia_ino = local->cont.fstat.ino;
+        if (!strncmp (key, AFR_XATTR_PREFIX,
+                      strlen (AFR_XATTR_PREFIX))) {
 
-		AFR_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf);
-	}
+                xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
+                if (!xkey)
+                        return -1;
 
-	return 0;
+                xkey->key = key;
+                INIT_LIST_HEAD (&xkey->list);
+
+                list_add_tail (&xkey->list, list);
+        }
+        return 0;
 }
 
 
-int32_t
-afr_fstat (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd)
+void
+afr_filter_xattrs (dict_t *dict)
 {
-	afr_private_t * priv       = NULL;
-	afr_local_t   * local      = NULL;
-	xlator_t **     children   = NULL;
+        struct list_head   keys = {0,};
+        struct _xattr_key *key  = NULL;
+        struct _xattr_key *tmp  = NULL;
 
-	int             call_child = 0;
-        int32_t         read_child = -1;
+        INIT_LIST_HEAD (&keys);
 
-	int32_t         op_ret     = -1;
-	int32_t         op_errno   = 0;
+        dict_foreach (dict, __gather_xattr_keys,
+                      (void *) &keys);
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (fd, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        list_for_each_entry_safe (key, tmp, &keys, list) {
+                dict_del (dict, key->key);
 
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
+                list_del_init (&key->list);
 
-	children = priv->children;
+                GF_FREE (key);
+        }
+}
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
 
-	frame->local = local;
+int
+afr_getxattr_cbk (call_frame_t *frame, void *cookie,
+                  xlator_t *this, int32_t op_ret, int32_t op_errno,
+                  dict_t *dict, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
 
-	VALIDATE_OR_GOTO (fd->inode, out);
+        local = frame->local;
 
-        read_child = afr_read_child (this, fd->inode);
+	if (op_ret < 0) {
+		local->op_ret = op_ret;
+		local->op_errno = op_errno;
 
-        if ((read_child >= 0) && (priv->child_up[read_child])) {
-                call_child = read_child;
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-                local->cont.fstat.last_tried = -1;
-        } else {
-                call_child = afr_first_up_child (priv);
+	if (dict)
+		afr_filter_xattrs (dict);
 
-                if (call_child == -1) {
-                        op_errno = ENOTCONN;
-                        gf_log (this->name, GF_LOG_DEBUG,
-				"no child is up");
-			goto out;
-		}
+	AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
 
-                local->cont.fstat.last_tried = call_child;
-        }
+        return 0;
+}
 
-	local->cont.fstat.ino = fd->inode->ino;
-	local->fd = fd_ref (fd);
 
-	STACK_WIND_COOKIE (frame, afr_fstat_cbk, (void *) (long) call_child,
-			   children[call_child],
-			   children[call_child]->fops->fstat,
-			   fd);
+int
+afr_getxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
 
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (fstat, frame, op_ret, op_errno, NULL);
+	local = frame->local;
+	priv = this->private;
+
+	if (subvol == -1) {
+                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
+				  local->op_errno, NULL, NULL);
+		return 0;
 	}
 
+	STACK_WIND_COOKIE (frame, afr_getxattr_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->getxattr,
+			   &local->loc, local->cont.getxattr.name,
+			   local->xdata_req);
 	return 0;
 }
 
-/* }}} */
 
-/* {{{ readlink */
+int32_t
+afr_getxattr_unwind (call_frame_t *frame, int op_ret, int op_errno,
+                     dict_t *dict, dict_t *xdata)
+
+{
+        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
+}
 
 int32_t
-afr_readlink_cbk (call_frame_t *frame, void *cookie,
-		  xlator_t *this, int32_t op_ret, int32_t op_errno,
-		  const char *buf, struct iatt *sbuf)
+afr_fgetxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
+                         xlator_t *this, int32_t op_ret, int32_t op_errno,
+                         dict_t *dict, dict_t *xdata)
 {
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
+        afr_local_t     *local                  = NULL;
+        afr_private_t   *priv                   = NULL;
+        xlator_t        **children              = NULL;
+        dict_t          *xattr                  = NULL;
+        char            *tmp_report             = NULL;
+        char            lk_summary[1024]        = {0,};
+        int             serz_len                = 0;
+        int32_t         callcnt                 = 0;
+        long int        cky                     = 0;
+        int             ret                     = 0;
+
+        priv     = this->private;
+        children = priv->children;
+
+        local = frame->local;
+        cky = (long) cookie;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+                if (op_ret == -1)
+                        local->replies[cky].op_errno = op_errno;
+
+                if (!local->dict)
+                        local->dict = dict_new ();
+                if (local->dict) {
+                        ret = dict_get_str (dict, local->cont.getxattr.name,
+                                            &tmp_report);
+                        if (ret)
+                                goto unlock;
+                        ret = dict_set_dynstr (local->dict,
+                                               children[cky]->name,
+                                               gf_strdup (tmp_report));
+                        if (ret)
+                                goto unlock;
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                xattr = dict_new ();
+                if (!xattr) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        goto unwind;
+                }
+                ret = dict_serialize_value_with_delim (local->dict,
+                                                       lk_summary,
+                                                       &serz_len, '\n');
+                if (ret) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error serializing dictionary");
+                        goto unwind;
+                }
+                if (serz_len == -1)
+                        snprintf (lk_summary, sizeof (lk_summary),
+                                  "No locks cleared.");
+                ret = dict_set_dynstr (xattr, local->cont.getxattr.name,
+                                       gf_strdup (lk_summary));
+                if (ret) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error setting dictionary");
+                        goto unwind;
+                }
 
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try = -1;
-        int read_child = -1;
+        unwind:
+                // Updating child_errno with more recent 'events'
+                op_errno = afr_final_errno (local, priv);
 
-	priv     = this->private;
-	children = priv->children;
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, xattr,
+                                  xdata);
+                if (xattr)
+                        dict_unref (xattr);
+        }
 
-	local = frame->local;
+        return ret;
+}
 
-        read_child = (long) cookie;
+int32_t
+afr_getxattr_clrlk_cbk (call_frame_t *frame, void *cookie,
+                        xlator_t *this, int32_t op_ret, int32_t op_errno,
+                        dict_t *dict, dict_t *xdata)
+{
+        afr_local_t     *local                  = NULL;
+        afr_private_t   *priv                   = NULL;
+        xlator_t        **children              = NULL;
+        dict_t          *xattr                  = NULL;
+        char            *tmp_report             = NULL;
+        char            lk_summary[1024]        = {0,};
+        int             serz_len                = 0;
+        int32_t         callcnt                 = 0;
+        long int        cky                     = 0;
+        int             ret                     = 0;
+
+        priv     = this->private;
+        children = priv->children;
+
+        local = frame->local;
+        cky = (long) cookie;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+                if (op_ret == -1)
+                        local->replies[cky].op_errno = op_errno;
+
+                if (!local->dict)
+                        local->dict = dict_new ();
+                if (local->dict) {
+                        ret = dict_get_str (dict, local->cont.getxattr.name,
+                                            &tmp_report);
+                        if (ret)
+                                goto unlock;
+                        ret = dict_set_dynstr (local->dict,
+                                               children[cky]->name,
+                                               gf_strdup (tmp_report));
+                        if (ret)
+                                goto unlock;
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                xattr = dict_new ();
+                if (!xattr) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        goto unwind;
+                }
+                ret = dict_serialize_value_with_delim (local->dict,
+                                                       lk_summary,
+                                                       &serz_len, '\n');
+                if (ret) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error serializing dictionary");
+                        goto unwind;
+                }
+                if (serz_len == -1)
+                        snprintf (lk_summary, sizeof (lk_summary),
+                                  "No locks cleared.");
+                ret = dict_set_dynstr (xattr, local->cont.getxattr.name,
+                                       gf_strdup (lk_summary));
+                if (ret) {
+                        op_ret = -1;
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Error setting dictionary");
+                        goto unwind;
+                }
 
-	if (op_ret == -1) {
-        retry:
-		last_tried = local->cont.readlink.last_tried;
+        unwind:
+                // Updating child_errno with more recent 'events'
+                op_errno = afr_final_errno (local, priv);
 
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
-		this_try = ++local->cont.readlink.last_tried;
+                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
 
-                if (this_try == read_child) {
-                        goto retry;
-                }
+                if (xattr)
+                        dict_unref (xattr);
+        }
 
-		unwind = 0;
-		STACK_WIND_COOKIE (frame, afr_readlink_cbk,
-				   (void *) (long) read_child,
-				   children[this_try],
-				   children[this_try]->fops->readlink,
-				   &local->loc,
-				   local->cont.readlink.size);
-	}
+        return ret;
+}
 
-out:
-	if (unwind) {
-                if (sbuf)
-                        sbuf->ia_ino = local->cont.readlink.ino;
+/**
+ * node-uuid cbk uses next child querying mechanism
+ */
+int32_t
+afr_getxattr_node_uuid_cbk (call_frame_t *frame, void *cookie,
+                            xlator_t *this, int32_t op_ret, int32_t op_errno,
+                            dict_t *dict, dict_t *xdata)
+{
+        afr_private_t  *priv            = NULL;
+        afr_local_t    *local           = NULL;
+        xlator_t      **children        = NULL;
+        int             unwind          = 1;
+        int             curr_call_child = 0;
+
+        priv = this->private;
+        children = priv->children;
+
+        local = frame->local;
+
+        if (op_ret == -1) { /** query the _next_ child */
+
+                /**
+                 * _current_ becomes _next_
+                 * If done with all childs and yet no success; give up !
+                 */
+                curr_call_child = (int) ((long)cookie);
+                if (++curr_call_child == priv->child_count)
+                        goto unwind;
+
+                gf_log (this->name, GF_LOG_WARNING,
+                        "op_ret (-1): Re-querying afr-child (%d/%d)",
+                        curr_call_child, priv->child_count);
+
+                unwind = 0;
+                STACK_WIND_COOKIE (frame, afr_getxattr_node_uuid_cbk,
+                                   (void *) (long) curr_call_child,
+                                   children[curr_call_child],
+                                   children[curr_call_child]->fops->getxattr,
+                                   &local->loc,
+                                   local->cont.getxattr.name,
+                                   NULL);
+        }
 
-		AFR_STACK_UNWIND (readlink, frame, op_ret, op_errno, buf, sbuf);
-	}
+ unwind:
+        if (unwind)
+                AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict,
+                                  NULL);
 
-	return 0;
+        return 0;
 }
 
-
 int32_t
-afr_readlink (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, size_t size)
+afr_getxattr_lockinfo_cbk (call_frame_t *frame, void *cookie,
+                           xlator_t *this, int32_t op_ret, int32_t op_errno,
+                           dict_t *dict, dict_t *xdata)
 {
-	afr_private_t * priv       = NULL;
-	xlator_t **     children   = NULL;
-	int             call_child = 0;
-	afr_local_t     *local     = NULL;
+        int          call_cnt     = 0, len = 0;
+        char        *lockinfo_buf = NULL;
+        dict_t      *lockinfo     = NULL, *newdict = NULL;
+        afr_local_t *local        = NULL;
 
-        int32_t         read_child = -1;
+        LOCK (&frame->lock);
+        {
+                local = frame->local;
 
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
+                call_cnt = --local->call_count;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+                if ((op_ret < 0) || (!dict && !xdata)) {
+                        goto unlock;
+                }
 
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
+                if (xdata) {
+                        if (!local->xdata_rsp) {
+                                local->xdata_rsp = dict_new ();
+                                if (!local->xdata_rsp) {
+                                        local->op_ret = -1;
+                                        local->op_errno = ENOMEM;
+                                        goto unlock;
+                                }
+                        }
+                }
 
-	children = priv->children;
+                if (!dict) {
+                        goto unlock;
+                }
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+                op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
+                                               (void **)&lockinfo_buf, &len);
 
-	frame->local = local;
+                if (!lockinfo_buf) {
+                        goto unlock;
+                }
 
-        read_child = afr_read_child (this, loc->inode);
+                if (!local->dict) {
+                        local->dict = dict_new ();
+                        if (!local->dict) {
+                                local->op_ret = -1;
+                                local->op_errno = ENOMEM;
+                                goto unlock;
+                        }
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (lockinfo_buf != NULL) {
+                lockinfo = dict_new ();
+                if (lockinfo == NULL) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                } else {
+                        op_ret = dict_unserialize (lockinfo_buf, len,
+                                                   &lockinfo);
+
+                        if (lockinfo && local->dict) {
+                                dict_copy (lockinfo, local->dict);
+                        }
+                }
+        }
 
-        if ((read_child >= 0) && (priv->child_up[read_child])) {
-                call_child = read_child;
+        if (xdata && local->xdata_rsp) {
+                dict_copy (xdata, local->xdata_rsp);
+        }
 
-                local->cont.readlink.last_tried = -1;
+        if (!call_cnt) {
+                newdict = dict_new ();
+                if (!newdict) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
 
-        } else {
-                call_child = afr_first_up_child (priv);
+                len = dict_serialized_length (local->dict);
+                if (len == 0) {
+                        goto unwind;
+                }
 
-                if (call_child == -1) {
-                        op_errno = ENOTCONN;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "no child is up");
-                        goto out;
+                lockinfo_buf = GF_CALLOC (1, len, gf_common_mt_char);
+                if (!lockinfo_buf) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
                 }
 
-                local->cont.readlink.last_tried = call_child;
-        }
+                op_ret = dict_serialize (local->dict, lockinfo_buf);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                }
 
-	loc_copy (&local->loc, loc);
+                op_ret = dict_set_dynptr (newdict, GF_XATTR_LOCKINFO_KEY,
+                                          (void *)lockinfo_buf, len);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                        goto unwind;
+                }
 
-	local->cont.readlink.size       = size;
-        local->cont.readlink.ino        = loc->inode->ino;
+        unwind:
+                AFR_STACK_UNWIND (getxattr, frame, op_ret,
+                                  op_errno, newdict,
+                                  local->xdata_rsp);
+        }
 
-	STACK_WIND_COOKIE (frame, afr_readlink_cbk,
-			   (void *) (long) call_child,
-			   children[call_child], children[call_child]->fops->readlink,
-			   loc, size);
+        dict_unref (lockinfo);
 
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (readlink, frame, op_ret, op_errno, NULL, NULL);
-	}
-	return 0;
+        return 0;
 }
 
+int32_t
+afr_fgetxattr_lockinfo_cbk (call_frame_t *frame, void *cookie,
+                            xlator_t *this, int32_t op_ret, int32_t op_errno,
+                            dict_t *dict, dict_t *xdata)
+{
+        int          call_cnt     = 0, len = 0;
+        char        *lockinfo_buf = NULL;
+        dict_t      *lockinfo     = NULL, *newdict = NULL;
+        afr_local_t *local        = NULL;
 
-/* }}} */
+        LOCK (&frame->lock);
+        {
+                local = frame->local;
 
-/* {{{ getxattr */
+                call_cnt = --local->call_count;
 
-struct _xattr_key {
-        char *key;
-        struct list_head list;
-};
+                if ((op_ret < 0) || (!dict && !xdata)) {
+                        goto unlock;
+                }
 
+                if (xdata) {
+                        if (!local->xdata_rsp) {
+                                local->xdata_rsp = dict_new ();
+                                if (!local->xdata_rsp) {
+                                        local->op_ret = -1;
+                                        local->op_errno = ENOMEM;
+                                        goto unlock;
+                                }
+                        }
+                }
 
-void
-__gather_xattr_keys (dict_t *dict, char *key, data_t *value,
-                     void *data)
-{
-        struct list_head *  list  = data;
-        struct _xattr_key * xkey  = NULL;
+                if (!dict) {
+                        goto unlock;
+                }
 
-        if (!strncmp (key, AFR_XATTR_PREFIX,
-                      strlen (AFR_XATTR_PREFIX))) {
+                op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
+                                               (void **)&lockinfo_buf, &len);
 
-                xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
-                if (!xkey)
-                        return;
+                if (!lockinfo_buf) {
+                        goto unlock;
+                }
 
-                xkey->key = key;
-                INIT_LIST_HEAD (&xkey->list);
+                if (!local->dict) {
+                        local->dict = dict_new ();
+                        if (!local->dict) {
+                                local->op_ret = -1;
+                                local->op_errno = ENOMEM;
+                                goto unlock;
+                        }
+                }
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (lockinfo_buf != NULL) {
+                lockinfo = dict_new ();
+                if (lockinfo == NULL) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                } else {
+                        op_ret = dict_unserialize (lockinfo_buf, len,
+                                                   &lockinfo);
+
+                        if (lockinfo && local->dict) {
+                                dict_copy (lockinfo, local->dict);
+                        }
+                }
+        }
 
-                list_add_tail (&xkey->list, list);
+        if (xdata && local->xdata_rsp) {
+                dict_copy (xdata, local->xdata_rsp);
         }
-}
 
+        if (!call_cnt) {
+                newdict = dict_new ();
+                if (!newdict) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
 
-void
-__filter_xattrs (dict_t *dict)
+                len = dict_serialized_length (local->dict);
+                if (len <= 0) {
+                        goto unwind;
+                }
+
+                lockinfo_buf = GF_CALLOC (1, len, gf_common_mt_char);
+                if (!lockinfo_buf) {
+                        local->op_ret = -1;
+                        local->op_errno = ENOMEM;
+                        goto unwind;
+                }
+
+                op_ret = dict_serialize (local->dict, lockinfo_buf);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                }
+
+                op_ret = dict_set_dynptr (newdict, GF_XATTR_LOCKINFO_KEY,
+                                          (void *)lockinfo_buf, len);
+                if (op_ret < 0) {
+                        local->op_ret = -1;
+                        local->op_errno = -op_ret;
+                        goto unwind;
+                }
+
+        unwind:
+                AFR_STACK_UNWIND (fgetxattr, frame, op_ret,
+                                  op_errno, newdict,
+                                  local->xdata_rsp);
+        }
+
+        dict_unref (lockinfo);
+
+        return 0;
+}
+
+int32_t
+afr_fgetxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
+                            xlator_t *this, int32_t op_ret, int32_t op_errno,
+                            dict_t *dict, dict_t *xdata)
 {
-        struct list_head keys;
+        afr_local_t *local          = NULL;
+        int32_t      callcnt        = 0;
+        int          ret            = 0;
+        char        *xattr          = NULL;
+        char        *xattr_serz     = NULL;
+        char        xattr_cky[1024] = {0,};
+        dict_t      *nxattr         = NULL;
+        long         cky            = 0;
+        int32_t      padding        = 0;
+        int32_t      tlen           = 0;
+
+        if (!frame || !frame->local || !this) {
+                gf_log ("", GF_LOG_ERROR, "possible NULL deref");
+                goto out;
+        }
 
-        struct _xattr_key *key;
-        struct _xattr_key *tmp;
+        local = frame->local;
+        cky = (long) cookie;
 
-        INIT_LIST_HEAD (&keys);
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
 
-        dict_foreach (dict, __gather_xattr_keys,
-                      (void *) &keys);
+                if (op_ret < 0) {
+                        local->op_errno = op_errno;
+                } else {
+                        local->op_ret = op_ret;
+                        if (!local->xdata_rsp && xdata)
+                                local->xdata_rsp = dict_ref (xdata);
+                }
 
-        list_for_each_entry_safe (key, tmp, &keys, list) {
-                dict_del (dict, key->key);
+                if (!dict || (op_ret < 0))
+                        goto out;
 
-                list_del_init (&key->list);
+                if (!local->dict)
+                        local->dict = dict_new ();
+
+                if (local->dict) {
+                        ret = dict_get_str (dict,
+                                            local->cont.getxattr.name,
+                                            &xattr);
+                        if (ret)
+                                goto out;
+
+                        xattr = gf_strdup (xattr);
+
+                        (void)snprintf (xattr_cky, 1024, "%s-%ld",
+                                        local->cont.getxattr.name, cky);
+                        ret = dict_set_dynstr (local->dict,
+                                               xattr_cky, xattr);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Cannot set xattr cookie key");
+                                goto out;
+                        }
+
+                        local->cont.getxattr.xattr_len
+                                += strlen (xattr) + 1;
+                }
+        }
+out:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (!local->cont.getxattr.xattr_len)
+                        goto unwind;
+
+                nxattr = dict_new ();
+                if (!nxattr)
+                        goto unwind;
+
+                /* extra bytes for decorations (brackets and <>'s) */
+                padding += strlen (this->name)
+                        + strlen (AFR_PATHINFO_HEADER) + 4;
+                local->cont.getxattr.xattr_len += (padding + 2);
+
+                xattr_serz = GF_CALLOC (local->cont.getxattr.xattr_len,
+                                        sizeof (char), gf_common_mt_char);
+
+                if (!xattr_serz)
+                        goto unwind;
+
+                /* the xlator info */
+                (void) sprintf (xattr_serz, "(<"AFR_PATHINFO_HEADER"%s> ",
+                                this->name);
+
+                /* actual series of pathinfo */
+                ret = dict_serialize_value_with_delim (local->dict,
+                                                       xattr_serz
+                                                       + strlen (xattr_serz),
+                                                       &tlen, ' ');
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Error serializing"
+                                " dictionary");
+                        goto unwind;
+                }
 
-                GF_FREE (key);
+                /* closing part */
+                *(xattr_serz + padding + tlen) = ')';
+                *(xattr_serz + padding + tlen + 1) = '\0';
+
+                ret = dict_set_dynstr (nxattr, local->cont.getxattr.name,
+                                       xattr_serz);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR, "Cannot set pathinfo"
+                                " key in dict");
+
+        unwind:
+                AFR_STACK_UNWIND (fgetxattr, frame, local->op_ret,
+                                  local->op_errno, nxattr, local->xdata_rsp);
+
+                if (nxattr)
+                        dict_unref (nxattr);
         }
+
+        return ret;
 }
 
+int32_t
+afr_getxattr_pathinfo_cbk (call_frame_t *frame, void *cookie,
+                           xlator_t *this, int32_t op_ret, int32_t op_errno,
+                           dict_t *dict, dict_t *xdata)
+{
+        afr_local_t *local          = NULL;
+        int32_t      callcnt        = 0;
+        int          ret            = 0;
+        char        *xattr          = NULL;
+        char        *xattr_serz     = NULL;
+        char        xattr_cky[1024] = {0,};
+        dict_t      *nxattr         = NULL;
+        long         cky            = 0;
+        int32_t      padding        = 0;
+        int32_t      tlen           = 0;
+
+        if (!frame || !frame->local || !this) {
+                gf_log ("", GF_LOG_ERROR, "possible NULL deref");
+                goto out;
+        }
 
+        local = frame->local;
+        cky = (long) cookie;
+
+        LOCK (&frame->lock);
+                {
+                        callcnt = --local->call_count;
+
+                        if (op_ret < 0) {
+                                local->op_errno = op_errno;
+                        } else {
+                                local->op_ret = op_ret;
+                                if (!local->xdata_rsp && xdata)
+                                        local->xdata_rsp = dict_ref (xdata);
+                        }
+
+                        if (!dict || (op_ret < 0))
+                                goto out;
+
+                        if (!local->dict)
+                                local->dict = dict_new ();
+
+                        if (local->dict) {
+                                ret = dict_get_str (dict,
+                                                    local->cont.getxattr.name,
+                                                    &xattr);
+                                if (ret)
+                                        goto out;
+
+                                xattr = gf_strdup (xattr);
+
+                                (void)snprintf (xattr_cky, 1024, "%s-%ld",
+                                                local->cont.getxattr.name, cky);
+                                ret = dict_set_dynstr (local->dict,
+                                                       xattr_cky, xattr);
+                                if (ret) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Cannot set xattr cookie key");
+                                        goto out;
+                                }
+
+                                local->cont.getxattr.xattr_len += strlen (xattr) + 1;
+                        }
+                }
+ out:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (!local->cont.getxattr.xattr_len)
+                        goto unwind;
+
+                nxattr = dict_new ();
+                if (!nxattr)
+                        goto unwind;
+
+                /* extra bytes for decorations (brackets and <>'s) */
+                padding += strlen (this->name) + strlen (AFR_PATHINFO_HEADER) + 4;
+                local->cont.getxattr.xattr_len += (padding + 2);
+
+                xattr_serz = GF_CALLOC (local->cont.getxattr.xattr_len,
+                                        sizeof (char), gf_common_mt_char);
+
+                if (!xattr_serz)
+                        goto unwind;
+
+                /* the xlator info */
+                (void) sprintf (xattr_serz, "(<"AFR_PATHINFO_HEADER"%s> ",
+                                this->name);
+
+                /* actual series of pathinfo */
+                ret = dict_serialize_value_with_delim (local->dict,
+                                                       xattr_serz + strlen (xattr_serz),
+                                                       &tlen, ' ');
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Error serializing"
+                                " dictionary");
+                        goto unwind;
+                }
 
-int32_t
-afr_getxattr_cbk (call_frame_t *frame, void *cookie,
-		  xlator_t *this, int32_t op_ret, int32_t op_errno,
-		  dict_t *dict)
+                /* closing part */
+                *(xattr_serz + padding + tlen) = ')';
+                *(xattr_serz + padding + tlen + 1) = '\0';
+
+                ret = dict_set_dynstr (nxattr, local->cont.getxattr.name,
+                                       xattr_serz);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR, "Cannot set pathinfo"
+                                " key in dict");
+
+        unwind:
+                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
+                                  local->op_errno, nxattr, local->xdata_rsp);
+
+                if (nxattr)
+                        dict_unref (nxattr);
+        }
+
+        return ret;
+}
+
+static int
+afr_aggregate_stime_xattr (dict_t *this, char *key, data_t *value, void *data)
 {
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
+        int ret = 0;
 
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try = -1;
-        int read_child = -1;
+        if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0)
+                ret = gf_get_max_stime (THIS, data, key, value);
 
-	priv     = this->private;
-	children = priv->children;
+        return ret;
+}
 
-	local = frame->local;
+int32_t
+afr_common_getxattr_stime_cbk (call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               dict_t *dict, dict_t *xdata)
+{
+        afr_local_t *local          = NULL;
+        int32_t      callcnt        = 0;
 
-        read_child = (long) cookie;
+        if (!frame || !frame->local || !this) {
+                gf_log ("", GF_LOG_ERROR, "possible NULL deref");
+                goto out;
+        }
 
-	if (op_ret == -1) {
-        retry:
-		last_tried = local->cont.getxattr.last_tried;
+        local = frame->local;
 
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
-		this_try = ++local->cont.getxattr.last_tried;
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
 
-                if (this_try == read_child) {
-                        goto retry;
+                if (!dict || (op_ret < 0)) {
+                        local->op_errno = op_errno;
+                        goto cleanup;
                 }
 
-		unwind = 0;
-		STACK_WIND_COOKIE (frame, afr_getxattr_cbk,
-				   (void *) (long) read_child,
-				   children[this_try],
-				   children[this_try]->fops->getxattr,
-				   &local->loc,
-				   local->cont.getxattr.name);
-	}
+                if (!local->dict)
+                        local->dict = dict_copy_with_ref (dict, NULL);
+                else
+                        dict_foreach (dict, afr_aggregate_stime_xattr,
+                                      local->dict);
+                local->op_ret = 0;
+        }
+
+cleanup:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                AFR_STACK_UNWIND (getxattr, frame, local->op_ret,
+                                  local->op_errno, local->dict, xdata);
+        }
 
 out:
-	if (unwind) {
-                if (op_ret >= 0 && dict)
-                        __filter_xattrs (dict);
+        return 0;
+}
 
-		AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
-	}
 
-	return 0;
+static gf_boolean_t
+afr_is_special_xattr (const char *name, fop_getxattr_cbk_t *cbk,
+                      gf_boolean_t is_fgetxattr)
+{
+        gf_boolean_t    is_spl = _gf_true;
+
+        GF_ASSERT (cbk);
+        if (!cbk || !name) {
+                is_spl = _gf_false;
+                goto out;
+        }
+
+        if (!strcmp (name, GF_XATTR_PATHINFO_KEY) ||
+	    !strcmp (name, GF_XATTR_USER_PATHINFO_KEY)) {
+                if (is_fgetxattr) {
+                        *cbk = afr_fgetxattr_pathinfo_cbk;
+                } else {
+                        *cbk = afr_getxattr_pathinfo_cbk;
+                }
+        } else if (!strncmp (name, GF_XATTR_CLRLK_CMD,
+                             strlen (GF_XATTR_CLRLK_CMD))) {
+                if (is_fgetxattr) {
+                        *cbk = afr_fgetxattr_clrlk_cbk;
+                } else {
+                        *cbk = afr_getxattr_clrlk_cbk;
+                }
+        } else if (!strncmp (name, GF_XATTR_LOCKINFO_KEY,
+                             strlen (GF_XATTR_LOCKINFO_KEY))) {
+                if (is_fgetxattr) {
+                        *cbk = afr_fgetxattr_lockinfo_cbk;
+                } else {
+                        *cbk = afr_getxattr_lockinfo_cbk;
+                }
+        } else if (fnmatch (GF_XATTR_STIME_PATTERN, name, FNM_NOESCAPE) == 0) {
+                *cbk = afr_common_getxattr_stime_cbk;
+        } else {
+                is_spl = _gf_false;
+        }
+
+out:
+        return is_spl;
+}
+
+static void
+afr_getxattr_all_subvols (xlator_t *this, call_frame_t *frame,
+			  const char *name, loc_t *loc,
+			  fop_getxattr_cbk_t cbk)
+{
+        afr_private_t   *priv           = NULL;
+        afr_local_t     *local          = NULL;
+        int             i               = 0;
+        int             call_count      = 0;
+
+        priv     = this->private;
+
+        local = frame->local;
+        //local->call_count set in afr_local_init
+        call_count = local->call_count;
+
+        //If up-children count is 0, afr_local_init would have failed already
+        //and the call would have unwound so not handling it here.
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND_COOKIE (frame, cbk,
+                                           (void *) (long) i, priv->children[i],
+                                           priv->children[i]->fops->getxattr,
+                                           loc, name, NULL);
+                        if (!--call_count)
+                                break;
+                }
+        }
+        return;
 }
 
 int32_t
 afr_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *name)
+              loc_t *loc, const char *name, dict_t *xdata)
 {
-	afr_private_t *   priv       = NULL;
-	xlator_t **       children   = NULL;
-	int               call_child = 0;
-	afr_local_t     * local      = NULL;
+        afr_private_t           *priv         = NULL;
+        xlator_t                **children    = NULL;
+        afr_local_t             *local        = NULL;
+        xlator_list_t           *trav         = NULL;
+        xlator_t                **sub_volumes = NULL;
+        int                     i             = 0;
+        int32_t                 op_errno      = 0;
+        int                     ret           = -1;
+        fop_getxattr_cbk_t      cbk           = NULL;
+        int                     afr_xtime_gauge[MCNT_MAX] = {0,};
 
-        int               read_child = -1;
 
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
+        priv     = this->private;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        children = priv->children;
 
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
+        loc_copy (&local->loc, loc);
 
-	children = priv->children;
+	local->op = GF_FOP_GETXATTR;
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-	frame->local = local;
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
 
-        if (name) {
-                if (!strncmp (name, AFR_XATTR_PREFIX,
-                              strlen (AFR_XATTR_PREFIX))) {
+        if (!name)
+                goto no_name;
 
-                        op_errno = ENODATA;
-                        goto out;
-                }
+        local->cont.getxattr.name = gf_strdup (name);
+
+	if (!local->cont.getxattr.name) {
+		op_errno = ENOMEM;
+		goto out;
+	}
 
+        if (!strncmp (name, AFR_XATTR_PREFIX,
+                      strlen (AFR_XATTR_PREFIX))) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%s: no data present for key %s",
+                        loc->path, name);
+                op_errno = ENODATA;
+                goto out;
         }
+        if ((strcmp (GF_XATTR_MARKER_KEY, name) == 0)
+            && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
 
-        read_child = afr_read_child (this, loc->inode);
+                local->marker.call_count = priv->child_count;
 
-        if ((read_child >= 0) && (priv->child_up[read_child])) {
-                call_child = read_child;
+                sub_volumes = alloca ( priv->child_count * sizeof (xlator_t *));
+                for (i = 0, trav = this->children; trav ;
+                     trav = trav->next, i++) {
 
-                local->cont.getxattr.last_tried = -1;
-        } else {
-                call_child = afr_first_up_child (priv);
+                        *(sub_volumes + i)  = trav->xlator;
+                }
 
-                if (call_child == -1) {
-                        op_errno = ENOTCONN;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "no child is up");
+                if (cluster_getmarkerattr (frame, this, loc, name,
+                                           local, afr_getxattr_unwind,
+                                           sub_volumes,
+                                           priv->child_count,
+                                           MARKER_UUID_TYPE,
+                                           marker_uuid_default_gauge,
+                                           priv->vol_uuid)) {
+
+                        gf_log (this->name, GF_LOG_INFO,
+                                "%s: failed to get marker attr (%s)",
+                                loc->path, name);
+                        op_errno = EINVAL;
                         goto out;
                 }
 
-                local->cont.getxattr.last_tried = call_child;
+                return 0;
         }
 
-	loc_copy (&local->loc, loc);
-	if (name)
-	  local->cont.getxattr.name       = gf_strdup (name);
+        /*
+         * if we are doing getxattr with pathinfo as the key then we
+         * collect information from all childs
+         */
+        if (afr_is_special_xattr (name, &cbk, 0)) {
+                afr_getxattr_all_subvols (this, frame, name, loc, cbk);
+                return 0;
+        }
 
-	STACK_WIND_COOKIE (frame, afr_getxattr_cbk,
-			   (void *) (long) call_child,
-			   children[call_child], children[call_child]->fops->getxattr,
-			   loc, name);
+        if (XATTR_IS_NODE_UUID (name)) {
+                i = 0;
+                STACK_WIND_COOKIE (frame, afr_getxattr_node_uuid_cbk,
+                                   (void *) (long) i,
+                                   children[i],
+                                   children[i]->fops->getxattr,
+                                   loc, name, xdata);
+                return 0;
+        }
 
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, NULL);
-	}
-	return 0;
-}
+        if (*priv->vol_uuid) {
+                if ((match_uuid_local (name, priv->vol_uuid) == 0)
+                    && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+                        local->marker.call_count = priv->child_count;
+
+                        sub_volumes = alloca ( priv->child_count
+                                               * sizeof (xlator_t *));
+                        for (i = 0, trav = this->children; trav ;
+                             trav = trav->next, i++) {
+
+                                *(sub_volumes + i)  = trav->xlator;
+
+                        }
+
+                        /* don't err out on getting ENOTCONN (brick down)
+                         * from a subset of the bricks
+                         */
+                        memcpy (afr_xtime_gauge, marker_xtime_default_gauge,
+                                sizeof (afr_xtime_gauge));
+                        afr_xtime_gauge[MCNT_NOTFOUND] = 0;
+                        afr_xtime_gauge[MCNT_ENOTCONN] = 0;
+                        if (cluster_getmarkerattr (frame, this, loc,
+                                                   name, local,
+                                                   afr_getxattr_unwind,
+                                                   sub_volumes,
+                                                   priv->child_count,
+                                                   MARKER_XTIME_TYPE,
+                                                   afr_xtime_gauge,
+                                                   priv->vol_uuid)) {
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "%s: failed to get marker attr (%s)",
+                                        loc->path, name);
+                                op_errno = EINVAL;
+                                goto out;
+                        }
+
+                        return 0;
+                }
+        }
 
+no_name:
 
-/* }}} */
+	afr_read_txn (frame, this, local->loc.inode, afr_getxattr_wind,
+		      AFR_METADATA_TRANSACTION);
 
-/* {{{ readv */
+        ret = 0;
+out:
+        if (ret < 0)
+                AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
+
+/* {{{ fgetxattr */
 
-/**
- * read algorithm:
- *
- * if the user has specified a read subvolume, use it
- * otherwise -
- *   use the inode number to hash it to one of the subvolumes, and
- *   read from there (to balance read load)
- *
- * if any of the above read's fail, try the children in sequence
- * beginning at the beginning
- */
 
 int32_t
-afr_readv_cbk (call_frame_t *frame, void *cookie,
-	       xlator_t *this, int32_t op_ret, int32_t op_errno,
-	       struct iovec *vector, int32_t count, struct iatt *buf,
-               struct iobref *iobref)
+afr_fgetxattr_cbk (call_frame_t *frame, void *cookie,
+                   xlator_t *this, int32_t op_ret, int32_t op_errno,
+                   dict_t *dict, dict_t *xdata)
 {
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
+	afr_local_t *local = NULL;
 
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try   = -1;
-        int read_child = -1;
+	local = frame->local;
+
+	if (op_ret < 0) {
+		local->op_ret = -1;
+		local->op_errno = op_errno;
+
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
+
+	if (dict)
+		afr_filter_xattrs (dict);
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+	AFR_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict, xdata);
 
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
+	return 0;
+}
 
-	children = priv->children;
+int
+afr_fgetxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
 
 	local = frame->local;
+	priv = this->private;
 
-        read_child = (long) cookie;
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (fgetxattr, frame, local->op_ret,
+				  local->op_errno, NULL, NULL);
+		return 0;
+	}
 
-	if (op_ret == -1) {
-	retry:
-		last_tried = local->cont.readv.last_tried;
+	STACK_WIND_COOKIE (frame, afr_fgetxattr_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->fgetxattr,
+			   local->fd, local->cont.getxattr.name,
+			   local->xdata_req);
+	return 0;
+}
 
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
-		this_try = ++local->cont.readv.last_tried;
-
-		if (this_try == read_child) {
-			/*
-			   skip the read child since if we are here
-			   we must have already tried that child
-			*/
-			goto retry;
-		}
 
-		unwind = 0;
+static void
+afr_fgetxattr_all_subvols (xlator_t *this, call_frame_t *frame,
+			   fop_fgetxattr_cbk_t cbk)
+{
+        afr_private_t   *priv           = NULL;
+        afr_local_t     *local          = NULL;
+        int             i               = 0;
+        int             call_count      = 0;
+
+        priv     = this->private;
+
+        local = frame->local;
+        //local->call_count set in afr_local_init
+        call_count = local->call_count;
+
+        //If up-children count is 0, afr_local_init would have failed already
+        //and the call would have unwound so not handling it here.
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND_COOKIE (frame, cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->fgetxattr,
+                                           local->fd, local->cont.getxattr.name,
+					   NULL);
+                        if (!--call_count)
+                                break;
+                }
+        }
 
-		STACK_WIND_COOKIE (frame, afr_readv_cbk,
-				   (void *) (long) read_child,
-				   children[this_try],
-				   children[this_try]->fops->readv,
-				   local->fd, local->cont.readv.size,
-				   local->cont.readv.offset);
-	}
+        return;
+}
 
-out:
-	if (unwind) {
-                if (buf && local)
-                        buf->ia_ino = local->cont.readv.ino;
 
-		AFR_STACK_UNWIND (readv, frame, op_ret, op_errno,
-                                  vector, count, buf, iobref);
+int
+afr_fgetxattr (call_frame_t *frame, xlator_t *this,
+               fd_t *fd, const char *name, dict_t *xdata)
+{
+        afr_local_t          *local      = NULL;
+        int32_t               op_errno   = 0;
+        fop_fgetxattr_cbk_t   cbk        = NULL;
+
+        local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
+
+	local->op = GF_FOP_FGETXATTR;
+        local->fd = fd_ref (fd);
+        if (name) {
+                local->cont.getxattr.name = gf_strdup (name);
+		if (!local->cont.getxattr.name) {
+			op_errno = ENOMEM;
+			goto out;
+		}
 	}
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
+
+        /* pathinfo gets handled only in getxattr(), but we need to handle
+         * lockinfo.
+         * If we are doing fgetxattr with lockinfo as the key then we
+         * collect information from all children.
+         */
+        if (afr_is_special_xattr (name, &cbk, 1)) {
+                afr_fgetxattr_all_subvols (this, frame, cbk);
+                return 0;
+        }
+
+        afr_fix_open (fd, this);
+
+	afr_read_txn (frame, this, fd->inode, afr_fgetxattr_wind,
+		      AFR_METADATA_TRANSACTION);
 
 	return 0;
+out:
+	AFR_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL, NULL);
+
+        return 0;
 }
 
 
-int32_t
-afr_readv (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, size_t size, off_t offset)
+/* }}} */
+
+/* {{{ readv */
+
+int
+afr_readv_cbk (call_frame_t *frame, void *cookie,
+               xlator_t *this, int32_t op_ret, int32_t op_errno,
+               struct iovec *vector, int32_t count, struct iatt *buf,
+               struct iobref *iobref, dict_t *xdata)
 {
-	afr_private_t * priv       = NULL;
-	afr_local_t   * local      = NULL;
-	xlator_t **     children   = NULL;
+	afr_local_t *local = NULL;
 
-        int32_t         read_child = -1;
-	int             call_child = 0;
+	local = frame->local;
+
+	if (op_ret < 0) {
+		local->op_ret = -1;
+		local->op_errno = op_errno;
 
-	int32_t         op_ret     = -1;
-	int32_t         op_errno   = 0;
+		afr_read_txn_continue (frame, this, (long) cookie);
+		return 0;
+	}
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-	VALIDATE_OR_GOTO (fd, out);
+	AFR_STACK_UNWIND (readv, frame, op_ret, op_errno,
+			  vector, count, buf, iobref, xdata);
+	return 0;
+}
 
-	priv     = this->private;
-	children = priv->children;
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
+int
+afr_readv_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
 
-	frame->local = local;
+	local = frame->local;
+	priv = this->private;
 
-        read_child = afr_read_child (this, fd->inode);
+	if (subvol == -1) {
+		AFR_STACK_UNWIND (readv, frame, local->op_ret, local->op_errno,
+				  0, 0, 0, 0, 0);
+		return 0;
+	}
 
-        if ((read_child >= 0) && (priv->child_up[read_child])) {
-                call_child = read_child;
+	STACK_WIND_COOKIE (frame, afr_readv_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->readv,
+			   local->fd, local->cont.readv.size,
+			   local->cont.readv.offset, local->cont.readv.flags,
+			   local->xdata_req);
+	return 0;
+}
 
-		/*
-		   if read fails from the read child, we try
-		   all children starting with the first one
-		*/
-                local->cont.readv.last_tried = -1;
 
-        } else {
-		call_child = afr_first_up_child (priv);
-		if (call_child == -1) {
-			op_errno = ENOTCONN;
-			gf_log (this->name, GF_LOG_DEBUG,
-				"no child is up");
-			goto out;
-		}
+int
+afr_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+	   off_t offset, uint32_t flags, dict_t *xdata)
+{
+        afr_local_t   * local      = NULL;
+        int32_t         op_errno   = 0;
 
-		local->cont.readv.last_tried = call_child;
-	}
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-	local->fd                    = fd_ref (fd);
+	local->op = GF_FOP_READ;
+        local->fd = fd_ref (fd);
+        local->cont.readv.size = size;
+        local->cont.readv.offset = offset;
+        local->cont.readv.flags = flags;
+	if (xdata)
+		local->xdata_req = dict_ref (xdata);
 
-        local->cont.readv.ino        = fd->inode->ino;
-	local->cont.readv.size       = size;
-	local->cont.readv.offset     = offset;
+        afr_fix_open (fd, this);
 
-	STACK_WIND_COOKIE (frame, afr_readv_cbk,
-			   (void *) (long) call_child,
-			   children[call_child],
-			   children[call_child]->fops->readv,
-			   fd, size, offset);
+	afr_read_txn (frame, this, fd->inode, afr_readv_wind,
+		      AFR_DATA_TRANSACTION);
 
-	op_ret = 0;
+        return 0;
 out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (readv, frame, op_ret, op_errno, NULL, 0, NULL,
-                                  NULL);
-	}
+	AFR_STACK_UNWIND(readv, frame, -1, op_errno, 0, 0, 0, 0, 0);
+
 	return 0;
 }
 
diff --git a/xlators/cluster/afr/src/afr-inode-read.h b/xlators/cluster/afr/src/afr-inode-read.h
index 9b29d5055..e4091a793 100644
--- a/xlators/cluster/afr/src/afr-inode-read.h
+++ b/xlators/cluster/afr/src/afr-inode-read.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __INODE_READ_H__
@@ -22,26 +13,30 @@
 
 int32_t
 afr_access (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t mask);
+	    loc_t *loc, int32_t mask, dict_t *xdata);
 
 int32_t
 afr_stat (call_frame_t *frame, xlator_t *this,
-	  loc_t *loc);
+	  loc_t *loc, dict_t *xdata);
 
 int32_t
 afr_fstat (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd);
+	   fd_t *fd, dict_t *xdata);
 
 int32_t
 afr_readlink (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, size_t size);
+	      loc_t *loc, size_t size, dict_t *xdata);
 
 int32_t
 afr_readv (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, size_t size, off_t offset);
+	   fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata);
 
 int32_t
 afr_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *name);
+	      loc_t *loc, const char *name, dict_t *xdata);
+
+int32_t
+afr_fgetxattr (call_frame_t *frame, xlator_t *this,
+               fd_t *fd, const char *name, dict_t *xdata);
 
 #endif /* __INODE_READ_H__ */
diff --git a/xlators/cluster/afr/src/afr-inode-write.c b/xlators/cluster/afr/src/afr-inode-write.c
index 2eaf99c55..00e0d2676 100644
--- a/xlators/cluster/afr/src/afr-inode-write.c
+++ b/xlators/cluster/afr/src/afr-inode-write.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -47,145 +38,301 @@
 #include "afr.h"
 #include "afr-transaction.h"
 
-/* {{{ writev */
 
-int
-afr_writev_unwind (call_frame_t *frame, xlator_t *this)
+static void
+__afr_inode_write_finalize (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int read_subvol = 0;
+	int i = 0;
+
+	local = frame->local;
+	priv = this->private;
+
+	if (local->inode) {
+		if (local->transaction.type == AFR_METADATA_TRANSACTION)
+			read_subvol = afr_metadata_subvol_get (local->inode, this,
+							       NULL, NULL);
+		else
+			read_subvol = afr_data_subvol_get (local->inode, this,
+							   NULL, NULL);
+	}
+
+	local->op_ret = -1;
+	local->op_errno = afr_final_errno (local, priv);
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->replies[i].valid)
+			continue;
+		if (local->replies[i].op_ret < 0) {
+			afr_inode_read_subvol_reset (local->inode, this);
+			continue;
+		}
+
+		/* Order of checks in the compound conditional
+		   below is important.
+
+		   - Highest precedence: largest op_ret
+		   - Next precendence: if all op_rets are equal, read subvol
+		   - Least precedence: any succeeded subvol
+		*/
+		if ((local->op_ret < local->replies[i].op_ret) ||
+		    ((local->op_ret == local->replies[i].op_ret) &&
+		     (i == read_subvol))) {
+
+			local->op_ret = local->replies[i].op_ret;
+			local->op_errno = local->replies[i].op_errno;
+
+			local->cont.inode_wfop.prebuf =
+				local->replies[i].prestat;
+			local->cont.inode_wfop.postbuf =
+				local->replies[i].poststat;
+
+			if (local->replies[i].xdata) {
+				if (local->xdata_rsp)
+					dict_unref (local->xdata_rsp);
+				local->xdata_rsp =
+					dict_ref (local->replies[i].xdata);
+			}
+		}
+	}
+}
 
-        local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+static void
+__afr_inode_write_fill (call_frame_t *frame, xlator_t *this, int child_index,
+			int op_ret, int op_errno,
+			struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t *local = NULL;
 
-        if (main_frame) {
-                local->cont.writev.prebuf.ia_ino  = local->cont.writev.ino;
-                local->cont.writev.postbuf.ia_ino = local->cont.writev.ino;
+        local = frame->local;
 
-                AFR_STACK_UNWIND (writev, main_frame,
-                                  local->op_ret, local->op_errno,
-                                  &local->cont.writev.prebuf,
-                                  &local->cont.writev.postbuf);
-        }
-        return 0;
+	local->replies[child_index].valid = 1;
+	local->replies[child_index].op_ret = op_ret;
+	local->replies[child_index].op_errno = op_errno;
+
+	if (op_ret >= 0) {
+		if (prebuf)
+			local->replies[child_index].prestat = *prebuf;
+		if (postbuf)
+			local->replies[child_index].poststat = *postbuf;
+		if (xdata)
+			local->replies[child_index].xdata = dict_ref (xdata);
+	} else {
+		afr_transaction_fop_failed (frame, this, child_index);
+	}
+
+        return;
 }
 
 
-int
-afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf)
+static int
+__afr_inode_write_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-
+        afr_local_t *local = NULL;
         int child_index = (long) cookie;
-        int call_count  = -1;
-        int read_child  = 0;
+        int call_count = -1;
 
         local = frame->local;
 
-        read_child = afr_read_child (this, local->fd->inode);
-
         LOCK (&frame->lock);
         {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
-
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
-
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret              = op_ret;
-                                local->cont.writev.prebuf  = *prebuf;
-                                local->cont.writev.postbuf = *postbuf;
-                        }
-
-                        if (child_index == read_child) {
-                                local->cont.writev.prebuf  = *prebuf;
-                                local->cont.writev.postbuf = *postbuf;
-                        }
-                }
-
-                local->op_errno = op_errno;
+                __afr_inode_write_fill (frame, this, child_index, op_ret,
+					op_errno, prebuf, postbuf, xdata);
         }
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
-                local->transaction.unwind (frame, this);
+		__afr_inode_write_finalize (frame, this);
+
+		if (afr_txn_nothing_failed (frame, this))
+			local->transaction.unwind (frame, this);
 
                 local->transaction.resume (frame, this);
         }
+
         return 0;
 }
 
+/* {{{ writev */
+
+void
+afr_writev_copy_outvars (call_frame_t *src_frame, call_frame_t *dst_frame)
+{
+        afr_local_t *src_local = NULL;
+        afr_local_t *dst_local = NULL;
+
+        src_local = src_frame->local;
+        dst_local = dst_frame->local;
+
+        dst_local->op_ret = src_local->op_ret;
+        dst_local->op_errno = src_local->op_errno;
+        dst_local->cont.inode_wfop.prebuf = src_local->cont.inode_wfop.prebuf;
+        dst_local->cont.inode_wfop.postbuf = src_local->cont.inode_wfop.postbuf;
+	if (src_local->xdata_rsp)
+		dst_local->xdata_rsp = dict_ref (src_local->xdata_rsp);
+}
+
+void
+afr_writev_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *   local = NULL;
+        local = frame->local;
+
+        AFR_STACK_UNWIND (writev, frame,
+                          local->op_ret, local->op_errno,
+                          &local->cont.inode_wfop.prebuf,
+                          &local->cont.inode_wfop.postbuf,
+                          local->xdata_rsp);
+}
+
 
 int
-afr_writev_wind (call_frame_t *frame, xlator_t *this)
+afr_transaction_writev_unwind (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+        call_frame_t *fop_frame = NULL;
 
-        int i = 0;
-        int call_count = -1;
+        fop_frame = afr_transaction_detach_fop_frame (frame);
+
+        if (fop_frame) {
+                afr_writev_copy_outvars (frame, fop_frame);
+                afr_writev_unwind (fop_frame, this);
+        }
+        return 0;
+}
+
+static void
+afr_writev_handle_short_writes (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t   *local = NULL;
+        afr_private_t *priv  = NULL;
+        int           i      = 0;
 
         local = frame->local;
         priv = this->private;
+        /*
+         * We already have the best case result of the writev calls staged
+         * as the return value. Any writev that returns some value less
+         * than the best case is now out of sync, so mark the fop as
+         * failed. Note that fops that have returned with errors have
+         * already been marked as failed.
+         */
+        for (i = 0; i < priv->child_count; i++) {
+                if ((!local->replies[i].valid) ||
+                    (local->replies[i].op_ret == -1))
+                        continue;
 
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
+                if (local->replies[i].op_ret < local->op_ret)
+                        afr_transaction_fop_failed(frame, this, i);
         }
+}
 
-        local->call_count = call_count;
+int
+afr_writev_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        afr_local_t *   local = NULL;
+        call_frame_t    *fop_frame = NULL;
+        int child_index = (long) cookie;
+        int call_count  = -1;
+        int ret = 0;
+        uint32_t open_fd_count = 0;
+        uint32_t write_is_append = 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_writev_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->writev,
-                                           local->fd,
-                                           local->cont.writev.vector,
-                                           local->cont.writev.count,
-                                           local->cont.writev.offset,
-                                           local->cont.writev.iobref);
-
-                        if (!--call_count)
-                                break;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                __afr_inode_write_fill (frame, this, child_index, op_ret,
+					op_errno, prebuf, postbuf, xdata);
+		if (op_ret == -1 || !xdata)
+			goto unlock;
+
+		write_is_append = 0;
+		ret = dict_get_uint32 (xdata, GLUSTERFS_WRITE_IS_APPEND,
+				       &write_is_append);
+		if (ret || !write_is_append)
+			local->append_write = _gf_false;
+
+		ret = dict_get_uint32 (xdata, GLUSTERFS_OPEN_FD_COUNT,
+				       &open_fd_count);
+		if (ret == -1)
+			goto unlock;
+		if ((open_fd_count > local->open_fd_count)) {
+			local->open_fd_count = open_fd_count;
+			local->update_open_fd_count = _gf_true;
+		}
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        call_count = afr_frame_return (frame);
+
+        if (call_count == 0) {
+		if (!local->stable_write && !local->append_write)
+			/* An appended write removes the necessity to
+			   fsync() the file. This is because self-heal
+			   has the logic to check for larger file when
+			   the xattrs are not reliably pointing at
+			   a stale file.
+			*/
+			afr_fd_report_unstable_write (this, local->fd);
+
+		__afr_inode_write_finalize (frame, this);
+
+                afr_writev_handle_short_writes (frame, this);
+
+                if (local->update_open_fd_count)
+                        afr_handle_open_fd_count (frame, this);
+
+                if (!afr_txn_nothing_failed (frame, this)) {
+                        //Don't unwind until post-op is complete
+                        local->transaction.resume (frame, this);
+                } else {
+                /*
+                 * Generally inode-write fops do transaction.unwind then
+                 * transaction.resume, but writev needs to make sure that
+                 * delayed post-op frame is placed in fdctx before unwind
+                 * happens. This prevents the race of flush doing the
+                 * changelog wakeup first in fuse thread and then this
+                 * writev placing its delayed post-op frame in fdctx.
+                 * This helps flush make sure all the delayed post-ops are
+                 * completed.
+                 */
+
+                        fop_frame = afr_transaction_detach_fop_frame (frame);
+                        afr_writev_copy_outvars (frame, fop_frame);
+                        local->transaction.resume (frame, this);
+                        afr_writev_unwind (fop_frame, this);
                 }
         }
-
         return 0;
 }
 
 
 int
-afr_writev_done (call_frame_t *frame, xlator_t *this)
+afr_writev_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
         local = frame->local;
+        priv = this->private;
 
-        iobref_unref (local->cont.writev.iobref);
-        local->cont.writev.iobref = NULL;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
+	STACK_WIND_COOKIE (frame, afr_writev_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->writev,
+			   local->fd, local->cont.writev.vector,
+			   local->cont.writev.count, local->cont.writev.offset,
+			   local->cont.writev.flags, local->cont.writev.iobref,
+			   local->xdata_req);
         return 0;
 }
 
@@ -193,35 +340,39 @@ afr_writev_done (call_frame_t *frame, xlator_t *this)
 int
 afr_do_writev (call_frame_t *frame, xlator_t *this)
 {
-        call_frame_t * transaction_frame = NULL;
-        afr_local_t *  local             = NULL;
-
-        int op_ret   = -1;
-        int op_errno = 0;
-
-        local = frame->local;
+        call_frame_t    *transaction_frame = NULL;
+        afr_local_t     *local             = NULL;
+        int             ret   = -1;
+        int             op_errno = ENOMEM;
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
-                op_errno = ENOMEM;
+        if (!transaction_frame)
                 goto out;
-        }
 
+        local = frame->local;
         transaction_frame->local = local;
-        frame->local = NULL;
+	frame->local = NULL;
 
-        local->op = GF_FOP_WRITE;
+	if (!AFR_FRAME_INIT (frame, op_errno))
+		goto out;
 
-        local->success_count      = 0;
+        local->op = GF_FOP_WRITE;
 
-        local->transaction.fop    = afr_writev_wind;
-        local->transaction.done   = afr_writev_done;
-        local->transaction.unwind = afr_writev_unwind;
+        local->transaction.wind   = afr_writev_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_transaction_writev_unwind;
 
         local->transaction.main_frame = frame;
+
         if (local->fd->flags & O_APPEND) {
+               /*
+                * Backend vfs ignores the 'offset' for append mode fd so
+                * locking just the region provided for the writev does not
+                * give consistency gurantee. The actual write may happen at a
+                * completely different range than the one provided by the
+                * offset, len in the fop. So lock the entire file.
+                */
                 local->transaction.start   = 0;
                 local->transaction.len     = 0;
         } else {
@@ -230,16 +381,18 @@ afr_do_writev (call_frame_t *frame, xlator_t *this)
                                                          local->cont.writev.count);
         }
 
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (ret < 0) {
+            op_errno = -ret;
+            goto out;
+        }
 
-        op_ret = 0;
+	return 0;
 out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (writev, frame, op_ret, op_errno, NULL, NULL);
-        }
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
 
+	AFR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -247,62 +400,67 @@ out:
 int
 afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
             struct iovec *vector, int32_t count, off_t offset,
-            struct iobref *iobref)
+            uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
+        afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+        int op_errno = ENOMEM;
 
-        int ret = -1;
+        priv = this->private;
 
-        int op_ret   = -1;
-        int op_errno = 0;
+        QUORUM_CHECK(writev,out);
 
-        uint64_t ctx;
-        afr_fd_ctx_t *fd_ctx = NULL;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        local->cont.writev.vector = iov_dup (vector, count);
+	if (!local->cont.writev.vector)
+		goto out;
+        local->cont.writev.count      = count;
+        local->cont.writev.offset     = offset;
+        local->cont.writev.flags      = flags;
+        local->cont.writev.iobref     = iobref_ref (iobref);
 
-        priv = this->private;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+	if (!local->xdata_req)
+		goto out;
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+        local->fd = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
 
-        frame->local = local;
+	if (dict_set_uint32 (local->xdata_req, GLUSTERFS_OPEN_FD_COUNT, 4)) {
+		op_errno = ENOMEM;
+		goto out;
+	}
 
-        local->cont.writev.vector     = iov_dup (vector, count);
-        local->cont.writev.count      = count;
-        local->cont.writev.offset     = offset;
-        local->cont.writev.ino        = fd->inode->ino;
-        local->cont.writev.iobref     = iobref_ref (iobref);
+	if (dict_set_uint32 (local->xdata_req, GLUSTERFS_WRITE_IS_APPEND, 4)) {
+		op_errno = ENOMEM;
+		goto out;
+	}
 
-        local->fd                = fd_ref (fd);
+	/* Set append_write to be true speculatively. If on any
+	   server it turns not be true, we unset it in the
+	   callback.
+	*/
+	local->append_write = _gf_true;
 
-        ret = fd_ctx_get (fd, this, &ctx);
-        if (ret < 0) {
-                goto out;
-        }
+	/* detect here, but set it in writev_wind_cbk *after* the unstable
+	   write is performed
+	*/
+	local->stable_write = !!((fd->flags|flags)&(O_SYNC|O_DSYNC));
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+        afr_fix_open (fd, this);
 
-        if (fd_ctx->up_count < priv->up_count) {
-                local->openfd_flush_cbk = afr_do_writev;
-                afr_openfd_flush (frame, this, fd);
-        } else {
-                afr_do_writev (frame, this);
-        }
+        afr_do_writev (frame, this);
 
-        op_ret = 0;
+	return 0;
 out:
-        if (op_ret == -1) {
-                AFR_STACK_UNWIND (writev, frame, op_ret, op_errno, NULL, NULL);
-        }
+	AFR_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
 
         return 0;
 }
@@ -320,24 +478,13 @@ afr_truncate_unwind (call_frame_t *frame, xlator_t *this)
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                local->cont.truncate.prebuf.ia_ino  = local->cont.truncate.ino;
-                local->cont.truncate.postbuf.ia_ino = local->cont.truncate.ino;
-
-                AFR_STACK_UNWIND (truncate, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.truncate.prebuf,
-                                  &local->cont.truncate.postbuf);
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (truncate, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
         return 0;
 }
 
@@ -345,181 +492,99 @@ afr_truncate_unwind (call_frame_t *frame, xlator_t *this)
 int
 afr_truncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                       struct iatt *postbuf)
+                       struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
-
-        int child_index = (long) cookie;
-        int read_child  = 0;
-        int call_count  = -1;
-        int need_unwind = 0;
-
-        local = frame->local;
-        priv  = this->private;
-
-        read_child = afr_read_child (this, local->loc.inode);
-
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
-
-                if (afr_fop_failed (op_ret, op_errno) && op_errno != EFBIG)
-                        afr_transaction_fop_failed (frame, this, child_index);
+	afr_local_t *local = NULL;
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.truncate.prebuf  = *prebuf;
-                                local->cont.truncate.postbuf = *postbuf;
-                        }
+	local = frame->local;
 
-                        if (child_index == read_child) {
-                                local->cont.truncate.prebuf  = *prebuf;
-                                local->cont.truncate.postbuf = *postbuf;
-                        }
+	if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
+		local->stable_write = _gf_false;
 
-                        local->success_count++;
-
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
-
-        return 0;
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      prebuf, postbuf, xdata);
 }
 
 
-int32_t
-afr_truncate_wind (call_frame_t *frame, xlator_t *this)
+int
+afr_truncate_wind (call_frame_t *frame, xlator_t *this, int subvol)
 {
         afr_local_t *local = NULL;
         afr_private_t *priv = NULL;
 
-        int call_count = -1;
-        int i = 0;
-
         local = frame->local;
         priv = this->private;
 
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
-
-        local->call_count = call_count;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_truncate_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->truncate,
-                                           &local->loc,
-                                           local->cont.truncate.offset);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-        return 0;
-}
-
-
-int
-afr_truncate_done (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
-
+	STACK_WIND_COOKIE (frame, afr_truncate_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->truncate,
+			   &local->loc, local->cont.truncate.offset,
+			   local->xdata_req);
         return 0;
 }
 
 
 int
 afr_truncate (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, off_t offset)
+              loc_t *loc, off_t offset, dict_t *xdata)
 {
         afr_private_t * priv  = NULL;
         afr_local_t   * local = NULL;
         call_frame_t   *transaction_frame = NULL;
-
-        int ret = -1;
-
-        int op_ret   = -1;
-        int op_errno = 0;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+	int ret = -1;
+        int op_errno = ENOMEM;
 
         priv = this->private;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
-                goto out;
-        }
-
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+        QUORUM_CHECK(truncate,out);
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
                 goto out;
-        }
 
-        transaction_frame->local = local;
-
-        local->op_ret = -1;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
         local->cont.truncate.offset  = offset;
-        local->cont.truncate.ino     = loc->inode->ino;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
 
-        local->transaction.fop    = afr_truncate_wind;
-        local->transaction.done   = afr_truncate_done;
+	local->transaction.wind   = afr_truncate_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
         local->transaction.unwind = afr_truncate_unwind;
 
         loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
+
+        local->op = GF_FOP_TRUNCATE;
 
         local->transaction.main_frame = frame;
-        local->transaction.start   = 0;
-        local->transaction.len     = offset;
+        local->transaction.start   = offset;
+        local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+	/* Set it true speculatively, will get reset in afr_truncate_wind_cbk
+	   if truncate was not a NOP */
+	local->stable_write = _gf_true;
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (truncate, frame, op_ret, op_errno, NULL, NULL);
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -537,23 +602,13 @@ afr_ftruncate_unwind (call_frame_t *frame, xlator_t *this)
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
-
-        if (main_frame) {
-                local->cont.ftruncate.prebuf.ia_ino  = local->cont.ftruncate.ino;
-                local->cont.ftruncate.postbuf.ia_ino = local->cont.ftruncate.ino;
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
-                AFR_STACK_UNWIND (ftruncate, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.ftruncate.prebuf,
-                                  &local->cont.ftruncate.postbuf);
-        }
+	AFR_STACK_UNWIND (ftruncate, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
         return 0;
 }
 
@@ -561,1034 +616,1136 @@ afr_ftruncate_unwind (call_frame_t *frame, xlator_t *this)
 int
 afr_ftruncate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf)
+                        struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
+	afr_local_t *local = NULL;
 
-        int child_index = (long) cookie;
-        int call_count  = -1;
-        int need_unwind = 0;
-        int read_child  = 0;
+	local = frame->local;
 
-        local = frame->local;
-        priv  = this->private;
+	if (op_ret == 0 && prebuf->ia_size != postbuf->ia_size)
+		local->stable_write = _gf_false;
 
-        read_child = afr_read_child (this, local->fd->inode);
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      prebuf, postbuf, xdata);
+}
 
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
+int
+afr_ftruncate_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.ftruncate.prebuf  = *prebuf;
-                                local->cont.ftruncate.postbuf = *postbuf;
-                        }
+	local = frame->local;
+	priv = this->private;
 
-                        if (child_index == read_child) {
-                                local->cont.ftruncate.prebuf  = *prebuf;
-                                local->cont.ftruncate.postbuf = *postbuf;
-                        }
+	STACK_WIND_COOKIE (frame, afr_ftruncate_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->ftruncate,
+			   local->fd, local->cont.ftruncate.offset,
+			   local->xdata_req);
+        return 0;
+}
 
-                        local->success_count++;
 
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+int
+afr_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	       dict_t *xdata)
+{
+        afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+	call_frame_t *transaction_frame = NULL;
+	int ret = -1;
+        int op_errno = ENOMEM;
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
+        priv = this->private;
 
-        call_count = afr_frame_return (frame);
+        QUORUM_CHECK(ftruncate,out);
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+	transaction_frame = copy_frame (frame);
+	if (!frame)
+		goto out;
 
-        return 0;
-}
+        local = AFR_FRAME_INIT (transaction_frame, op_errno);
+        if (!local)
+		goto out;
 
+        local->cont.ftruncate.offset  = offset;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-int
-afr_ftruncate_wind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+	if (!local->xdata_req)
+		goto out;
 
-        int call_count = -1;
-        int i = 0;
+        local->fd = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
 
-        local = frame->local;
-        priv = this->private;
+        local->op = GF_FOP_FTRUNCATE;
 
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
+	local->transaction.wind   = afr_ftruncate_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_ftruncate_unwind;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+        local->transaction.main_frame = frame;
 
-        local->call_count = call_count;
+        local->transaction.start   = local->cont.ftruncate.offset;
+        local->transaction.len     = 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_ftruncate_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->ftruncate,
-                                           local->fd, local->cont.ftruncate.offset);
-
-                        if (!--call_count)
-                                break;
-                }
+        afr_fix_open (fd, this);
+
+	/* Set it true speculatively, will get reset in afr_ftruncate_wind_cbk
+	   if truncate was not a NOP */
+	local->stable_write = _gf_true;
+
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	AFR_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+
         return 0;
 }
 
+/* }}} */
+
+/* {{{ setattr */
 
 int
-afr_ftruncate_done (call_frame_t *frame, xlator_t *this)
+afr_setattr_unwind (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t *local = NULL;
+        call_frame_t *main_frame = NULL;
 
         local = frame->local;
 
-        local->transaction.unwind (frame, this);
-
-        AFR_STACK_DESTROY (frame);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (setattr, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf,
+			  local->xdata_rsp);
         return 0;
 }
 
 
 int
-afr_do_ftruncate (call_frame_t *frame, xlator_t *this)
+afr_setattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno,
+                      struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
-        call_frame_t * transaction_frame = NULL;
-        afr_local_t *  local             = NULL;
-
-        int op_ret   = -1;
-        int op_errno = 0;
-
-        local = frame->local;
-
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
-                goto out;
-        }
-
-        transaction_frame->local = local;
-        frame->local = NULL;
-
-        local->op = GF_FOP_FTRUNCATE;
-
-        local->transaction.fop    = afr_ftruncate_wind;
-        local->transaction.done   = afr_ftruncate_done;
-        local->transaction.unwind = afr_ftruncate_unwind;
-
-        local->transaction.main_frame = frame;
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      preop, postop, xdata);
+}
 
-        local->transaction.start   = 0;
-        local->transaction.len     = local->cont.ftruncate.offset;
 
-        afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+int
+afr_setattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, NULL, NULL);
-        }
+        local = frame->local;
+        priv = this->private;
 
+	STACK_WIND_COOKIE (frame, afr_setattr_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->setattr,
+			   &local->loc, &local->cont.setattr.in_buf,
+			   local->cont.setattr.valid, local->xdata_req);
         return 0;
 }
 
 
 int
-afr_ftruncate (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, off_t offset)
+afr_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
+	     int32_t valid, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-
+        afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+        call_frame_t *transaction_frame = NULL;
         int ret = -1;
-
-        int op_ret   = -1;
-        int op_errno = 0;
-
-        uint64_t ctx;
-        afr_fd_ctx_t *fd_ctx = NULL;
-
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        int op_errno = ENOMEM;
 
         priv = this->private;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+        QUORUM_CHECK(setattr,out);
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
                 goto out;
-        }
 
-        frame->local = local;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
-        local->cont.ftruncate.offset  = offset;
-        local->cont.ftruncate.ino     = fd->inode->ino;
+        local->cont.setattr.in_buf = *buf;
+        local->cont.setattr.valid  = valid;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        local->fd = fd_ref (fd);
+	if (!local->xdata_req)
+		goto out;
 
-        ret = fd_ctx_get (fd, this, &ctx);
-        if (ret < 0) {
-                goto out;
-        }
+        local->transaction.wind   = afr_setattr_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_setattr_unwind;
+
+        loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+	local->op = GF_FOP_SETATTR;
 
-        if (fd_ctx->up_count < priv->up_count) {
-                local->openfd_flush_cbk = afr_do_ftruncate;
-                afr_openfd_flush (frame, this, fd);
-        } else {
-                afr_do_ftruncate (frame, this);
+        local->transaction.main_frame = frame;
+        local->transaction.start   = LLONG_MAX - 1;
+        local->transaction.len     = 0;
+
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
-        op_ret = 0;
+	return 0;
 out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, NULL, NULL);
-        }
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
 
+	AFR_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
-/* }}} */
-
-/* {{{ setattr */
+/* {{{ fsetattr */
 
 int
-afr_setattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_fsetattr_unwind (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t *   local = NULL;
         call_frame_t   *main_frame = NULL;
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
+
+	AFR_STACK_UNWIND (fsetattr, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
+        return 0;
+}
 
-        if (main_frame) {
-                local->cont.setattr.preop_buf.ia_ino  = local->cont.setattr.ino;
-                local->cont.setattr.postop_buf.ia_ino = local->cont.setattr.ino;
 
-                AFR_STACK_UNWIND (setattr, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.setattr.preop_buf,
-                                  &local->cont.setattr.postop_buf);
-        }
+int
+afr_fsetattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno,
+                       struct iatt *preop, struct iatt *postop, dict_t *xdata)
+{
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      preop, postop, xdata);
+}
 
+
+int
+afr_fsetattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+
+        local = frame->local;
+        priv = this->private;
+
+	STACK_WIND_COOKIE (frame, afr_fsetattr_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->fsetattr,
+			   local->fd, &local->cont.fsetattr.in_buf,
+			   local->cont.fsetattr.valid, local->xdata_req);
         return 0;
 }
 
 
 int
-afr_setattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      struct iatt *preop, struct iatt *postop)
+afr_fsetattr (call_frame_t *frame, xlator_t *this,
+              fd_t *fd, struct iatt *buf, int32_t valid, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
+        afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+        call_frame_t *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = ENOMEM;
 
-        int child_index = (long) cookie;
-        int read_child  = 0;
-        int call_count  = -1;
-        int need_unwind = 0;
+        priv = this->private;
 
-        local = frame->local;
-        priv  = this->private;
+        QUORUM_CHECK(fsetattr,out);
 
-        read_child = afr_read_child (this, local->loc.inode);
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
+                goto out;
 
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
+        local->cont.fsetattr.in_buf = *buf;
+        local->cont.fsetattr.valid  = valid;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.setattr.preop_buf  = *preop;
-                                local->cont.setattr.postop_buf = *postop;
-                        }
+	if (!local->xdata_req)
+		goto out;
 
-                        if (child_index == read_child) {
-                                local->cont.setattr.preop_buf  = *preop;
-                                local->cont.setattr.postop_buf = *postop;
-                        }
+        local->transaction.wind   = afr_fsetattr_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_fsetattr_unwind;
 
-                        local->success_count++;
+        local->fd                 = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
 
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+	local->op = GF_FOP_FSETATTR;
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
+        afr_fix_open (fd, this);
 
-        call_count = afr_frame_return (frame);
+        local->transaction.main_frame = frame;
+        local->transaction.start   = LLONG_MAX - 1;
+        local->transaction.len     = 0;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
 
-int32_t
-afr_setattr_wind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
-
-        int call_count = -1;
-        int i = 0;
-
-        local = frame->local;
-        priv = this->private;
+/* {{{ setxattr */
 
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+int
+afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
 
-        local->call_count = call_count;
+        local = frame->local;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_setattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->setattr,
-                                           &local->loc,
-                                           &local->cont.setattr.in_buf,
-                                           local->cont.setattr.valid);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (setxattr, main_frame, local->op_ret, local->op_errno,
+			  local->xdata_rsp);
         return 0;
 }
 
 
 int
-afr_setattr_done (call_frame_t *frame, xlator_t *this)
+afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
+        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      NULL, NULL, xdata);
+}
 
-        local = frame->local;
 
-        local->transaction.unwind (frame, this);
+int
+afr_setxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t      *local         = NULL;
+        afr_private_t    *priv          = NULL;
 
-        AFR_STACK_DESTROY (frame);
+        local = frame->local;
+        priv = this->private;
 
+	STACK_WIND_COOKIE (frame, afr_setxattr_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->setxattr,
+			   &local->loc, local->cont.setxattr.dict,
+			   local->cont.setxattr.flags, local->xdata_req);
         return 0;
 }
 
 
 int
-afr_setattr (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, struct iatt *buf, int32_t valid)
+afr_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+	      int32_t flags, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
+        afr_private_t  *priv              = NULL;
+        afr_local_t    *local             = NULL;
         call_frame_t   *transaction_frame = NULL;
+        int             ret               = -1;
+        int             op_errno          = EINVAL;
 
-        int ret = -1;
-
-        int op_ret   = -1;
-        int op_errno = 0;
+        GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
+                                   op_errno, out);
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
+                                   op_errno, out);
 
         priv = this->private;
 
+        QUORUM_CHECK(setxattr,out);
+
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
+        if (!transaction_frame)
                 goto out;
-        }
-
-        ALLOC_OR_GOTO (local, afr_local_t, out);
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
                 goto out;
-        }
-
-        transaction_frame->local = local;
-
-        local->op_ret = -1;
 
-        local->cont.setattr.ino     = loc->inode->ino;
+        local->cont.setxattr.dict  = dict_ref (dict);
+        local->cont.setxattr.flags = flags;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        local->cont.setattr.in_buf = *buf;
-        local->cont.setattr.valid  = valid;
+	if (!local->xdata_req)
+		goto out;
 
-        local->transaction.fop    = afr_setattr_wind;
-        local->transaction.done   = afr_setattr_done;
-        local->transaction.unwind = afr_setattr_unwind;
+        local->transaction.wind   = afr_setxattr_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_setxattr_unwind;
 
         loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
 
         local->transaction.main_frame = frame;
         local->transaction.start   = LLONG_MAX - 1;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+	local->op = GF_FOP_SETXATTR;
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (setattr, frame, op_ret, op_errno, NULL, NULL);
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
-/* {{{ fsetattr */
+/* {{{ fsetxattr */
+
 
 int
-afr_fsetattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_fsetxattr_unwind (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t *   local = NULL;
-        call_frame_t   *main_frame = NULL;
+        afr_local_t    *local         = NULL;
+        call_frame_t   *main_frame    = NULL;
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
-        if (main_frame) {
-                local->cont.fsetattr.preop_buf.ia_ino  =
-                        local->cont.fsetattr.ino;
-                local->cont.fsetattr.postop_buf.ia_ino =
-                        local->cont.fsetattr.ino;
+	AFR_STACK_UNWIND (fsetxattr, main_frame, local->op_ret, local->op_errno,
+			  local->xdata_rsp);
+        return 0;
+}
 
-                AFR_STACK_UNWIND (fsetattr, main_frame, local->op_ret,
-                                  local->op_errno,
-                                  &local->cont.fsetattr.preop_buf,
-                                  &local->cont.fsetattr.postop_buf);
-        }
 
+int
+afr_fsetxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      NULL, NULL, xdata);
+}
+
+
+int
+afr_fsetxattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t        *local       = NULL;
+        afr_private_t      *priv        = NULL;
+
+        local = frame->local;
+        priv = this->private;
+
+	STACK_WIND_COOKIE (frame, afr_fsetxattr_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->fsetxattr,
+			   local->fd, local->cont.fsetxattr.dict,
+			   local->cont.fsetxattr.flags, local->xdata_req);
         return 0;
 }
 
 
 int
-afr_fsetattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      struct iatt *preop, struct iatt *postop)
+afr_fsetxattr (call_frame_t *frame, xlator_t *this,
+               fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
+        afr_private_t    *priv              = NULL;
+        afr_local_t      *local             = NULL;
+        call_frame_t     *transaction_frame = NULL;
+        int               ret               = -1;
+        int               op_errno          = ENOMEM;
 
-        int child_index = (long) cookie;
-        int read_child  = 0;
-        int call_count  = -1;
-        int need_unwind = 0;
+        GF_IF_INTERNAL_XATTR_GOTO ("trusted.afr.*", dict,
+                                   op_errno, out);
 
-        local = frame->local;
-        priv  = this->private;
+        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.afr.*", dict,
+                                   op_errno, out);
 
-        read_child = afr_read_child (this, local->fd->inode);
+        priv = this->private;
 
-        LOCK (&frame->lock);
-        {
-                if (child_index == read_child) {
-                        local->read_child_returned = _gf_true;
-                }
+        QUORUM_CHECK(fsetxattr,out);
 
-                if (afr_fop_failed (op_ret, op_errno))
-                        afr_transaction_fop_failed (frame, this, child_index);
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
+                goto out;
 
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                                local->cont.fsetattr.preop_buf  = *preop;
-                                local->cont.fsetattr.postop_buf = *postop;
-                        }
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
-                        if (child_index == read_child) {
-                                local->cont.fsetattr.preop_buf  = *preop;
-                                local->cont.fsetattr.postop_buf = *postop;
-                        }
+        local->cont.fsetxattr.dict  = dict_ref (dict);
+        local->cont.fsetxattr.flags = flags;
 
-                        local->success_count++;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-                        if ((local->success_count >= priv->wait_count)
-                            && local->read_child_returned) {
-                                need_unwind = 1;
-                        }
-                }
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
+	if (!local->xdata_req)
+		goto out;
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
+        local->transaction.wind   = afr_fsetxattr_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_fsetxattr_unwind;
 
-        call_count = afr_frame_return (frame);
+        local->fd                 = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
+	local->op = GF_FOP_FSETXATTR;
+
+        local->transaction.main_frame = frame;
+        local->transaction.start  = LLONG_MAX - 1;
+        local->transaction.len    = 0;
+
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
         return 0;
 }
 
+/* }}} */
 
-int32_t
-afr_fsetattr_wind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
 
-        int call_count = -1;
-        int i = 0;
-
-        local = frame->local;
-        priv = this->private;
+/* {{{ removexattr */
 
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+int
+afr_removexattr_unwind (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
 
-        local->call_count = call_count;
+        local = frame->local;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_fsetattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->fsetattr,
-                                           local->fd,
-                                           &local->cont.fsetattr.in_buf,
-                                           local->cont.fsetattr.valid);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (removexattr, main_frame, local->op_ret, local->op_errno,
+			  local->xdata_rsp);
         return 0;
 }
 
 
 int
-afr_fsetattr_done (call_frame_t *frame, xlator_t *this)
+afr_removexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *local = NULL;
+        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      NULL, NULL, xdata);
+}
 
-        local = frame->local;
 
-        local->transaction.unwind (frame, this);
+int
+afr_removexattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
-        AFR_STACK_DESTROY (frame);
+	local = frame->local;
+	priv = this->private;
 
+	STACK_WIND_COOKIE (frame, afr_removexattr_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->removexattr,
+			   &local->loc, local->cont.removexattr.name,
+			   local->xdata_req);
         return 0;
 }
 
 
 int
-afr_fsetattr (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, struct iatt *buf, int32_t valid)
+afr_removexattr (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, const char *name, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-
-        int ret = -1;
+        afr_private_t   *priv              = NULL;
+        afr_local_t     *local             = NULL;
+        call_frame_t    *transaction_frame = NULL;
+        int              ret               = -1;
+        int              op_errno          = ENOMEM;
 
-        int op_ret   = -1;
-        int op_errno = 0;
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.afr.*",
+                                 name, op_errno, out);
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.afr.*",
+                                 name, op_errno, out);
 
         priv = this->private;
 
+        QUORUM_CHECK(removexattr,out);
+
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
+        if (!transaction_frame)
                 goto out;
-        }
-
-        ALLOC_OR_GOTO (local, afr_local_t, out);
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
-        transaction_frame->local = local;
+        local->cont.removexattr.name = gf_strdup (name);
 
-        local->op_ret = -1;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        local->cont.fsetattr.ino     = fd->inode->ino;
+	if (!local->xdata_req)
+		goto out;
 
-        local->cont.fsetattr.in_buf = *buf;
-        local->cont.fsetattr.valid  = valid;
+        local->transaction.wind   = afr_removexattr_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_removexattr_unwind;
 
-        local->transaction.fop    = afr_fsetattr_wind;
-        local->transaction.done   = afr_fsetattr_done;
-        local->transaction.unwind = afr_fsetattr_unwind;
+        loc_copy (&local->loc, loc);
+	local->inode = inode_ref (loc->inode);
 
-        local->fd                 = fd_ref (fd);
+	local->op = GF_FOP_REMOVEXATTR;
 
         local->transaction.main_frame = frame;
         local->transaction.start   = LLONG_MAX - 1;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
+        }
 
-        op_ret = 0;
+	return 0;
 out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (fsetattr, frame, op_ret, op_errno, NULL, NULL);
-        }
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
 
+	AFR_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
         return 0;
 }
 
-
-/* {{{ setxattr */
-
-
+/* ffremovexattr */
 int
-afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_fremovexattr_unwind (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t *   local = NULL;
         call_frame_t   *main_frame = NULL;
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (setxattr, main_frame,
-                                  local->op_ret, local->op_errno)
-        }
+	AFR_STACK_UNWIND (fremovexattr, main_frame, local->op_ret, local->op_errno,
+			  local->xdata_rsp);
         return 0;
 }
 
 
 int
-afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+afr_fremovexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      NULL, NULL, xdata);
+}
 
-        int call_count  = -1;
-        int need_unwind = 0;
+
+int
+afr_fremovexattr_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
         local = frame->local;
         priv = this->private;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->child_count) {
-                                need_unwind = 1;
-                        }
-                }
-
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
-
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
-
-        call_count = afr_frame_return (frame);
-
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
-
+	STACK_WIND_COOKIE (frame, afr_fremovexattr_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->fremovexattr,
+			   local->fd, local->cont.removexattr.name,
+			   local->xdata_req);
         return 0;
 }
 
 
 int
-afr_setxattr_wind (call_frame_t *frame, xlator_t *this)
+afr_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		  const char *name, dict_t *xdata)
 {
+	afr_private_t *priv = NULL;
         afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+        call_frame_t *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = ENOMEM;
 
-        int call_count = -1;
-        int i = 0;
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.afr.*",
+                                 name, op_errno, out);
 
-        local = frame->local;
-        priv = this->private;
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.afr.*",
+                                 name, op_errno, out);
 
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
+	priv = this->private;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+        QUORUM_CHECK(fremovexattr, out);
 
-        local->call_count = call_count;
+        transaction_frame = copy_frame (frame);
+        if (!transaction_frame)
+                goto out;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_setxattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->setxattr,
-                                           &local->loc,
-                                           local->cont.setxattr.dict,
-                                           local->cont.setxattr.flags);
-
-                        if (!--call_count)
-                                break;
-                }
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+                goto out;
+
+        local->cont.removexattr.name = gf_strdup (name);
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
+
+	if (!local->xdata_req)
+		goto out;
+
+        local->transaction.wind   = afr_fremovexattr_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_fremovexattr_unwind;
+
+        local->fd = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
+
+	local->op = GF_FOP_FREMOVEXATTR;
+
+        local->transaction.main_frame = frame;
+        local->transaction.start   = LLONG_MAX - 1;
+        local->transaction.len     = 0;
+
+        ret = afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
+
         return 0;
 }
 
 
 int
-afr_setxattr_done (call_frame_t *frame, xlator_t *this)
+afr_fallocate_unwind (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t * local = frame->local;
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
 
-        local->transaction.unwind (frame, this);
+        local = frame->local;
 
-        AFR_STACK_DESTROY (frame);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+	if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (fallocate, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
         return 0;
 }
 
+
 int
-afr_setxattr (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *dict, int32_t flags)
+afr_fallocate_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
-
-        int ret = -1;
+        return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      prebuf, postbuf, xdata);
+}
 
-        int op_ret   = -1;
-        int op_errno = 0;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
+int
+afr_fallocate_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
+        local = frame->local;
         priv = this->private;
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+	STACK_WIND_COOKIE (frame, afr_fallocate_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->fallocate,
+			   local->fd, local->cont.fallocate.mode,
+			   local->cont.fallocate.offset,
+			   local->cont.fallocate.len, local->xdata_req);
+        return 0;
+}
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+
+int
+afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+               off_t offset, size_t len, dict_t *xdata)
+{
+	afr_private_t *priv = NULL;
+        call_frame_t *transaction_frame = NULL;
+        afr_local_t *local = NULL;
+        int ret = -1;
+        int op_errno = ENOMEM;
+
+	priv = this->private;
+
+        QUORUM_CHECK(fallocate,out);
 
         transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
+        if (!transaction_frame)
                 goto out;
-        }
 
-        transaction_frame->local = local;
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
-        local->op_ret = -1;
+        local->cont.fallocate.mode = mode;
+        local->cont.fallocate.offset  = offset;
+        local->cont.fallocate.len = len;
 
-        local->cont.setxattr.dict  = dict_ref (dict);
-        local->cont.setxattr.flags = flags;
+        local->fd = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
 
-        local->transaction.fop    = afr_setxattr_wind;
-        local->transaction.done   = afr_setxattr_done;
-        local->transaction.unwind = afr_setxattr_unwind;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        loc_copy (&local->loc, loc);
+	if (!local->xdata_req)
+		goto out;
+
+        local->op = GF_FOP_FALLOCATE;
+
+        local->transaction.wind   = afr_fallocate_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_fallocate_unwind;
 
         local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
+
+        local->transaction.start   = local->cont.fallocate.offset;
         local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        afr_fix_open (fd, this);
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (setxattr, frame, op_ret, op_errno);
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
-/* }}} */
 
-/* {{{ removexattr */
+/* }}} */
 
+/* {{{ discard */
 
 int
-afr_removexattr_unwind (call_frame_t *frame, xlator_t *this)
+afr_discard_unwind (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t *   local = NULL;
         call_frame_t   *main_frame = NULL;
 
         local = frame->local;
 
-        LOCK (&frame->lock);
-        {
-                if (local->transaction.main_frame)
-                        main_frame = local->transaction.main_frame;
-                local->transaction.main_frame = NULL;
-        }
-        UNLOCK (&frame->lock);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
-        if (main_frame) {
-                AFR_STACK_UNWIND (removexattr, main_frame,
-                                  local->op_ret, local->op_errno)
-        }
+	AFR_STACK_UNWIND (discard, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
         return 0;
 }
 
 
 int
-afr_removexattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+afr_discard_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
 {
-        afr_local_t *   local = NULL;
-        afr_private_t * priv  = NULL;
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      prebuf, postbuf, xdata);
+}
 
-        int call_count  = -1;
-        int need_unwind = 0;
+
+int
+afr_discard_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
         local = frame->local;
         priv = this->private;
 
-        LOCK (&frame->lock);
-        {
-                if (op_ret != -1) {
-                        if (local->success_count == 0) {
-                                local->op_ret = op_ret;
-                        }
-                        local->success_count++;
-
-                        if (local->success_count == priv->wait_count) {
-                                need_unwind = 1;
-                        }
-                }
+	STACK_WIND_COOKIE (frame, afr_discard_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->discard,
+			   local->fd, local->cont.discard.offset,
+			   local->cont.discard.len, local->xdata_req);
+        return 0;
+}
 
-                local->op_errno = op_errno;
-        }
-        UNLOCK (&frame->lock);
 
-        if (need_unwind)
-                local->transaction.unwind (frame, this);
+int
+afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+        afr_local_t *local = NULL;
+        call_frame_t *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = ENOMEM;
 
-        call_count = afr_frame_return (frame);
+        priv = this->private;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-        }
+        QUORUM_CHECK(discard, out);
 
-        return 0;
-}
+	transaction_frame = copy_frame (frame);
+	if (!transaction_frame)
+		goto out;
 
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
-int32_t
-afr_removexattr_wind (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-        afr_private_t *priv = NULL;
+        local->cont.discard.offset  = offset;
+        local->cont.discard.len = len;
 
-        int call_count = -1;
-        int i = 0;
+        local->fd = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
 
-        local = frame->local;
-        priv = this->private;
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
+	if (!local->xdata_req)
+		goto out;
 
-        if (call_count == 0) {
-                local->transaction.resume (frame, this);
-                return 0;
-        }
+        local->op = GF_FOP_DISCARD;
 
-        local->call_count = call_count;
+        local->transaction.wind   = afr_discard_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_discard_unwind;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->child_up[i]) {
-                        STACK_WIND_COOKIE (frame, afr_removexattr_wind_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->removexattr,
-                                           &local->loc,
-                                           local->cont.removexattr.name);
-
-                        if (!--call_count)
-                                break;
-                }
+        local->transaction.main_frame = frame;
+
+        local->transaction.start   = local->cont.discard.offset;
+        local->transaction.len     = 0;
+
+        afr_fix_open (fd, this);
+
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
 
+/* {{{ zerofill */
+
 int
-afr_removexattr_done (call_frame_t *frame, xlator_t *this)
+afr_zerofill_unwind (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t * local = frame->local;
+        afr_local_t *   local = NULL;
+        call_frame_t   *main_frame = NULL;
 
-        local->transaction.unwind (frame, this);
+        local = frame->local;
 
-        AFR_STACK_DESTROY (frame);
+	main_frame = afr_transaction_detach_fop_frame (frame);
+        if (!main_frame)
+		return 0;
 
+	AFR_STACK_UNWIND (discard, main_frame, local->op_ret, local->op_errno,
+			  &local->cont.inode_wfop.prebuf,
+			  &local->cont.inode_wfop.postbuf, local->xdata_rsp);
         return 0;
 }
 
 
 int
-afr_removexattr (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, const char *name)
+afr_zerofill_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
 {
-        afr_private_t * priv  = NULL;
-        afr_local_t   * local = NULL;
-        call_frame_t   *transaction_frame = NULL;
+	return __afr_inode_write_cbk (frame, cookie, this, op_ret, op_errno,
+				      prebuf, postbuf, xdata);
+}
 
-        int ret = -1;
 
-        int op_ret   = -1;
-        int op_errno = 0;
+int
+afr_zerofill_wind (call_frame_t *frame, xlator_t *this, int subvol)
+{
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
 
-        VALIDATE_OR_GOTO (frame, out);
-        VALIDATE_OR_GOTO (this, out);
-        VALIDATE_OR_GOTO (this->private, out);
-        VALIDATE_OR_GOTO (loc, out);
+        local = frame->local;
+        priv = this->private;
+
+	STACK_WIND_COOKIE (frame, afr_zerofill_wind_cbk, (void *) (long) subvol,
+			   priv->children[subvol],
+			   priv->children[subvol]->fops->zerofill,
+			   local->fd, local->cont.zerofill.offset,
+			   local->cont.zerofill.len, local->xdata_req);
+        return 0;
+}
+
+int
+afr_zerofill (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata)
+{
+        afr_private_t *priv  = NULL;
+        afr_local_t *local = NULL;
+        call_frame_t *transaction_frame = NULL;
+        int ret = -1;
+        int op_errno = ENOMEM;
 
         priv = this->private;
 
-        transaction_frame = copy_frame (frame);
-        if (!transaction_frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
-                goto out;
-        }
+        QUORUM_CHECK(discard, out);
 
-        ALLOC_OR_GOTO (local, afr_local_t, out);
+	transaction_frame = copy_frame (frame);
+	if (!transaction_frame)
+		goto out;
 
-        ret = AFR_LOCAL_INIT (local, priv);
-        if (ret < 0) {
-                op_errno = -ret;
-                goto out;
-        }
+	local = AFR_FRAME_INIT (transaction_frame, op_errno);
+	if (!local)
+		goto out;
 
-        transaction_frame->local = local;
+        local->cont.zerofill.offset  = offset;
+        local->cont.zerofill.len = len;
 
-        local->op_ret = -1;
+        local->fd = fd_ref (fd);
+	local->inode = inode_ref (fd->inode);
 
-        local->cont.removexattr.name = gf_strdup (name);
+	if (xdata)
+		local->xdata_req = dict_copy_with_ref (xdata, NULL);
+	else
+		local->xdata_req = dict_new ();
 
-        local->transaction.fop    = afr_removexattr_wind;
-        local->transaction.done   = afr_removexattr_done;
-        local->transaction.unwind = afr_removexattr_unwind;
+	if (!local->xdata_req)
+		goto out;
 
-        loc_copy (&local->loc, loc);
+        local->op = GF_FOP_ZEROFILL;
+
+        local->transaction.wind   = afr_zerofill_wind;
+        local->transaction.fop    = __afr_txn_write_fop;
+        local->transaction.done   = __afr_txn_write_done;
+        local->transaction.unwind = afr_zerofill_unwind;
 
         local->transaction.main_frame = frame;
-        local->transaction.start   = LLONG_MAX - 1;
-        local->transaction.len     = 0;
 
-        afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+        local->transaction.start   = local->cont.discard.offset;
+        local->transaction.len     = len;
 
-        op_ret = 0;
-out:
-        if (op_ret == -1) {
-                if (transaction_frame)
-                        AFR_STACK_DESTROY (transaction_frame);
-                AFR_STACK_UNWIND (removexattr, frame, op_ret, op_errno);
+        afr_fix_open (fd, this);
+
+        ret = afr_transaction (transaction_frame, this, AFR_DATA_TRANSACTION);
+        if (ret < 0) {
+		op_errno = -ret;
+		goto out;
         }
 
+	return 0;
+out:
+	if (transaction_frame)
+		AFR_STACK_DESTROY (transaction_frame);
+
+	AFR_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
+
+/* }}} */
diff --git a/xlators/cluster/afr/src/afr-inode-write.h b/xlators/cluster/afr/src/afr-inode-write.h
index 1c6b87a97..7b1fc5528 100644
--- a/xlators/cluster/afr/src/afr-inode-write.h
+++ b/xlators/cluster/afr/src/afr-inode-write.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __INODE_WRITE_H__
@@ -22,51 +13,70 @@
 
 int32_t
 afr_chmod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode);
+	   loc_t *loc, mode_t mode, dict_t *xdata);
 
 int32_t
 afr_chown (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, uid_t uid, gid_t gid);
+	   loc_t *loc, uid_t uid, gid_t gid, dict_t *xdata);
 
 int
 afr_fchown (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, uid_t uid, gid_t gid);
+	    fd_t *fd, uid_t uid, gid_t gid, dict_t *xdata);
 
 int32_t
 afr_fchmod (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, mode_t mode);
+	    fd_t *fd, mode_t mode, dict_t *xdata);
 
 int32_t
-afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, 
+afr_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 	    struct iovec *vector, int32_t count, off_t offset,
-            struct iobref *iobref);
+            uint32_t flags, struct iobref *iobref, dict_t *xdata);
 
 int32_t
 afr_truncate (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, off_t offset);
+	      loc_t *loc, off_t offset, dict_t *xdata);
 
 int32_t
 afr_ftruncate (call_frame_t *frame, xlator_t *this,
-	       fd_t *fd, off_t offset);
+	       fd_t *fd, off_t offset, dict_t *xdata);
 
 int32_t
 afr_utimens (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, struct timespec tv[2]);
+	     loc_t *loc, struct timespec tv[2], dict_t *xdata);
 
 int
 afr_setattr (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, struct iatt *buf, int32_t valid);
+             loc_t *loc, struct iatt *buf, int32_t valid, dict_t *xdata);
 
 int
 afr_fsetattr (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, struct iatt *buf, int32_t valid);
+              fd_t *fd, struct iatt *buf, int32_t valid, dict_t *xdata);
 
 int32_t
 afr_setxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, dict_t *dict, int32_t flags);
+              loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata);
+
+int32_t
+afr_fsetxattr (call_frame_t *frame, xlator_t *this,
+               fd_t *fd, dict_t *dict, int32_t flags, dict_t *xdata);
 
 int32_t
 afr_removexattr (call_frame_t *frame, xlator_t *this,
-		 loc_t *loc, const char *name);
+		 loc_t *loc, const char *name, dict_t *xdata);
+
+int32_t
+afr_fremovexattr (call_frame_t *frame, xlator_t *this,
+                  fd_t *fd, const char *name, dict_t *xdata);
 
+int
+afr_discard (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             size_t len, dict_t *xdata);
+
+int
+afr_fallocate (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+               off_t offset, size_t len, dict_t *xdata);
+
+int
+afr_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+             off_t len, dict_t *xdata);
 #endif /* __INODE_WRITE_H__ */
diff --git a/xlators/cluster/afr/src/afr-lk-common.c b/xlators/cluster/afr/src/afr-lk-common.c
index 69160d72f..a2a758f35 100644
--- a/xlators/cluster/afr/src/afr-lk-common.c
+++ b/xlators/cluster/afr/src/afr-lk-common.c
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2007-2009 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "dict.h"
@@ -31,8 +22,69 @@
 #define LOCKED_YES      0x1        /* for DATA, METADATA, ENTRY and higher_path */
 #define LOCKED_LOWER    0x2        /* for lower path */
 
+#define AFR_TRACE_INODELK_IN(frame, this, params ...)           \
+        do {                                                    \
+                afr_private_t *_priv = this->private;           \
+                if (!_priv->inodelk_trace)                      \
+                        break;                                  \
+                afr_trace_inodelk_in (frame, this, params);     \
+        } while (0);
+
+#define AFR_TRACE_INODELK_OUT(frame, this, params ...)          \
+        do {                                                    \
+                afr_private_t *_priv = this->private;           \
+                if (!_priv->inodelk_trace)                      \
+                        break;                                  \
+                afr_trace_inodelk_out (frame, this, params);    \
+        } while (0);
+
+#define AFR_TRACE_ENTRYLK_IN(frame, this, params ...)           \
+        do {                                                    \
+                afr_private_t *_priv = this->private;           \
+                if (!_priv->entrylk_trace)                      \
+                        break;                                  \
+                afr_trace_entrylk_in (frame, this, params);     \
+        } while (0);
+
+#define AFR_TRACE_ENTRYLK_OUT(frame, this, params ...)          \
+        do {                                                    \
+                afr_private_t *_priv = this->private;           \
+                if (!_priv->entrylk_trace)                      \
+                        break;                                  \
+                afr_trace_entrylk_out (frame, this, params);    \
+        } while (0);
+
 int
-afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index);
+afr_entry_lockee_cmp (const void *l1, const void *l2)
+{
+        const afr_entry_lockee_t       *r1 = l1;
+        const afr_entry_lockee_t       *r2 = l2;
+        int                            ret = 0;
+        uuid_t                         gfid1 = {0};
+        uuid_t                         gfid2 = {0};
+
+        loc_gfid ((loc_t*)&r1->loc, gfid1);
+        loc_gfid ((loc_t*)&r2->loc, gfid2);
+        ret = uuid_compare (gfid1, gfid2);
+        /*Entrylks with NULL basename are the 'smallest'*/
+        if (ret == 0) {
+                if (!r1->basename)
+                        return -1;
+                if (!r2->basename)
+                        return 1;
+                ret = strcmp (r1->basename, r2->basename);
+        }
+
+        if (ret <= 0)
+                return -1;
+        else
+                return 1;
+}
+
+int afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index);
+
+static int
+afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this);
 
 static uint64_t afr_lock_number = 1;
 
@@ -57,14 +109,13 @@ afr_set_lock_number (call_frame_t *frame, xlator_t *this)
 }
 
 void
-afr_set_lk_owner (call_frame_t *frame, xlator_t *this)
+afr_set_lk_owner (call_frame_t *frame, xlator_t *this, void *lk_owner)
 {
-        if (!frame->root->lk_owner) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Setting lk-owner=%llu",
-                        (unsigned long long) (unsigned long)frame->root);
-                frame->root->lk_owner = (uint64_t) (unsigned long)frame->root;
-        }
+        gf_log (this->name, GF_LOG_TRACE,
+                "Setting lk-owner=%llu",
+                (unsigned long long) (unsigned long)lk_owner);
+
+        set_lk_owner_from_ptr (&frame->root->lk_owner, lk_owner);
 }
 
 static int
@@ -90,29 +141,19 @@ is_afr_lock_selfheal (afr_local_t *local)
 }
 
 int32_t
-internal_lock_count (call_frame_t *frame, xlator_t *this,
-                     afr_fd_ctx_t *fd_ctx)
+internal_lock_count (call_frame_t *frame, xlator_t *this)
 {
         afr_local_t   *local = NULL;
         afr_private_t *priv  = NULL;
-
         int32_t call_count = 0;
         int i = 0;
 
         local = frame->local;
         priv  = this->private;
 
-        if (fd_ctx) {
-                GF_ASSERT (local->fd);
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i] && fd_ctx->opened_on[i])
-                                ++call_count;
-                }
-        } else {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i])
-                                ++call_count;
-                }
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i])
+                        ++call_count;
         }
 
         return call_count;
@@ -120,7 +161,7 @@ internal_lock_count (call_frame_t *frame, xlator_t *this,
 
 static void
 afr_print_inodelk (char *str, int size, int cmd,
-                   struct gf_flock *flock, uint64_t owner)
+                   struct gf_flock *flock, gf_lkowner_t *owner)
 {
         char *cmd_str = NULL;
         char *type_str = NULL;
@@ -168,11 +209,11 @@ afr_print_inodelk (char *str, int size, int cmd,
         }
 
         snprintf (str, size, "lock=INODELK, cmd=%s, type=%s, "
-                  "start=%llu, len=%llu, pid=%llu, lk-owner=%llu",
+                  "start=%llu, len=%llu, pid=%llu, lk-owner=%s",
                   cmd_str, type_str, (unsigned long long) flock->l_start,
                   (unsigned long long) flock->l_len,
                   (unsigned long long) flock->l_pid,
-                  (unsigned long long) owner);
+                  lkowner_utoa (owner));
 
 }
 
@@ -188,11 +229,11 @@ afr_print_lockee (char *str, int size, loc_t *loc, fd_t *fd,
 
 void
 afr_print_entrylk (char *str, int size, const char *basename,
-                   uint64_t owner)
+                   gf_lkowner_t *owner)
 {
-        snprintf (str, size, "Basename=%s, lk-owner=%llu",
+        snprintf (str, size, "Basename=%s, lk-owner=%s",
                   basename ? basename : "<nul>",
-                  (unsigned long long)owner);
+                  lkowner_utoa (owner));
 }
 
 static void
@@ -246,27 +287,20 @@ afr_set_lock_call_type (afr_lock_call_type_t lock_call_type,
 }
 
 static void
-afr_trace_inodelk_out (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
+afr_trace_inodelk_out (call_frame_t *frame, xlator_t *this,
+                       afr_lock_call_type_t lock_call_type,
                        afr_lock_op_type_t lk_op_type, struct gf_flock *flock,
                        int op_ret, int op_errno, int32_t child_index)
 {
-        xlator_t            *this     = NULL;
         afr_internal_lock_t *int_lock = NULL;
         afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
 
         char lockee[256];
         char lock_call_type_str[256];
         char verdict[16];
 
-        this     = THIS;
         local    = frame->local;
         int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        if (!priv->inodelk_trace) {
-                return;
-        }
 
         afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
 
@@ -274,45 +308,37 @@ afr_trace_inodelk_out (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
 
         afr_print_verdict (op_ret, op_errno, verdict);
 
-        gf_log (this->name, GF_LOG_NORMAL,
-                "[%s %s] [%s] Lockee={%s} Number={%llu}",
+        gf_log (this->name, GF_LOG_INFO,
+                "[%s %s] [%s] lk-owner=%s Lockee={%s} Number={%llu}",
                 lock_call_type_str,
                 lk_op_type == AFR_LOCK_OP ? "LOCK REPLY" : "UNLOCK REPLY",
-                verdict,
-                lockee,
+                verdict, lkowner_utoa (&frame->root->lk_owner), lockee,
                 (unsigned long long) int_lock->lock_number);
 
 }
 
 static void
-afr_trace_inodelk_in (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
+afr_trace_inodelk_in (call_frame_t *frame, xlator_t *this,
+                      afr_lock_call_type_t lock_call_type,
                       afr_lock_op_type_t lk_op_type, struct gf_flock *flock,
                       int32_t cmd, int32_t child_index)
 {
-        xlator_t            *this     = NULL;
         afr_local_t         *local    = NULL;
         afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
 
         char lock[256];
         char lockee[256];
         char lock_call_type_str[256];
 
-        this     = THIS;
         local    = frame->local;
         int_lock = &local->internal_lock;
-        priv     = this->private;
 
-        if (!priv->inodelk_trace) {
-                return;
-        }
-
-        afr_print_inodelk (lock, 256, cmd, flock, frame->root->lk_owner);
+        afr_print_inodelk (lock, 256, cmd, flock, &frame->root->lk_owner);
         afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
 
         afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
 
-        gf_log (this->name, GF_LOG_NORMAL,
+        gf_log (this->name, GF_LOG_INFO,
                 "[%s %s] Lock={%s} Lockee={%s} Number={%llu}",
                 lock_call_type_str,
                 lk_op_type == AFR_LOCK_OP ? "LOCK REQUEST" : "UNLOCK REQUEST",
@@ -322,20 +348,21 @@ afr_trace_inodelk_in (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
 }
 
 static void
-afr_trace_entrylk_in (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
+afr_trace_entrylk_in (call_frame_t *frame, xlator_t *this,
+                      afr_lock_call_type_t lock_call_type,
                       afr_lock_op_type_t lk_op_type, const char *basename,
-                      int32_t child_index)
+                      int32_t cookie)
 {
-        xlator_t            *this     = NULL;
         afr_local_t         *local    = NULL;
         afr_internal_lock_t *int_lock = NULL;
         afr_private_t       *priv     = NULL;
+        int                 child_index = 0;
+        int                 lockee_no = 0;
 
         char lock[256];
         char lockee[256];
         char lock_call_type_str[256];
 
-        this     = THIS;
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
@@ -343,36 +370,41 @@ afr_trace_entrylk_in (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
         if (!priv->entrylk_trace) {
                 return;
         }
+        lockee_no = cookie / priv->child_count;
+        child_index = cookie % priv->child_count;
 
-        afr_print_entrylk (lock, 256, basename, frame->root->lk_owner);
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
+        afr_print_entrylk (lock, 256, basename, &frame->root->lk_owner);
+        afr_print_lockee (lockee, 256, &int_lock->lockee[lockee_no].loc, local->fd,
+                          child_index);
 
         afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
 
-        gf_log (this->name, GF_LOG_NORMAL,
-                "[%s %s] Lock={%s} Lockee={%s} Number={%llu}",
+        gf_log (this->name, GF_LOG_INFO,
+                "[%s %s] Lock={%s} Lockee={%s} Number={%llu}, Cookie={%d}",
                 lock_call_type_str,
                 lk_op_type == AFR_LOCK_OP ? "LOCK REQUEST" : "UNLOCK REQUEST",
                 lock, lockee,
-                (unsigned long long) int_lock->lock_number);
+                (unsigned long long) int_lock->lock_number,
+                cookie);
 }
 
 static void
-afr_trace_entrylk_out (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
-                       afr_lock_op_type_t lk_op_type, const char *basename, int op_ret,
-                       int op_errno, int32_t child_index)
+afr_trace_entrylk_out (call_frame_t *frame, xlator_t *this,
+                       afr_lock_call_type_t lock_call_type,
+                       afr_lock_op_type_t lk_op_type, const char *basename,
+                       int op_ret, int op_errno, int32_t cookie)
 {
-        xlator_t            *this     = NULL;
         afr_internal_lock_t *int_lock = NULL;
         afr_local_t         *local    = NULL;
         afr_private_t       *priv     = NULL;
+        int                 lockee_no = 0;
+        int                 child_index = 0;
 
         char lock[256];
         char lockee[256];
         char lock_call_type_str[256];
         char verdict[16];
 
-        this     = THIS;
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
@@ -380,20 +412,25 @@ afr_trace_entrylk_out (call_frame_t *frame, afr_lock_call_type_t lock_call_type,
         if (!priv->entrylk_trace) {
                 return;
         }
+        lockee_no = cookie / priv->child_count;
+        child_index = cookie % priv->child_count;
 
-        afr_print_lockee (lockee, 256, &local->loc, local->fd, child_index);
+        afr_print_entrylk (lock, 256, basename, &frame->root->lk_owner);
+        afr_print_lockee (lockee, 256, &int_lock->lockee[lockee_no].loc, local->fd,
+                          child_index);
 
         afr_set_lock_call_type (lock_call_type, lock_call_type_str, int_lock);
 
         afr_print_verdict (op_ret, op_errno, verdict);
 
-        gf_log (this->name, GF_LOG_NORMAL,
-                "[%s %s] [%s] Lock={%s} Lockee={%s} Number={%llu}",
+        gf_log (this->name, GF_LOG_INFO,
+                "[%s %s] [%s] Lock={%s} Lockee={%s} Number={%llu} Cookie={%d}",
                 lock_call_type_str,
                 lk_op_type == AFR_LOCK_OP ? "LOCK REPLY" : "UNLOCK REPLY",
                 verdict,
                 lock, lockee,
-                (unsigned long long) int_lock->lock_number);
+                (unsigned long long) int_lock->lock_number,
+                cookie);
 
 }
 
@@ -431,8 +468,8 @@ is_afr_lock_transaction (afr_local_t *local)
         int ret = 0;
 
         switch (local->transaction.type) {
-	case AFR_DATA_TRANSACTION:
-	case AFR_METADATA_TRANSACTION:
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
                 ret = 1;
                 break;
 
@@ -446,6 +483,47 @@ is_afr_lock_transaction (afr_local_t *local)
         return ret;
 }
 
+int
+afr_init_entry_lockee (afr_entry_lockee_t *lockee, afr_local_t *local,
+                       loc_t *loc, char *basename, int child_count)
+{
+        int     ret     = -1;
+
+        loc_copy (&lockee->loc, loc);
+        lockee->basename        = (basename)? gf_strdup (basename): NULL;
+        if (basename && !lockee->basename)
+                goto out;
+
+        lockee->locked_count    = 0;
+        lockee->locked_nodes    = GF_CALLOC (child_count,
+                                             sizeof (*lockee->locked_nodes),
+                                             gf_afr_mt_afr_node_character);
+
+        if (!lockee->locked_nodes)
+                goto out;
+
+        ret = 0;
+out:
+        return ret;
+
+}
+
+void
+afr_entry_lockee_cleanup (afr_internal_lock_t *int_lock)
+{
+        int     i   = 0;
+
+        for (i = 0; i < int_lock->lockee_count; i++) {
+                loc_wipe (&int_lock->lockee[i].loc);
+                if (int_lock->lockee[i].basename)
+                        GF_FREE (int_lock->lockee[i].basename);
+                if (int_lock->lockee[i].locked_nodes)
+                       GF_FREE (int_lock->lockee[i].locked_nodes);
+        }
+
+        return;
+}
+
 static int
 initialize_entrylk_variables (call_frame_t *frame, xlator_t *this)
 {
@@ -463,8 +541,13 @@ initialize_entrylk_variables (call_frame_t *frame, xlator_t *this)
         int_lock->lock_op_ret        = -1;
         int_lock->lock_op_errno      = 0;
 
-        for (i = 0; i < priv->child_count; i++) {
-                int_lock->entry_locked_nodes[i] = 0;
+        for (i = 0; i < AFR_LOCKEE_COUNT_MAX; i++) {
+                if (!int_lock->lockee[i].locked_nodes)
+                        break;
+                int_lock->lockee[i].locked_count = 0;
+                memset (int_lock->lockee[i].locked_nodes, 0,
+                        sizeof (*int_lock->lockee[i].locked_nodes) *
+                        priv->child_count);
         }
 
         return 0;
@@ -476,45 +559,44 @@ initialize_inodelk_variables (call_frame_t *frame, xlator_t *this)
         afr_local_t         *local    = NULL;
         afr_internal_lock_t *int_lock = NULL;
         afr_private_t       *priv     = NULL;
-
-        int i = 0;
+        afr_inodelk_t       *inodelk  = NULL;
 
         priv     = this->private;
         local    = frame->local;
         int_lock = &local->internal_lock;
 
-        int_lock->inodelk_lock_count = 0;
-        int_lock->lock_op_ret        = -1;
-        int_lock->lock_op_errno      = 0;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
 
-        for (i = 0; i < priv->child_count; i++) {
-                int_lock->inode_locked_nodes[i] = 0;
-        }
+        inodelk->lock_count    = 0;
+        int_lock->lk_attempted_count = 0;
+        int_lock->lock_op_ret   = -1;
+        int_lock->lock_op_errno = 0;
+
+        memset (inodelk->locked_nodes, 0,
+                sizeof (*inodelk->locked_nodes) * priv->child_count);
+        memset (int_lock->locked_nodes, 0,
+                sizeof (*int_lock->locked_nodes) * priv->child_count);
 
         return 0;
 }
 
-loc_t *
-lower_path (loc_t *l1, const char *b1, loc_t *l2, const char *b2)
+int
+afr_lockee_locked_nodes_count (afr_internal_lock_t *int_lock)
 {
-	int ret = 0;
-
-	ret = strcmp (l1->path, l2->path);
+        int call_count  = 0;
+        int i           = 0;
 
-	if (ret == 0)
-		ret = strcmp (b1, b2);
+        for (i = 0; i < int_lock->lockee_count; i++)
+                call_count += int_lock->lockee[i].locked_count;
 
-	if (ret <= 0)
-		return l1;
-	else
-		return l2;
+        return call_count;
 }
 
 int
 afr_locked_nodes_count (unsigned char *locked_nodes, int child_count)
 
 {
-        int i;
+        int i = 0;
         int call_count = 0;
 
         for (i = 0; i < child_count; i++) {
@@ -528,44 +610,63 @@ afr_locked_nodes_count (unsigned char *locked_nodes, int child_count)
 /* FIXME: What if UNLOCK fails */
 static int32_t
 afr_unlock_common_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	afr_local_t         *local    = NULL;
+        afr_local_t         *local    = NULL;
         afr_internal_lock_t *int_lock = NULL;
-	int call_count = 0;
+        int call_count = 0;
 
-	local    = frame->local;
+        local    = frame->local;
         int_lock = &local->internal_lock;
 
-	LOCK (&frame->lock);
-	{
-		call_count = --int_lock->lk_call_count;
-	}
-	UNLOCK (&frame->lock);
+        LOCK (&frame->lock);
+        {
+                call_count = --int_lock->lk_call_count;
+        }
+        UNLOCK (&frame->lock);
 
-	if (call_count == 0) {
+        if (call_count == 0) {
                 gf_log (this->name, GF_LOG_TRACE,
                         "All internal locks unlocked");
                 int_lock->lock_cbk (frame, this);
         }
 
-	return 0;
+        return 0;
 }
 
 static int32_t
 afr_unlock_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_trace_inodelk_out (frame, AFR_INODELK_TRANSACTION,
+        afr_local_t         *local = NULL;
+        afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk = NULL;
+        int32_t             child_index = (long)cookie;
+        afr_private_t       *priv = NULL;
+
+        local = frame->local;
+        int_lock = &local->internal_lock;
+
+        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_TRANSACTION,
                                AFR_UNLOCK_OP, NULL, op_ret,
-                               op_errno, (long) cookie);
+                               op_errno, child_index);
+
+        priv = this->private;
 
         if (op_ret < 0 && op_errno != ENOTCONN && op_errno != EBADFD) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Unlock failed for some reason");
+                gf_log (this->name, GF_LOG_INFO, "%s: unlock failed on subvolume %s "
+                        "with lock owner %s", local->loc.path,
+                        priv->children[child_index]->name,
+                        lkowner_utoa (&frame->root->lk_owner));
         }
 
-        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno);
+
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+        inodelk->locked_nodes[child_index] &= LOCKED_NO;
+        if (local->transaction.eager_lock)
+                local->transaction.eager_lock[child_index] = 0;
+
+        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno, xdata);
 
         return 0;
 
@@ -575,22 +676,30 @@ static int
 afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
         afr_private_t       *priv     = NULL;
-
-        struct gf_flock flock;
+        struct gf_flock flock = {0,};
+        struct gf_flock full_flock = {0,};
+        struct gf_flock *flock_use = NULL;
         int call_count = 0;
         int i = 0;
+        int piggyback = 0;
+        afr_fd_ctx_t        *fd_ctx      = NULL;
+
 
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
 
-        flock.l_start = int_lock->lk_flock.l_start;
-        flock.l_len   = int_lock->lk_flock.l_len;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+
+        flock.l_start = inodelk->flock.l_start;
+        flock.l_len   = inodelk->flock.l_len;
         flock.l_type  = F_UNLCK;
 
-        call_count = afr_locked_nodes_count (int_lock->inode_locked_nodes,
+        full_flock.l_type = F_UNLCK;
+        call_count = afr_locked_nodes_count (inodelk->locked_nodes,
                                              priv->child_count);
 
         int_lock->lk_call_count = call_count;
@@ -602,55 +711,107 @@ afr_unlock_inodelk (call_frame_t *frame, xlator_t *this)
                 goto out;
         }
 
+        if (local->fd)
+                fd_ctx = afr_fd_ctx_get (local->fd, this);
+
         for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->inode_locked_nodes[i] & LOCKED_YES) {
-                        if (local->fd) {
-                                afr_trace_inodelk_in (frame, AFR_INODELK_TRANSACTION,
-                                                      AFR_UNLOCK_OP, &flock, F_SETLK, i);
-
-                                STACK_WIND_COOKIE (frame, afr_unlock_inodelk_cbk,
-                                                   (void *) (long)i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->finodelk,
-                                                   this->name, local->fd,
-                                                   F_SETLK, &flock);
+                if ((inodelk->locked_nodes[i] & LOCKED_YES) != LOCKED_YES)
+                        continue;
 
-                                if (!--call_count)
-                                        break;
+                if (local->fd) {
+                        flock_use = &flock;
+                        if (!local->transaction.eager_lock[i]) {
+                                goto wind;
+                        }
 
-                        } else {
-                                afr_trace_inodelk_in (frame, AFR_INODELK_TRANSACTION,
-                                                      AFR_UNLOCK_OP, &flock, F_SETLK, i);
+                        piggyback = 0;
 
-                                STACK_WIND_COOKIE (frame, afr_unlock_inodelk_cbk,
-                                                   (void *) (long)i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->inodelk,
-                                                   this->name, &local->loc,
-                                                   F_SETLK, &flock);
+                        LOCK (&local->fd->lock);
+                        {
+                                if (fd_ctx->lock_piggyback[i]) {
+                                        fd_ctx->lock_piggyback[i]--;
+                                        piggyback = 1;
+                                } else {
+                                        fd_ctx->lock_acquired[i]--;
+                                }
+                        }
+                        UNLOCK (&local->fd->lock);
 
+                        if (piggyback) {
+                                afr_unlock_inodelk_cbk (frame, (void *) (long) i,
+                                                        this, 1, 0, NULL);
                                 if (!--call_count)
                                         break;
-
+                                continue;
                         }
 
-                }
+                        flock_use = &full_flock;
+                wind:
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_TRANSACTION,
+                                              AFR_UNLOCK_OP, flock_use, F_SETLK,
+                                              i);
 
-        }
+                        STACK_WIND_COOKIE (frame, afr_unlock_inodelk_cbk,
+                                           (void *) (long)i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->finodelk,
+                                           int_lock->domain, local->fd,
+                                           F_SETLK, flock_use, NULL);
+
+                        if (!--call_count)
+                                break;
 
+                } else {
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_TRANSACTION,
+                                              AFR_UNLOCK_OP, &flock, F_SETLK, i);
+
+                        STACK_WIND_COOKIE (frame, afr_unlock_inodelk_cbk,
+                                           (void *) (long)i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->inodelk,
+                                           int_lock->domain, &local->loc,
+                                           F_SETLK, &flock, NULL);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
 out:
         return 0;
 }
 
 static int32_t
 afr_unlock_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno)
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_trace_entrylk_out (frame, AFR_ENTRYLK_TRANSACTION,
-                              AFR_UNLOCK_OP, NULL, op_ret,
-                               op_errno, (long) cookie);
+        afr_local_t         *local = NULL;
+        afr_private_t       *priv  = NULL;
+        afr_internal_lock_t *int_lock = NULL;
+        int32_t             child_index = 0;
+        int                 lockee_no   = 0;
+
+        priv = this->private;
+        lockee_no = (int)((long) cookie) / priv->child_count;
+        child_index = (int) ((long) cookie) % priv->child_count;
 
-        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno);
+        local = frame->local;
+        int_lock = &local->internal_lock;
+
+        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
+                               AFR_UNLOCK_OP,
+                               int_lock->lockee[lockee_no].basename, op_ret,
+                               op_errno, (int) ((long)cookie));
+
+        if (op_ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: unlock failed on %d, reason: %s",
+                        local->loc.path, child_index, strerror (op_errno));
+        }
+
+        int_lock->lockee[lockee_no].locked_nodes[child_index] &= LOCKED_NO;
+        afr_unlock_common_cbk (frame, cookie, this, op_ret, op_errno, NULL);
 
         return 0;
 }
@@ -658,25 +819,22 @@ afr_unlock_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 static int
 afr_unlock_entrylk (call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-
-        int call_count = 0;
-        int i = -1;
+        afr_internal_lock_t     *int_lock       = NULL;
+        afr_local_t             *local          = NULL;
+        afr_private_t           *priv           = NULL;
+        int                     call_count      = 0;
+        int                     index           = 0;
+        int                     lockee_no       = 0;
+        int                     copies          = 0;
+        int                     i               = -1;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
+        copies   = priv->child_count;
 
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
+        call_count = afr_lockee_locked_nodes_count (int_lock);
 
-        call_count = afr_locked_nodes_count (int_lock->entry_locked_nodes,
-                                             priv->child_count);
         int_lock->lk_call_count = call_count;
 
         if (!call_count){
@@ -686,18 +844,23 @@ afr_unlock_entrylk (call_frame_t *frame, xlator_t *this)
                 goto out;
         }
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->entry_locked_nodes[i] & LOCKED_YES) {
-                        afr_trace_entrylk_in (frame, AFR_ENTRYLK_NB_TRANSACTION,
-                                              AFR_UNLOCK_OP, basename, i);
+        for (i = 0; i < int_lock->lockee_count * priv->child_count; i++) {
+                lockee_no = i / copies;
+                index     = i % copies;
+                if (int_lock->lockee[lockee_no].locked_nodes[index] & LOCKED_YES) {
+                        AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
+                                              AFR_UNLOCK_OP,
+                                              int_lock->lockee[lockee_no].basename,
+                                              i);
 
                         STACK_WIND_COOKIE (frame, afr_unlock_entrylk_cbk,
                                            (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->entrylk,
-                                           this->name,
-                                           loc, basename,
-                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
+                                           priv->children[index],
+                                           priv->children[index]->fops->entrylk,
+                                           int_lock->domain,
+                                           &int_lock->lockee[lockee_no].loc,
+                                           int_lock->lockee[lockee_no].basename,
+                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL);
 
                         if (!--call_count)
                                 break;
@@ -711,155 +874,85 @@ out:
 
 static int32_t
 afr_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	      int32_t op_ret, int32_t op_errno)
-{
-        afr_internal_lock_t *int_lock = NULL;
-	afr_local_t         *local    = NULL;
-	afr_private_t       *priv     = NULL;
-
-	int done = 0;
-	int child_index = (long) cookie;
-
-	local    = frame->local;
-        int_lock = &local->internal_lock;
-	priv     = this->private;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			if (op_errno == ENOSYS) {
-				/* return ENOTSUP */
-				gf_log (this->name, GF_LOG_ERROR,
-					"subvolume does not support locking. "
-					"please load features/posix-locks xlator on server");
-				local->op_ret = op_ret;
-                                int_lock->lock_op_ret = op_ret;
-				done = 1;
-			}
-
-			local->child_up[child_index] = 0;
-			local->op_errno              = op_errno;
-                        int_lock->lock_op_errno      = op_errno;
-		}
-	}
-	UNLOCK (&frame->lock);
-
-        if ((op_ret == -1) &&
-            (op_errno == ENOSYS)) {
-                afr_unlock (frame, this);
-        } else {
-                if (op_ret == 0) {
-                        int_lock->locked_nodes[child_index]
-                                |= LOCKED_YES;
-                        int_lock->lock_count++;
-                }
-                afr_lock_blocking (frame, this, child_index + 1);
-        }
-
-	return 0;
-}
-
-static int32_t
-afr_blocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_trace_inodelk_out (frame, AFR_INODELK_TRANSACTION,
-                               AFR_LOCK_OP, NULL, op_ret,
-                               op_errno, (long) cookie);
-
-        afr_lock_cbk (frame, cookie, this, op_ret, op_errno);
-        return 0;
-
-}
-
-static int32_t
-afr_lock_lower_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_private_t       *priv        = NULL;
-        afr_local_t         *local       = NULL;
-        loc_t               *lower       = NULL;
-        loc_t               *higher      = NULL;
-        const char          *lower_name  = NULL;
-        const char          *higher_name = NULL;
-
-        int child_index = (long) cookie;
+        afr_internal_lock_t     *int_lock       = NULL;
+        afr_local_t             *local          = NULL;
+        afr_private_t           *priv           = NULL;
+        int                     cky             = (long) cookie;
+        int                     child_index     = 0;
+        int                     lockee_no       = 0;
 
         priv     = this->private;
         local    = frame->local;
         int_lock = &local->internal_lock;
 
+        child_index = ((int)cky) % priv->child_count;
+        lockee_no   = ((int)cky) / priv->child_count;
+
         LOCK (&frame->lock);
         {
                 if (op_ret == -1) {
                         if (op_errno == ENOSYS) {
                                 /* return ENOTSUP */
-
                                 gf_log (this->name, GF_LOG_ERROR,
                                         "subvolume does not support locking. "
-                                        "please load features/posix-locks xlator on server");
-
-                                local->op_ret   = op_ret;
+                                        "please load features/locks xlator on server");
+                                local->op_ret = op_ret;
+                                int_lock->lock_op_ret = op_ret;
                         }
 
-                        local->child_up[child_index] = 0;
-                        local->op_errno = op_errno;
+                        local->op_errno              = op_errno;
+                        int_lock->lock_op_errno      = op_errno;
                 }
+
+		int_lock->lk_attempted_count++;
         }
         UNLOCK (&frame->lock);
 
-        if (op_ret != 0) {
+        if ((op_ret == -1) &&
+            (op_errno == ENOSYS)) {
                 afr_unlock (frame, this);
-                goto out;
         } else {
-                int_lock->lower_locked_nodes[child_index] |= LOCKED_LOWER;
-                int_lock->lock_count++;
+                if (op_ret == 0) {
+                        if (local->transaction.type == AFR_ENTRY_TRANSACTION ||
+                            local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
+                                int_lock->lockee[lockee_no].locked_nodes[child_index] |= LOCKED_YES;
+                                int_lock->lockee[lockee_no].locked_count++;
+                                int_lock->entrylk_lock_count++;
+                        } else {
+                                int_lock->locked_nodes[child_index] |= LOCKED_YES;
+                                int_lock->lock_count++;
+                        }
+                }
+                afr_lock_blocking (frame, this, cky + 1);
         }
 
-        /* The lower path has been locked. Now lock the higher path */
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        lower_name = (lower == &local->transaction.parent_loc ?
-                      local->transaction.basename :
-                      local->transaction.new_basename);
-
-        higher = (lower == &local->transaction.parent_loc ?
-                  &local->transaction.new_parent_loc :
-                  &local->transaction.parent_loc);
-
-        higher_name = (higher == &local->transaction.parent_loc ?
-                       local->transaction.basename :
-                       local->transaction.new_basename);
-
-        afr_trace_entrylk_in (frame, AFR_ENTRYLK_TRANSACTION,
-                              AFR_LOCK_OP, higher_name, child_index);
-
+        return 0;
+}
 
-        STACK_WIND_COOKIE (frame, afr_lock_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->entrylk,
-                           this->name, higher, higher_name,
-                           ENTRYLK_LOCK, ENTRYLK_WRLCK);
+static int32_t
+afr_blocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_TRANSACTION,
+                               AFR_LOCK_OP, NULL, op_ret,
+                               op_errno, (long) cookie);
 
-out:
+        afr_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
         return 0;
+
 }
 
 static int32_t
 afr_blocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        afr_trace_entrylk_out (frame, AFR_ENTRYLK_TRANSACTION,
+        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
                                AFR_LOCK_OP, NULL, op_ret,
                                op_errno, (long)cookie);
 
-        afr_lock_cbk (frame, cookie, this, op_ret, op_errno);
+        afr_lock_cbk (frame, cookie, this, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -867,6 +960,7 @@ static int
 afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
         afr_private_t       *priv     = NULL;
 
@@ -875,20 +969,18 @@ afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this)
         int_lock = &local->internal_lock;
 
         switch (local->transaction.type) {
-	case AFR_DATA_TRANSACTION:
-	case AFR_METADATA_TRANSACTION:
-                memcpy (int_lock->inode_locked_nodes,
-                        int_lock->locked_nodes,
-                        priv->child_count);
-                int_lock->inodelk_lock_count = int_lock->lock_count;
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+                memcpy (inodelk->locked_nodes, int_lock->locked_nodes,
+                        sizeof (*inodelk->locked_nodes) * priv->child_count);
+                inodelk->lock_count = int_lock->lock_count;
                 break;
 
         case AFR_ENTRY_RENAME_TRANSACTION:
         case AFR_ENTRY_TRANSACTION:
-                memcpy (int_lock->entry_locked_nodes,
-                        int_lock->locked_nodes,
-                        priv->child_count);
-                int_lock->entrylk_lock_count = int_lock->lock_count;
+                /*entrylk_count is being used in both non-blocking and blocking
+                 * modes */
                 break;
         }
 
@@ -896,42 +988,78 @@ afr_copy_locked_nodes (call_frame_t *frame, xlator_t *this)
 
 }
 
+static inline gf_boolean_t
+afr_is_entrylk (afr_internal_lock_t *int_lock,
+                afr_transaction_type trans_type)
+{
+        gf_boolean_t is_entrylk = _gf_false;
+
+        if ((int_lock->transaction_lk_type == AFR_SELFHEAL_LK) &&
+            int_lock->selfheal_lk_type == AFR_ENTRY_SELF_HEAL_LK) {
+
+                is_entrylk = _gf_true;
+
+        } else if ((int_lock->transaction_lk_type == AFR_TRANSACTION_LK) &&
+                 (trans_type == AFR_ENTRY_TRANSACTION ||
+                  trans_type == AFR_ENTRY_RENAME_TRANSACTION)) {
+
+                is_entrylk = _gf_true;
+
+        } else {
+                is_entrylk = _gf_false;
+        }
+
+        return is_entrylk;
+}
+
+static gf_boolean_t
+_is_lock_wind_needed (afr_local_t *local, int child_index)
+{
+        if (!local->child_up[child_index])
+                return _gf_false;
+
+        return _gf_true;
+}
+
 int
-afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
+afr_lock_blocking (call_frame_t *frame, xlator_t *this, int cookie)
 {
         afr_internal_lock_t *int_lock    = NULL;
+        afr_inodelk_t       *inodelk     = NULL;
         afr_local_t         *local       = NULL;
-	afr_private_t       *priv        = NULL;
-        afr_fd_ctx_t        *fd_ctx      = NULL;
-	loc_t               *lower       = NULL;
-	loc_t               *higher      = NULL;
-	const char          *lower_name  = NULL;
-	const char          *higher_name = NULL;
-
-	struct gf_flock flock;
-        uint64_t ctx;
+        afr_private_t       *priv        = NULL;
+        struct gf_flock flock = {0,};
+        uint64_t ctx = 0;
         int ret = 0;
-
-	local    = frame->local;
-        int_lock = &local->internal_lock;
-	priv     = this->private;
-
-	flock.l_start = int_lock->lk_flock.l_start;
-	flock.l_len   = int_lock->lk_flock.l_len;
-	flock.l_type  = int_lock->lk_flock.l_type;
+        int child_index = 0;
+        int lockee_no   = 0;
+        gf_boolean_t is_entrylk = _gf_false;
+
+        local         = frame->local;
+        int_lock      = &local->internal_lock;
+        priv          = this->private;
+        child_index   = cookie % priv->child_count;
+        lockee_no     = cookie / priv->child_count;
+        is_entrylk    = afr_is_entrylk (int_lock, local->transaction.type);
+
+
+        if (!is_entrylk) {
+                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+                flock.l_start = inodelk->flock.l_start;
+                flock.l_len   = inodelk->flock.l_len;
+                flock.l_type  = inodelk->flock.l_type;
+        }
 
         if (local->fd) {
                 ret = fd_ctx_get (local->fd, this, &ctx);
 
                 if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
+                        gf_log (this->name, GF_LOG_INFO,
                                 "unable to get fd ctx for fd=%p",
                                 local->fd);
 
                         local->op_ret           = -1;
                         int_lock->lock_op_ret   = -1;
-                        local->op_errno         = EINVAL;
-                        int_lock->lock_op_errno = EINVAL;
 
                         afr_copy_locked_nodes (frame, this);
 
@@ -939,49 +1067,27 @@ afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
 
                         return 0;
                 }
-
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-                /* skip over children that or down
-                   or don't have the fd open */
-
-                while ((child_index < priv->child_count)
-                       && (!local->child_up[child_index]
-                           || !fd_ctx->opened_on[child_index]))
-
-                        child_index++;
-        } else {
-                /* skip over children that are down */
-                while ((child_index < priv->child_count)
-                       && !local->child_up[child_index])
-                        child_index++;
         }
 
-	if ((child_index == priv->child_count) &&
-	    int_lock->lock_count == 0) {
-
-		gf_log (this->name, GF_LOG_DEBUG,
-			"unable to lock on even one child");
-
-		local->op_ret           = -1;
-                int_lock->lock_op_ret   = -1;
-		local->op_errno         = EAGAIN;
-                int_lock->lock_op_errno = EAGAIN;
+        if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
+                if ((is_entrylk && int_lock->entrylk_lock_count == 0) ||
+                    (!is_entrylk && int_lock->lock_count == 0)) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "unable to lock on even one child");
 
-                afr_copy_locked_nodes (frame, this);
-
-                afr_unlock(frame, this);
+                        local->op_ret           = -1;
+                        int_lock->lock_op_ret   = -1;
 
-		return 0;
+                        afr_copy_locked_nodes (frame, this);
 
-	}
+                        afr_unlock(frame, this);
 
-	if ((child_index == priv->child_count)
-	    || (int_lock->lock_count ==
-		afr_up_children_count (priv->child_count,
-                                       local->child_up))) {
+                        return 0;
+                }
+        }
 
-		/* we're done locking */
+        if (int_lock->lk_expected_count == int_lock->lk_attempted_count) {
+                /* we're done locking */
 
                 gf_log (this->name, GF_LOG_DEBUG,
                         "we're done locking");
@@ -990,107 +1096,85 @@ afr_lock_blocking (call_frame_t *frame, xlator_t *this, int child_index)
 
                 int_lock->lock_op_ret = 0;
                 int_lock->lock_cbk (frame, this);
-		return 0;
-	}
+                return 0;
+        }
+
+        if (!_is_lock_wind_needed (local, child_index)) {
+                afr_lock_blocking (frame, this, cookie + 1);
+                return 0;
+        }
 
-	switch (local->transaction.type) {
-	case AFR_DATA_TRANSACTION:
-	case AFR_METADATA_TRANSACTION:
+        switch (local->transaction.type) {
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
 
-		if (local->fd) {
-                        afr_trace_inodelk_in (frame, AFR_INODELK_TRANSACTION,
+                if (local->fd) {
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_TRANSACTION,
                                               AFR_LOCK_OP, &flock, F_SETLKW,
                                               child_index);
 
-			STACK_WIND_COOKIE (frame, afr_blocking_inodelk_cbk,
-					   (void *) (long) child_index,
-					   priv->children[child_index],
-					   priv->children[child_index]->fops->finodelk,
-					   this->name, local->fd,
-                                           F_SETLKW, &flock);
+                        STACK_WIND_COOKIE (frame, afr_blocking_inodelk_cbk,
+                                           (void *) (long) child_index,
+                                           priv->children[child_index],
+                                           priv->children[child_index]->fops->finodelk,
+                                           int_lock->domain, local->fd,
+                                           F_SETLKW, &flock, NULL);
 
-		} else {
-                        afr_trace_inodelk_in (frame, AFR_INODELK_TRANSACTION,
+                } else {
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_TRANSACTION,
                                               AFR_LOCK_OP, &flock, F_SETLKW,
                                               child_index);
 
-			STACK_WIND_COOKIE (frame, afr_blocking_inodelk_cbk,
-					   (void *) (long) child_index,
-					   priv->children[child_index],
-					   priv->children[child_index]->fops->inodelk,
-					   this->name, &local->loc,
-                                           F_SETLKW, &flock);
-		}
-
-		break;
-
-	case AFR_ENTRY_RENAME_TRANSACTION:
-	{
-		lower = lower_path (&local->transaction.parent_loc,
-				    local->transaction.basename,
-				    &local->transaction.new_parent_loc,
-				    local->transaction.new_basename);
-
-		lower_name = (lower == &local->transaction.parent_loc ?
-			      local->transaction.basename :
-			      local->transaction.new_basename);
-
-		higher = (lower == &local->transaction.parent_loc ?
-			  &local->transaction.new_parent_loc :
-			  &local->transaction.parent_loc);
-
-		higher_name = (higher == &local->transaction.parent_loc ?
-			       local->transaction.basename :
-			       local->transaction.new_basename);
-
-                afr_trace_entrylk_in (frame, AFR_ENTRYLK_TRANSACTION,
-                                      AFR_LOCK_OP, lower_name, child_index);
-
-
-		STACK_WIND_COOKIE (frame, afr_lock_lower_cbk,
-				   (void *) (long) child_index,
-				   priv->children[child_index],
-				   priv->children[child_index]->fops->entrylk,
-				   this->name, lower, lower_name,
-				   ENTRYLK_LOCK, ENTRYLK_WRLCK);
-
-		break;
-	}
+                        STACK_WIND_COOKIE (frame, afr_blocking_inodelk_cbk,
+                                           (void *) (long) child_index,
+                                           priv->children[child_index],
+                                           priv->children[child_index]->fops->inodelk,
+                                           int_lock->domain, &local->loc,
+                                           F_SETLKW, &flock, NULL);
+                }
 
-	case AFR_ENTRY_TRANSACTION:
-		if (local->fd) {
-                        afr_trace_entrylk_in (frame, AFR_ENTRYLK_TRANSACTION,
-                                              AFR_LOCK_OP, local->transaction.basename,
-                                              child_index);
+                break;
 
-			STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
-					   (void *) (long) child_index,
-					   priv->children[child_index],
-					   priv->children[child_index]->fops->fentrylk,
-					   this->name, local->fd,
-					   local->transaction.basename,
-					   ENTRYLK_LOCK, ENTRYLK_WRLCK);
-		} else {
-                        afr_trace_entrylk_in (frame, AFR_ENTRYLK_TRANSACTION,
+        case AFR_ENTRY_RENAME_TRANSACTION:
+        case AFR_ENTRY_TRANSACTION:
+                /*Accounting for child_index increments on 'down'
+                 *and 'fd-less' children */
+
+                if (local->fd) {
+                        AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_TRANSACTION,
+                                              AFR_LOCK_OP,
+                                              int_lock->lockee[lockee_no].basename,
+                                              cookie);
+
+                        STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
+                                           (void *) (long) cookie,
+                                           priv->children[child_index],
+                                           priv->children[child_index]->fops->fentrylk,
+                                           int_lock->domain, local->fd,
+                                           int_lock->lockee[lockee_no].basename,
+                                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+                } else {
+                        AFR_TRACE_ENTRYLK_IN (frame, this,
+                                              AFR_ENTRYLK_TRANSACTION,
                                               AFR_LOCK_OP, local->transaction.basename,
                                               child_index);
 
-			STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
-					   (void *) (long) child_index,
-					   priv->children[child_index],
-					   priv->children[child_index]->fops->entrylk,
-					   this->name,
-                                           &local->transaction.parent_loc,
-					   local->transaction.basename,
-					   ENTRYLK_LOCK, ENTRYLK_WRLCK);
-		}
-
-		break;
-	}
-
-	return 0;
+                        STACK_WIND_COOKIE (frame, afr_blocking_entrylk_cbk,
+                                           (void *) (long) cookie,
+                                           priv->children[child_index],
+                                           priv->children[child_index]->fops->entrylk,
+                                           int_lock->domain,
+                                           &int_lock->lockee[lockee_no].loc,
+                                           int_lock->lockee[lockee_no].basename,
+                                           ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+                }
 
+                break;
+        }
 
+        return 0;
 }
 
 int32_t
@@ -1099,19 +1183,24 @@ afr_blocking_lock (call_frame_t *frame, xlator_t *this)
         afr_internal_lock_t *int_lock = NULL;
         afr_local_t         *local    = NULL;
         afr_private_t       *priv     = NULL;
+        int                  up_count = 0;
 
         priv     = this->private;
         local    = frame->local;
         int_lock = &local->internal_lock;
 
         switch (local->transaction.type) {
-	case AFR_DATA_TRANSACTION:
-	case AFR_METADATA_TRANSACTION:
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
                 initialize_inodelk_variables (frame, this);
                 break;
 
         case AFR_ENTRY_RENAME_TRANSACTION:
         case AFR_ENTRY_TRANSACTION:
+                up_count = AFR_COUNT (local->child_up, priv->child_count);
+                int_lock->lk_call_count = int_lock->lk_expected_count
+                                        = (int_lock->lockee_count *
+                                           up_count);
                 initialize_entrylk_variables (frame, this);
                 break;
         }
@@ -1123,60 +1212,68 @@ afr_blocking_lock (call_frame_t *frame, xlator_t *this)
 
 static int32_t
 afr_nonblocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno)
+                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_internal_lock_t *int_lock = NULL;
-	afr_local_t         *local    = NULL;
-	afr_private_t       *priv     = NULL;
-
+        afr_local_t         *local    = NULL;
         int call_count          = 0;
-	int child_index         = (long) cookie;
+        int child_index         = (long) cookie;
+        int copies              = 0;
+        int index               = 0;
+        int lockee_no           = 0;
+        afr_private_t       *priv = NULL;
+
+        priv = this->private;
+
+        copies = priv->child_count;
+        index = child_index % copies;
+        lockee_no = child_index / copies;
 
-	local    = frame->local;
+        local    = frame->local;
         int_lock = &local->internal_lock;
-	priv     = this->private;
 
-        afr_trace_entrylk_out (frame, AFR_ENTRYLK_TRANSACTION,
-                               AFR_LOCK_OP, NULL, op_ret,
+        AFR_TRACE_ENTRYLK_OUT (frame, this, AFR_ENTRYLK_TRANSACTION,
+                               AFR_LOCK_OP,
+                               int_lock->lockee[lockee_no].basename, op_ret,
                                op_errno, (long) cookie);
 
-        LOCK (&frame->lock);
-        {
-                call_count = --int_lock->lk_call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (op_ret < 0 ) {
+	LOCK (&frame->lock);
+	{
+		if (op_ret < 0 ) {
 			if (op_errno == ENOSYS) {
-				/* return ENOTSUP */
+                        /* return ENOTSUP */
 				gf_log (this->name, GF_LOG_ERROR,
 					"subvolume does not support locking. "
-					"please load features/posix-locks xlator on server");
+					"please load features/locks xlator on server");
 				local->op_ret         = op_ret;
-                                int_lock->lock_op_ret = op_ret;
+				int_lock->lock_op_ret = op_ret;
 
-                                local->child_up[child_index] = 0;
-                                int_lock->lock_op_errno      = op_errno;
-                                local->op_errno              = op_errno;
-                        }
-        } else if (op_ret == 0) {
-                        int_lock->entry_locked_nodes[child_index]
-                                |= LOCKED_YES;
-                        int_lock->entrylk_lock_count++;
+				int_lock->lock_op_errno      = op_errno;
+				local->op_errno              = op_errno;
+			}
+		} else if (op_ret == 0) {
+			int_lock->lockee[lockee_no].locked_nodes[index] |= \
+				LOCKED_YES;
+			int_lock->lockee[lockee_no].locked_count++;
+			int_lock->entrylk_lock_count++;
+		}
+
+                call_count = --int_lock->lk_call_count;
         }
+        UNLOCK (&frame->lock);
 
         if (call_count == 0) {
                 gf_log (this->name, GF_LOG_TRACE,
                         "Last locking reply received");
-                /* all locks successfull. Proceed to call FOP */
+                /* all locks successful. Proceed to call FOP */
                 if (int_lock->entrylk_lock_count ==
-                    afr_up_children_count (priv->child_count, local->child_up)) {
+                                int_lock->lk_expected_count) {
                         gf_log (this->name, GF_LOG_TRACE,
                                 "All servers locked. Calling the cbk");
                         int_lock->lock_op_ret = 0;
                         int_lock->lock_cbk (frame, this);
                 }
-                /* Not all locks were successfull. Unlock and try locking
+                /* Not all locks were successful. Unlock and try locking
                    again, this time with serially blocking locks */
                 else {
                         gf_log (this->name, GF_LOG_TRACE,
@@ -1193,33 +1290,27 @@ afr_nonblocking_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_fd_ctx_t        *fd_ctx   = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-
-        int32_t call_count = 0;
+        afr_internal_lock_t *int_lock   = NULL;
+        afr_local_t         *local      = NULL;
+        afr_private_t       *priv       = NULL;
+        afr_fd_ctx_t        *fd_ctx     = NULL;
+        int                 copies      = 0;
+        int                 index       = 0;
+        int                 lockee_no   = 0;
+        int32_t             call_count  = 0;
         int i = 0;
-        uint64_t  ctx;
-        int ret = 0;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
 
+        copies = priv->child_count;
         initialize_entrylk_variables (frame, this);
 
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
-
         if (local->fd) {
-                ret = fd_ctx_get (local->fd, this, &ctx);
-
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
+                fd_ctx = afr_fd_ctx_get (local->fd, this);
+                if (!fd_ctx) {
+                        gf_log (this->name, GF_LOG_INFO,
                                 "unable to get fd ctx for fd=%p",
                                 local->fd);
 
@@ -1228,113 +1319,147 @@ afr_nonblocking_entrylk (call_frame_t *frame, xlator_t *this)
                         local->op_errno         = EINVAL;
                         int_lock->lock_op_errno = EINVAL;
 
+			afr_unlock (frame, this);
                         return -1;
                 }
 
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-                call_count = internal_lock_count (frame, this, fd_ctx);
+                call_count = int_lock->lockee_count * internal_lock_count (frame, this);
                 int_lock->lk_call_count = call_count;
+                int_lock->lk_expected_count = call_count;
+
+                if (!call_count) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "fd not open on any subvolumes. aborting.");
+                        afr_unlock (frame, this);
+                        goto out;
+                }
 
                 /* Send non-blocking entrylk calls only on up children
                    and where the fd has been opened */
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i] && fd_ctx->opened_on[i]) {
-                                afr_trace_entrylk_in (frame, AFR_ENTRYLK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP, basename, i);
+                for (i = 0; i < int_lock->lockee_count*priv->child_count; i++) {
+                        index = i%copies;
+                        lockee_no = i/copies;
+                        if (local->child_up[index]) {
+                                AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
+                                                      AFR_LOCK_OP,
+                                                      int_lock->lockee[lockee_no].basename,
+                                                      i);
 
                                 STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
                                                    (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fentrylk,
+                                                   priv->children[index],
+                                                   priv->children[index]->fops->fentrylk,
                                                    this->name, local->fd,
-                                                   basename,
-                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK);
+                                                   int_lock->lockee[lockee_no].basename,
+                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
+                                                   NULL);
+                                if (!--call_count)
+                                        break;
                         }
                 }
         } else {
-                GF_ASSERT (loc);
-
-                call_count = internal_lock_count (frame, this, NULL);
+                call_count = int_lock->lockee_count * internal_lock_count (frame, this);
                 int_lock->lk_call_count = call_count;
+                int_lock->lk_expected_count = call_count;
 
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i]) {
-                                afr_trace_entrylk_in (frame, AFR_ENTRYLK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP, basename, i);
+                for (i = 0; i < int_lock->lockee_count*priv->child_count; i++) {
+                        index = i%copies;
+                        lockee_no = i/copies;
+                        if (local->child_up[index]) {
+                                AFR_TRACE_ENTRYLK_IN (frame, this, AFR_ENTRYLK_NB_TRANSACTION,
+                                                      AFR_LOCK_OP,
+                                                      int_lock->lockee[lockee_no].basename,
+                                                      i);
 
                                 STACK_WIND_COOKIE (frame, afr_nonblocking_entrylk_cbk,
                                                    (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->entrylk,
-                                                   this->name, loc, basename,
-                                                   ENTRYLK_LOCK, ENTRYLK_WRLCK);
+                                                   priv->children[index],
+                                                   priv->children[index]->fops->entrylk,
+                                                   this->name, &int_lock->lockee[lockee_no].loc,
+                                                   int_lock->lockee[lockee_no].basename,
+                                                   ENTRYLK_LOCK_NB, ENTRYLK_WRLCK,
+                                                   NULL);
 
                                 if (!--call_count)
                                         break;
-
                         }
                 }
         }
-
+out:
         return 0;
 }
 
 int32_t
 afr_nonblocking_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno)
+                             int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         afr_internal_lock_t *int_lock = NULL;
-	afr_local_t         *local    = NULL;
-	afr_private_t       *priv     = NULL;
-
+        afr_inodelk_t       *inodelk  = NULL;
+        afr_local_t         *local    = NULL;
         int call_count  = 0;
-	int child_index = (long) cookie;
+        int child_index = (long) cookie;
+        afr_fd_ctx_t        *fd_ctx = NULL;
 
-	local    = frame->local;
+
+        local    = frame->local;
         int_lock = &local->internal_lock;
-	priv     = this->private;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
 
-        afr_trace_inodelk_out (frame, AFR_INODELK_NB_TRANSACTION,
+        AFR_TRACE_INODELK_OUT (frame, this, AFR_INODELK_NB_TRANSACTION,
                                AFR_LOCK_OP, NULL, op_ret,
                                op_errno, (long) cookie);
 
+	if (local->fd)
+		fd_ctx = afr_fd_ctx_get (local->fd, this);
+
         LOCK (&frame->lock);
         {
-                call_count = --int_lock->lk_call_count;
-        }
-        UNLOCK (&frame->lock);
-
-        if (op_ret < 0 ) {
+		if (op_ret < 0) {
 			if (op_errno == ENOSYS) {
 				/* return ENOTSUP */
 				gf_log (this->name, GF_LOG_ERROR,
 					"subvolume does not support locking. "
-					"please load features/posix-locks xlator on server");
+					"please load features/locks xlator on "
+					"server");
 				local->op_ret                = op_ret;
-                                int_lock->lock_op_ret        = op_ret;
-                                local->child_up[child_index] = 0;
-                                int_lock->lock_op_errno      = op_errno;
-                                local->op_errno              = op_errno;
-                        }
-        } else if (op_ret == 0) {
-                        int_lock->inode_locked_nodes[child_index]
-                                |= LOCKED_YES;
-                        int_lock->inodelk_lock_count++;
+				int_lock->lock_op_ret        = op_ret;
+				int_lock->lock_op_errno      = op_errno;
+				local->op_errno              = op_errno;
+			}
+			if (local->transaction.eager_lock)
+				local->transaction.eager_lock[child_index] = 0;
+		} else {
+			inodelk->locked_nodes[child_index] |= LOCKED_YES;
+			inodelk->lock_count++;
+
+			if (local->transaction.eager_lock &&
+			    local->transaction.eager_lock[child_index] &&
+			    local->fd) {
+				/* piggybacked */
+				if (op_ret == 1) {
+					/* piggybacked */
+				} else if (op_ret == 0) {
+					/* lock acquired from server */
+                                        fd_ctx->lock_acquired[child_index]++;
+				}
+			}
+		}
+
+                call_count = --int_lock->lk_call_count;
         }
+        UNLOCK (&frame->lock);
 
         if (call_count == 0) {
                 gf_log (this->name, GF_LOG_TRACE,
                         "Last inode locking reply received");
-                /* all locks successfull. Proceed to call FOP */
-                if (int_lock->inodelk_lock_count ==
-                    afr_up_children_count (priv->child_count, local->child_up)) {
+                /* all locks successful. Proceed to call FOP */
+                if (inodelk->lock_count == int_lock->lk_expected_count) {
                         gf_log (this->name, GF_LOG_TRACE,
                                 "All servers locked. Calling the cbk");
                         int_lock->lock_op_ret = 0;
                         int_lock->lock_cbk (frame, this);
                 }
-                /* Not all locks were successfull. Unlock and try locking
+                /* Not all locks were successful. Unlock and try locking
                    again, this time with serially blocking locks */
                 else {
                         gf_log (this->name, GF_LOG_TRACE,
@@ -1352,31 +1477,36 @@ int
 afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
         afr_local_t         *local    = NULL;
         afr_private_t       *priv     = NULL;
         afr_fd_ctx_t        *fd_ctx   = NULL;
-
-        int32_t  call_count = 0;
-        uint64_t ctx        = 0;
-        int      i          = 0;
-        int      ret        = 0;
-        struct gf_flock flock;
+        int32_t             call_count = 0;
+        int                 i          = 0;
+        int                 ret        = 0;
+        struct              gf_flock flock = {0,};
+        struct              gf_flock full_flock = {0,};
+        struct              gf_flock *flock_use = NULL;
+        int                 piggyback = 0;
 
         local    = frame->local;
         int_lock = &local->internal_lock;
         priv     = this->private;
 
-	flock.l_start = int_lock->lk_flock.l_start;
-	flock.l_len   = int_lock->lk_flock.l_len;
-	flock.l_type  = int_lock->lk_flock.l_type;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+
+        flock.l_start = inodelk->flock.l_start;
+        flock.l_len   = inodelk->flock.l_len;
+        flock.l_type  = inodelk->flock.l_type;
+
+        full_flock.l_type = inodelk->flock.l_type;
 
         initialize_inodelk_variables (frame, this);
 
         if (local->fd) {
-                ret = fd_ctx_get (local->fd, this, &ctx);
-
-                if (ret < 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
+                fd_ctx = afr_fd_ctx_get (local->fd, this);
+                if (!fd_ctx) {
+                        gf_log (this->name, GF_LOG_INFO,
                                 "unable to get fd ctx for fd=%p",
                                 local->fd);
 
@@ -1385,273 +1515,96 @@ afr_nonblocking_inodelk (call_frame_t *frame, xlator_t *this)
                         local->op_errno         = EINVAL;
                         int_lock->lock_op_errno = EINVAL;
 
+			afr_unlock (frame, this);
                         ret = -1;
                         goto out;
                 }
 
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-                call_count = internal_lock_count (frame, this, fd_ctx);
+                call_count = internal_lock_count (frame, this);
                 int_lock->lk_call_count = call_count;
+                int_lock->lk_expected_count = call_count;
+
+                if (!call_count) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "fd not open on any subvolumes. aborting.");
+                        afr_unlock (frame, this);
+                        goto out;
+                }
 
                 /* Send non-blocking inodelk calls only on up children
                    and where the fd has been opened */
                 for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i] && fd_ctx->opened_on[i]) {
-                                afr_trace_inodelk_in (frame, AFR_INODELK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP, &flock, F_SETLK, i);
-
-                                STACK_WIND_COOKIE (frame, afr_nonblocking_inodelk_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->finodelk,
-                                                   this->name, local->fd,
-                                                   F_SETLK, &flock);
-
-                                if (!--call_count)
-                                        break;
+                        if (!local->child_up[i])
+                                continue;
 
+                        flock_use = &flock;
+                        if (!local->transaction.eager_lock_on) {
+                                goto wind;
                         }
 
-                }
-        } else {
-                call_count = internal_lock_count (frame, this, NULL);
-                int_lock->lk_call_count = call_count;
+                        piggyback = 0;
+                        local->transaction.eager_lock[i] = 1;
 
-                for (i = 0; i < priv->child_count; i++) {
-                        if (local->child_up[i]) {
-                                afr_trace_inodelk_in (frame, AFR_INODELK_NB_TRANSACTION,
-                                                      AFR_LOCK_OP, &flock, F_SETLK, i);
+			afr_set_delayed_post_op (frame, this);
 
-                                STACK_WIND_COOKIE (frame, afr_nonblocking_inodelk_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->inodelk,
-                                                   this->name, &local->loc,
-                                                   F_SETLK, &flock);
+                        LOCK (&local->fd->lock);
+                        {
+                                if (fd_ctx->lock_acquired[i]) {
+                                        fd_ctx->lock_piggyback[i]++;
+                                        piggyback = 1;
+                                }
+                        }
+                        UNLOCK (&local->fd->lock);
 
+                        if (piggyback) {
+                                /* (op_ret == 1) => indicate piggybacked lock */
+                                afr_nonblocking_inodelk_cbk (frame, (void *) (long) i,
+                                                             this, 1, 0, NULL);
                                 if (!--call_count)
                                         break;
-
+                                continue;
                         }
-                }
-        }
+                        flock_use = &full_flock;
+                wind:
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_NB_TRANSACTION,
+                                              AFR_LOCK_OP, flock_use, F_SETLK, i);
 
-out:
-        return ret;
-}
-
-static int
-__is_lower_locked (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_local_t         *local    = NULL;
-
-        int count = 0;
-        int i     = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->lower_locked_nodes[i] & LOCKED_LOWER)
-                        count++;
-        }
-
-        return count;
-
-}
-
-static int
-__is_higher_locked (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_private_t       *priv     = NULL;
-        afr_local_t         *local    = NULL;
-
-        int count = 0;
-        int i = 0;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        priv     = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->locked_nodes[i] & LOCKED_YES)
-                        count++;
-        }
-
-        return count;
-
-}
-
-static int
-afr_unlock_lower_entrylk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-        const char          *basename = NULL;
-        loc_t               *loc      = NULL;
-
-        int call_count = 0;
-        int i = -1;
-
-        local = frame->local;
-        int_lock = &local->internal_lock;
-        priv  = this->private;
-
-        basename = int_lock->lk_basename;
-        if (int_lock->lk_loc)
-                loc = int_lock->lk_loc;
-
-        call_count = __is_lower_locked (frame, this);
-        int_lock->lk_call_count = call_count;
+                        STACK_WIND_COOKIE (frame, afr_nonblocking_inodelk_cbk,
+                                           (void *) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->finodelk,
+                                           int_lock->domain, local->fd,
+                                           F_SETLK, flock_use, NULL);
 
-        if (!call_count){
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No internal locks unlocked");
-                int_lock->lock_cbk (frame, this);
-                goto out;
-        }
+                        if (!--call_count)
+                                break;
+                }
+        } else {
+                call_count = internal_lock_count (frame, this);
+                int_lock->lk_call_count = call_count;
+                int_lock->lk_expected_count = call_count;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (int_lock->lower_locked_nodes[i] & LOCKED_LOWER) {
-                        afr_trace_entrylk_in (frame, AFR_ENTRYLK_NB_TRANSACTION,
-                                              AFR_UNLOCK_OP, basename, i);
+                for (i = 0; i < priv->child_count; i++) {
+                        if (!local->child_up[i])
+                                continue;
+                        AFR_TRACE_INODELK_IN (frame, this,
+                                              AFR_INODELK_NB_TRANSACTION,
+                                              AFR_LOCK_OP, &flock, F_SETLK, i);
 
-                        STACK_WIND_COOKIE (frame, afr_unlock_entrylk_cbk,
+                        STACK_WIND_COOKIE (frame, afr_nonblocking_inodelk_cbk,
                                            (void *) (long) i,
                                            priv->children[i],
-                                           priv->children[i]->fops->entrylk,
-                                           this->name,
-                                           loc, basename,
-                                           ENTRYLK_UNLOCK, ENTRYLK_WRLCK);
+                                           priv->children[i]->fops->inodelk,
+                                           int_lock->domain, &local->loc,
+                                           F_SETLK, &flock, NULL);
 
                         if (!--call_count)
                                 break;
-
                 }
         }
-
 out:
-        return 0;
-
-}
-
-
-static int
-afr_post_unlock_higher_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->transaction.done (frame, this);
-        return 0;
-}
-
-static int
-afr_post_unlock_lower_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_local_t         *local       = NULL;
-	loc_t               *lower       = NULL;
-	loc_t               *higher      = NULL;
-	const char          *lower_name  = NULL;
-	const char          *higher_name = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        lower_name = (lower == &local->transaction.parent_loc ?
-                      local->transaction.basename :
-                      local->transaction.new_basename);
-
-        higher = (lower == &local->transaction.parent_loc ?
-                  &local->transaction.new_parent_loc :
-                  &local->transaction.parent_loc);
-
-        higher_name = (higher == &local->transaction.parent_loc ?
-                       local->transaction.basename :
-                       local->transaction.new_basename);
-
-        if (__is_higher_locked (frame, this)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "unlocking higher");
-                int_lock->lk_basename = higher_name;
-                int_lock->lk_loc      = higher;
-                int_lock->lock_cbk    = afr_post_unlock_higher_cbk;
-
-                afr_unlock_entrylk (frame, this);
-        } else
-                local->transaction.done (frame, this);
-
-        return 0;
-}
-
-static int
-afr_rename_unlock (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock    = NULL;
-        afr_local_t         *local       = NULL;
-	loc_t               *lower       = NULL;
-	loc_t               *higher      = NULL;
-	const char          *lower_name  = NULL;
-	const char          *higher_name = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        lower = lower_path (&local->transaction.parent_loc,
-                            local->transaction.basename,
-                            &local->transaction.new_parent_loc,
-                            local->transaction.new_basename);
-
-        lower_name = (lower == &local->transaction.parent_loc ?
-                      local->transaction.basename :
-                      local->transaction.new_basename);
-
-        higher = (lower == &local->transaction.parent_loc ?
-                  &local->transaction.new_parent_loc :
-                  &local->transaction.parent_loc);
-
-        higher_name = (higher == &local->transaction.parent_loc ?
-                       local->transaction.basename :
-                       local->transaction.new_basename);
-
-
-        if (__is_lower_locked (frame, this)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "unlocking lower");
-                int_lock->lk_basename = lower_name;
-                int_lock->lk_loc      = lower;
-                int_lock->lock_cbk    = afr_post_unlock_lower_cbk;
-
-                afr_unlock_lower_entrylk (frame, this);
-        } else
-                afr_post_unlock_lower_cbk (frame, this);
-
-        return 0;
-}
-
-static int
-afr_rename_transaction (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *local = NULL;
-
-        local = frame->local;
-
-        return (local->transaction.type ==
-                AFR_ENTRY_RENAME_TRANSACTION);
-
+        return ret;
 }
 
 int32_t
@@ -1665,10 +1618,8 @@ afr_unlock (call_frame_t *frame, xlator_t *this)
                 if (is_afr_lock_transaction (local))
                         afr_unlock_inodelk (frame, this);
                 else
-                        if (!afr_rename_transaction (frame, this))
-                                afr_unlock_entrylk (frame, this);
-                        else
-                                afr_rename_unlock (frame, this);
+                        afr_unlock_entrylk (frame, this);
+
         } else {
                 if (is_afr_lock_selfheal (local))
                         afr_unlock_inodelk (frame, this);
@@ -1680,490 +1631,37 @@ afr_unlock (call_frame_t *frame, xlator_t *this)
 }
 
 int
-afr_mark_locked_nodes (xlator_t *this, fd_t *fd,
-                       unsigned char *locked_nodes)
-{
-        afr_private_t *priv  = NULL;
-        afr_fd_ctx_t  *fdctx = NULL;
-        uint64_t       tmp   = 0;
-        int            ret   = 0;
-
-        priv = this->private;
-
-        afr_fd_ctx_set (this, fd);
-        if (ret < 0)
-                goto out;
-
-        ret = fd_ctx_get (fd, this, &tmp);
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-
-        GF_ASSERT (fdctx->locked_on);
-
-        memcpy (fdctx->locked_on, locked_nodes,
-                priv->child_count);
-
-out:
-        return ret;
-}
-
-static int
-__is_fd_saved (xlator_t *this, fd_t *fd)
-{
-        afr_locked_fd_t *locked_fd = NULL;
-        afr_private_t   *priv      = NULL;
-        int              found     = 0;
-
-        priv = this->private;
-
-        list_for_each_entry (locked_fd, &priv->saved_fds, list) {
-                if (locked_fd->fd == fd) {
-                        found = 1;
-                        break;
-                }
-        }
-
-        return found;
-}
-
-static int
-__afr_save_locked_fd (xlator_t *this, fd_t *fd)
-{
-        afr_private_t   *priv      = NULL;
-        afr_locked_fd_t *locked_fd = NULL;
-        int              ret       = 0;
-
-        priv = this->private;
-
-        locked_fd = GF_CALLOC (1, sizeof (*locked_fd),
-                               gf_afr_mt_locked_fd);
-        if (!locked_fd) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                ret = -1;
-                goto out;
-        }
-
-        locked_fd->fd = fd;
-        INIT_LIST_HEAD (&locked_fd->list);
-
-        list_add_tail (&locked_fd->list, &priv->saved_fds);
-
-out:
-        return ret;
-}
-
-int
-afr_save_locked_fd (xlator_t *this, fd_t *fd)
-{
-        afr_private_t   *priv      = NULL;
-        int              ret       = 0;
-
-        priv = this->private;
-
-        pthread_mutex_lock (&priv->mutex);
-        {
-                if (__is_fd_saved (this, fd)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "fd=%p already saved", fd);
-                        goto unlock;
-                }
-
-                ret = __afr_save_locked_fd (this, fd);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "fd=%p could not be saved");
-                        goto unlock;
-                }
-        }
-unlock:
-        pthread_mutex_unlock (&priv->mutex);
-
-        return ret;
-}
-
-static int
-afr_lock_recovery_cleanup (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t     *local     = NULL;
-        afr_locked_fd_t *locked_fd = NULL;
-
-        local = frame->local;
-
-        locked_fd = local->locked_fd;
-
-        STACK_DESTROY (frame->root);
-        afr_local_cleanup (local, this);
-
-        afr_save_locked_fd (this, locked_fd->fd);
-
-        return 0;
-
-}
-
-static int
-afr_get_source_lock_recovery (xlator_t *this, fd_t *fd)
-{
-        afr_fd_ctx_t  *fdctx        = NULL;
-        afr_private_t *priv         = NULL;
-        uint64_t      tmp           = 0;
-        int           i             = 0;
-        int           source_child  = -1;
-        int           ret           = 0;
-
-        priv = this->private;
-
-        ret = fd_ctx_get (fd, this, &tmp);
-        if (ret)
-                goto out;
-
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (fdctx->locked_on[i]) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Found lock recovery source=%d",
-                                i);
-                        source_child = i;
-                        break;
-                }
-
-        }
-
-out:
-        return source_child;
-
-}
-
-int32_t
-afr_get_locks_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock);
-int32_t
-afr_recover_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
-{
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
-        int32_t source_child = 0;
-        struct gf_flock flock   = {0,};
-
-        local = frame->local;
-        priv  = this->private;
-
-        if (op_ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "lock recovery failed");
-                goto cleanup;
-        }
-
-        source_child = local->source_child;
-
-        memcpy (&flock, lock, sizeof (*lock));
-
-        STACK_WIND_COOKIE (frame, afr_get_locks_fd_cbk,
-                           (void *) (long) source_child,
-                           priv->children[source_child],
-                           priv->children[source_child]->fops->lk,
-                           local->fd, F_GETLK_FD, &flock);
-
-        return 0;
-
-cleanup:
-        afr_lock_recovery_cleanup (frame, this);
-        return 0;
-}
-
-int
-afr_recover_lock (call_frame_t *frame, xlator_t *this,
-                  struct gf_flock *flock)
+afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src, char *dom,
+                          unsigned int child_count)
 {
-        afr_local_t   *local             = NULL;
-        afr_private_t *priv              = NULL;
-        int32_t      lock_recovery_child = 0;
-
-        priv  = this->private;
-        local = frame->local;
-
-        lock_recovery_child = local->lock_recovery_child;
-
-        frame->root->lk_owner = flock->l_owner;
-
-        STACK_WIND_COOKIE (frame, afr_recover_lock_cbk,
-                           (void *) (long) lock_recovery_child,
-                           priv->children[lock_recovery_child],
-                           priv->children[lock_recovery_child]->fops->lk,
-                           local->fd, F_SETLK, flock);
-
-        return 0;
-}
-
-static int
-is_afr_lock_eol (struct gf_flock *lock)
-{
-        int ret = 0;
-
-        if ((lock->l_type == GF_LK_EOL))
-                ret = 1;
-
-        return ret;
-}
-
-int32_t
-afr_get_locks_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
-{
-        if (op_ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Failed to get locks on fd");
-                goto cleanup;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Got a lock on fd");
-
-        if (is_afr_lock_eol (lock)) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Reached EOL on locks on fd");
-                goto cleanup;
-        }
-
-        afr_recover_lock (frame, this, lock);
-
-        return 0;
-
-cleanup:
-        afr_lock_recovery_cleanup (frame, this);
-
-        return 0;
-}
-
-static int
-afr_lock_recovery (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t   *local        = NULL;
-        afr_private_t *priv         = NULL;
-        fd_t          *fd           = NULL;
-        int            ret          = 0;
-        int32_t        source_child = 0;
-        struct gf_flock   flock        = {0,};
-
-        priv  = this->private;
-        local = frame->local;
-
-        fd = local->fd;
-
-        source_child = afr_get_source_lock_recovery (this, fd);
-        if (source_child < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Could not recover locks due to lock "
-                        "split brain");
-                ret = -1;
+        afr_local_t         *dst_local   = NULL;
+        afr_local_t         *src_local   = NULL;
+        afr_internal_lock_t *dst_lock    = NULL;
+        afr_internal_lock_t *src_lock    = NULL;
+        afr_inodelk_t       *dst_inodelk = NULL;
+        afr_inodelk_t       *src_inodelk = NULL;
+        int                 ret = -1;
+
+        src_local = src->local;
+        src_lock  = &src_local->internal_lock;
+        src_inodelk = afr_get_inodelk (src_lock, dom);
+        dst_local = dst->local;
+        dst_lock  = &dst_local->internal_lock;
+        dst_inodelk = afr_get_inodelk (dst_lock, dom);
+        if (!dst_inodelk || !src_inodelk)
                 goto out;
+        if (src_inodelk->locked_nodes) {
+                memcpy (dst_inodelk->locked_nodes, src_inodelk->locked_nodes,
+                        sizeof (*dst_inodelk->locked_nodes) * child_count);
+                memset (src_inodelk->locked_nodes, 0,
+                        sizeof (*src_inodelk->locked_nodes) * child_count);
         }
 
-        local->source_child = source_child;
-
-        /* the flock can be zero filled as we're querying incrementally
-           the locks held on the fd.
-        */
-        STACK_WIND_COOKIE (frame, afr_get_locks_fd_cbk,
-                           (void *) (long) source_child,
-                           priv->children[source_child],
-                           priv->children[source_child]->fops->lk,
-                           local->fd, F_GETLK_FD, &flock);
-
-out:
-        return ret;
-}
-
-
-static int
-afr_mark_fd_opened (xlator_t *this, fd_t *fd, int32_t child_index)
-{
-        afr_fd_ctx_t *fdctx = NULL;
-        uint64_t      tmp   = 0;
-        int           ret   = 0;
-
-        ret = fd_ctx_get (fd, this, &tmp);
-        if (ret)
-                goto out;
-
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-
-        fdctx->opened_on[child_index] = 1;
-
-out:
-        return ret;
-}
-
-int32_t
-afr_lock_recovery_preopen_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                               int32_t op_ret, int32_t op_errno, fd_t *fd)
-{
-        int32_t child_index = (long )cookie;
-        int ret = 0;
-
-        if (op_ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Reopen during lock-recovery failed");
-                goto cleanup;
-        }
-
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Open succeeded => proceed to recover locks");
-
-        ret = afr_lock_recovery (frame, this);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Lock recovery failed");
-                goto cleanup;
-        }
-
-        ret = afr_mark_fd_opened (this, fd, child_index);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Marking fd open failed");
-                goto cleanup;
-        }
-
-        return 0;
-
-cleanup:
-        afr_lock_recovery_cleanup (frame, this);
-        return 0;
-}
-
-static int
-afr_lock_recovery_preopen (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *priv  = NULL;
-        afr_local_t   *local = NULL;
-        uint64_t       tmp   = 0;
-        afr_fd_ctx_t  *fdctx = NULL;
-        loc_t          loc   = {0,};
-        int32_t        child_index = 0;
-        int            ret   = 0;
-
-        priv  = this->private;
-        local = frame->local;
-
-        GF_ASSERT (local && local->fd);
-
-        ret = fd_ctx_get (local->fd, this, &tmp);
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-        GF_ASSERT (fdctx);
-
-        child_index = local->lock_recovery_child;
-
-        inode_path (local->fd->inode, NULL, (char **)&loc.path);
-        loc.name   = strrchr (loc.path, '/');
-        loc.inode  = inode_ref (local->fd->inode);
-        loc.parent = inode_parent (local->fd->inode, 0, NULL);
-
-
-        STACK_WIND_COOKIE (frame, afr_lock_recovery_preopen_cbk,
-                           (void *)(long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->open,
-                           &loc, fdctx->flags, local->fd,
-                           fdctx->wbflags);
-
-        return 0;
-}
-
-static int
-is_fd_opened (fd_t *fd, int32_t child_index)
-{
-        afr_fd_ctx_t *fdctx = NULL;
-        uint64_t      tmp = 0;
-        int           ret = 0;
-
-        ret = fd_ctx_get (fd, THIS, &tmp);
-        if (ret)
-                goto out;
-
-        fdctx = (afr_fd_ctx_t *) (long) tmp;
-
-        if (fdctx->opened_on[child_index])
-                ret = 1;
-
-out:
-        return ret;
-}
-
-int
-afr_attempt_lock_recovery (xlator_t *this, int32_t child_index)
-{
-        call_frame_t    *frame     = NULL;
-        afr_private_t   *priv      = NULL;
-        afr_local_t     *local     = NULL;
-        afr_locked_fd_t *locked_fd = NULL;
-       afr_locked_fd_t  *tmp       = NULL;
-        int              ret       = 0;
-        struct list_head locks_list;
-
-
-        priv = this->private;
-
-        if (list_empty (&priv->saved_fds))
-                goto out;
-
-        frame = create_frame (this, this->ctx->pool);
-        if (!frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                ret = -1;
-                goto out;
-        }
-
-        local = GF_CALLOC (1, sizeof (*local),
-                           gf_afr_mt_afr_local_t);
-        if (!local) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Out of memory");
-                ret = -1;
-                goto out;
-        }
-
-        AFR_LOCAL_INIT (local, priv);
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                ret = -1;
-                goto out;
-        }
-
-        frame->local = local;
-
-        INIT_LIST_HEAD (&locks_list);
-
-        pthread_mutex_lock (&priv->mutex);
-        {
-                list_splice_init (&priv->saved_fds, &locks_list);
-        }
-        pthread_mutex_unlock (&priv->mutex);
-
-        list_for_each_entry_safe (locked_fd, tmp,
-                                  &locks_list, list) {
-
-                list_del_init (&locked_fd->list);
-
-                local->fd                  = fd_ref (locked_fd->fd);
-                local->lock_recovery_child = child_index;
-                local->locked_fd           = locked_fd;
-
-                if (!is_fd_opened (locked_fd->fd, child_index)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "attempting open before lock "
-                                "recovery");
-                        afr_lock_recovery_preopen (frame, this);
-                } else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "attempting lock recovery "
-                                "without a preopen");
-                        afr_lock_recovery (frame, this);
-                }
-        }
-
+        dst_lock->transaction_lk_type = src_lock->transaction_lk_type;
+        dst_lock->selfheal_lk_type    = src_lock->selfheal_lk_type;
+        dst_inodelk->lock_count = src_inodelk->lock_count;
+        src_inodelk->lock_count = 0;
+        ret = 0;
 out:
         return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-mem-types.h b/xlators/cluster/afr/src/afr-mem-types.h
index b4937efa2..05df90cc0 100644
--- a/xlators/cluster/afr/src/afr-mem-types.h
+++ b/xlators/cluster/afr/src/afr-mem-types.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -26,7 +17,6 @@
 enum gf_afr_mem_types_ {
         gf_afr_mt_iovec  = gf_common_mt_end + 1,
         gf_afr_mt_afr_fd_ctx_t,
-        gf_afr_mt_afr_local_t,
         gf_afr_mt_afr_private_t,
         gf_afr_mt_int32_t,
         gf_afr_mt_char,
@@ -42,6 +32,17 @@ enum gf_afr_mem_types_ {
         gf_afr_mt_entry_name,
         gf_afr_mt_pump_priv,
         gf_afr_mt_locked_fd,
+        gf_afr_mt_inode_ctx_t,
+        gf_afr_fd_paused_call_t,
+        gf_afr_mt_crawl_data_t,
+        gf_afr_mt_brick_pos_t,
+        gf_afr_mt_shd_bool_t,
+        gf_afr_mt_shd_timer_t,
+        gf_afr_mt_shd_event_t,
+        gf_afr_mt_time_t,
+        gf_afr_mt_pos_data_t,
+	gf_afr_mt_reply_t,
+	gf_afr_mt_subvol_healer_t,
         gf_afr_mt_end
 };
 #endif
diff --git a/xlators/cluster/afr/src/afr-open.c b/xlators/cluster/afr/src/afr-open.c
index 5569dd1e2..f86aa7fd8 100644
--- a/xlators/cluster/afr/src/afr-open.c
+++ b/xlators/cluster/afr/src/afr-open.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -52,422 +43,295 @@
 #include "afr-dir-read.h"
 #include "afr-dir-write.h"
 #include "afr-transaction.h"
-#include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
 
 
-int
-afr_open_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this, 
-			int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                        struct iatt *postbuf)
+gf_boolean_t
+afr_is_fd_fixable (fd_t *fd)
 {
-	afr_local_t * local = frame->local;
-
-	AFR_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
-			  local->fd);
-	return 0;
-}
-
-
-int
-afr_open_cbk (call_frame_t *frame, void *cookie,
-	      xlator_t *this, int32_t op_ret, int32_t op_errno,
-	      fd_t *fd)
-{
-	afr_local_t *  local = NULL;
-
-        int child_index = (long) cookie;
-
-        uint64_t      ctx;
-        afr_fd_ctx_t *fd_ctx;
-
-        int ret = 0;
-
-	int call_count = -1;
-
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-		}
-
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-                        local->success_count++;
-
-                        ret = afr_fd_ctx_set (this, fd);
-
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "could not set fd ctx for fd=%p",
-                                        fd);
-
-                                local->op_ret   = -1;
-                                local->op_errno = -ret;
-                                goto unlock;
-                        }
-
-                        ret = fd_ctx_get (fd, this, &ctx);
-
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "could not get fd ctx for fd=%p", fd);
-                                local->op_ret   = -1;
-                                local->op_errno = -ret;
-                                goto unlock;
-                        }
-
-                        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-                        fd_ctx->opened_on[child_index] = 1;
-                        fd_ctx->flags                  = local->cont.open.flags;
-                        fd_ctx->wbflags                = local->cont.open.wbflags;
-		}
-	}
-unlock:
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		if ((local->cont.open.flags & O_TRUNC)
-		    && (local->op_ret >= 0)) {
-			STACK_WIND (frame, afr_open_ftruncate_cbk,
-				    this, this->fops->ftruncate,
-				    fd, 0);
-		} else {
-                        AFR_STACK_UNWIND (open, frame, local->op_ret,
-                                          local->op_errno, local->fd);
-		}
-	}
-
-	return 0;
+        if (!fd || !fd->inode)
+                return _gf_false;
+        else if (fd_is_anonymous (fd))
+                return _gf_false;
+        else if (uuid_is_null (fd->inode->gfid))
+                return _gf_false;
+
+        return _gf_true;
 }
 
 
 int
-afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, int32_t wbflags)
+afr_open_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t *   local = NULL;
-
-	int     i = 0;
-	int   ret = -1;
-
-	int32_t call_count = 0;	
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-	int32_t wind_flags = flags & (~O_TRUNC);
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-	VALIDATE_OR_GOTO (loc, out);
-
-	priv = this->private;
-
-        if (afr_is_split_brain (this, loc->inode)) {
-		/* self-heal failed */
-		op_errno = EIO;
-		goto out;
-	}
+        afr_local_t * local = frame->local;
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-	frame->local = local;
-	call_count   = local->call_count;
-
-        loc_copy (&local->loc, loc);
-
-	local->cont.open.flags   = flags;
-        local->cont.open.wbflags = wbflags;
-
-	local->fd = fd_ref (fd);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_open_cbk, (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->open,
-					   loc, wind_flags, fd, wbflags);
-
-			if (!--call_count)
-				break;
-		}
-	}
-
-	op_ret = 0;
-out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (open, frame, op_ret, op_errno, fd);
-	}
-
-	return 0;
+        AFR_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
+                          local->fd, xdata);
+        return 0;
 }
 
 
 int
-afr_openfd_sh_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, fd_t *fd)
+afr_open_cbk (call_frame_t *frame, void *cookie,
+              xlator_t *this, int32_t op_ret, int32_t op_errno,
+              fd_t *fd, dict_t *xdata)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_private_t       *priv     = NULL;
-
-        int ret = 0;
-
-        uint64_t      ctx;
-        afr_fd_ctx_t *fd_ctx;
-
-        int call_count  = 0;
-        int child_index = (long) cookie;
+        afr_local_t *  local       = NULL;
+        int            call_count  = -1;
+        int            child_index = (long) cookie;
+	afr_fd_ctx_t  *fd_ctx = NULL;
 
-        priv     = this->private;
-        local    = frame->local;
-        int_lock = &local->internal_lock;
+        local = frame->local;
+	fd_ctx = local->fd_ctx;
 
         LOCK (&frame->lock);
         {
-                if (op_ret >= 0) {
-                        ret = fd_ctx_get (fd, this, &ctx);
-
-                        if (ret < 0) {
-                                goto out;
-                        }
-
-                        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-                        fd_ctx->opened_on[child_index] = 1;
-
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "fd for %s opened successfully on subvolume %s",
-                                local->loc.path, priv->children[child_index]->name);
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+			fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+                } else {
+                        local->op_ret = op_ret;
+			fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+			if (!local->xdata_rsp && xdata)
+				local->xdata_rsp = dict_ref (xdata);
                 }
         }
-out:
         UNLOCK (&frame->lock);
 
         call_count = afr_frame_return (frame);
 
         if (call_count == 0) {
-                int_lock->lock_cbk = local->transaction.done;
-                local->transaction.resume (frame, this);
+                if ((fd_ctx->flags & O_TRUNC) && (local->op_ret >= 0)) {
+                        STACK_WIND (frame, afr_open_ftruncate_cbk,
+                                    this, this->fops->ftruncate,
+                                    fd, 0, NULL);
+                } else {
+                        AFR_STACK_UNWIND (open, frame, local->op_ret,
+                                          local->op_errno, local->fd,
+					  local->xdata_rsp);
+                }
         }
 
         return 0;
 }
 
-
-static int
-__unopened_count (int child_count, unsigned int *opened_on, unsigned char *child_up)
-{
-        int i;
-        int count = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (!opened_on[i] && child_up[i])
-                        count++;
-        }
-
-        return count;
-}
-
-
 int
-afr_openfd_sh_unwind (call_frame_t *frame, xlator_t *this)
+afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+          fd_t *fd, dict_t *xdata)
 {
-        afr_local_t *local  = NULL;
-        afr_private_t *priv = NULL;
-
-        uint64_t      ctx;
-        afr_fd_ctx_t *fd_ctx;
-
-        int abandon = 0;
-        int ret = 0;
-        int i;
-        int call_count = 0;
-
-        priv  = this->private;
-        local = frame->local;
+        afr_private_t * priv       = NULL;
+        afr_local_t *   local      = NULL;
+        int             i          = 0;
+        int32_t         call_count = 0;
+        int32_t         op_errno   = 0;
+	afr_fd_ctx_t   *fd_ctx = NULL;
 
-        /*
-         * Some subvolumes might have come up on which we never
-         * opened this fd in the first place. Re-open fd's on those
-         * subvolumes now.
-         */
+        //We can't let truncation to happen outside transaction.
 
-        ret = fd_ctx_get (local->fd, this, &ctx);
+        priv = this->private;
 
-        if (ret < 0) {
-                abandon = 1;
-                goto out;
+        if (flags & (O_CREAT|O_TRUNC)) {
+                QUORUM_CHECK(open,out);
         }
 
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-        LOCK (&local->fd->lock);
-        {
-                call_count = __unopened_count (priv->child_count,
-                                               fd_ctx->opened_on,
-                                               local->child_up);
-                for (i = 0; i < priv->child_count; i++) {
-                        fd_ctx->pre_op_done[i] = 0;
-                        fd_ctx->pre_op_piggyback[i] = 0;
-                }
-        }
-        UNLOCK (&local->fd->lock);
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx) {
+		op_errno = ENOMEM;
+		goto out;
+	}
 
-        if (call_count == 0) {
-                abandon = 1;
-                goto out;
-        }
+        local->fd = fd_ref (fd);
+	local->fd_ctx = fd_ctx;
+	fd_ctx->flags = flags;
 
-        local->call_count = call_count;
+        call_count = local->call_count;
 
-        for (i = 0; i < priv->child_count; i++) {
-                if (!fd_ctx->opened_on[i] && local->child_up[i]) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "opening fd for %s on subvolume %s",
-                                local->loc.path, priv->children[i]->name);
+        local->cont.open.flags = flags;
 
-                        STACK_WIND_COOKIE (frame, afr_openfd_sh_open_cbk,
-                                           (void *)(long) i,
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->child_up[i]) {
+                        STACK_WIND_COOKIE (frame, afr_open_cbk, (void *) (long) i,
                                            priv->children[i],
                                            priv->children[i]->fops->open,
-                                           &local->loc, fd_ctx->flags, local->fd,
-                                           fd_ctx->wbflags);
-
+                                           loc, (flags & ~O_TRUNC), fd, xdata);
                         if (!--call_count)
                                 break;
                 }
         }
 
+	return 0;
 out:
-        if (abandon)
-                local->transaction.resume (frame, this);
+	AFR_STACK_UNWIND (open, frame, -1, op_errno, fd, NULL);
 
         return 0;
 }
 
-
 int
-afr_openfd_sh (call_frame_t *frame, xlator_t *this)
+afr_openfd_fix_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, fd_t *fd,
+                         dict_t *xdata)
 {
-        afr_local_t     *local  = NULL;
-        afr_self_heal_t *sh = NULL;
-        char            sh_type_str[256] = {0,};
-
-        local = frame->local;
-        sh    = &local->self_heal;
+        afr_local_t   *local      = NULL;
+        afr_private_t *priv       = NULL;
+        afr_fd_ctx_t  *fd_ctx     = NULL;
+        int           call_count  = 0;
+        int           child_index = (long) cookie;
 
-        inode_path (local->fd->inode, NULL, (char **)&local->loc.path);
-        local->loc.name   = strrchr (local->loc.path, '/');
-        local->loc.inode  = inode_ref (local->fd->inode);
-        local->loc.parent = inode_parent (local->fd->inode, 0, NULL);
-
-        /* forcibly trigger missing-entries self-heal */
+        priv     = this->private;
+        local    = frame->local;
 
-        local->success_count    = 1;
-        local->enoent_count     = 1;
+        if (op_ret >= 0) {
+                gf_log (this->name, GF_LOG_DEBUG, "fd for %s opened "
+                        "successfully on subvolume %s", local->loc.path,
+                        priv->children[child_index]->name);
+        } else {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to open %s "
+                        "on subvolume %s", local->loc.path,
+                        priv->children[child_index]->name);
+        }
 
-        sh->data_lock_held      = _gf_true;
-        sh->need_data_self_heal = _gf_true;
-        sh->type                = local->fd->inode->ia_type;
-        sh->background          = _gf_false;
-        sh->unwind              = afr_openfd_sh_unwind;
+        fd_ctx = local->fd_ctx;
 
-        afr_self_heal_type_str_get(&local->self_heal,
-                                   sh_type_str,
-                                   sizeof(sh_type_str));
-        gf_log (this->name, GF_LOG_NORMAL, "%s self-heal triggered. "
-                "path: %s, reason: Replicate up down flush, data lock is held",
-                sh_type_str, local->loc.path);
+        LOCK (&local->fd->lock);
+        {
+                if (op_ret >= 0) {
+                        fd_ctx->opened_on[child_index] = AFR_FD_OPENED;
+                } else {
+                        fd_ctx->opened_on[child_index] = AFR_FD_NOT_OPENED;
+                }
+        }
+        UNLOCK (&local->fd->lock);
 
-        afr_self_heal (frame, this);
+        call_count = afr_frame_return (frame);
+        if (call_count == 0)
+                AFR_STACK_DESTROY (frame);
 
         return 0;
 }
 
 
-int
-afr_openfd_flush_done (call_frame_t *frame, xlator_t *this)
+static int
+afr_fd_ctx_need_open (fd_t *fd, xlator_t *this, unsigned char *need_open)
 {
-        afr_private_t *priv = NULL;
-	afr_local_t *local  = NULL;
+	afr_fd_ctx_t *fd_ctx = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int count = 0;
 
-        uint64_t       ctx;
-        afr_fd_ctx_t * fd_ctx = NULL;
+	priv = this->private;
 
-        int _ret = -1;
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+		return 0;
 
-        priv  = this->private;
-	local = frame->local;
+	LOCK (&fd->lock);
+	{
+		for (i = 0; i < priv->child_count; i++) {
+			if (fd_ctx->opened_on[i] == AFR_FD_NOT_OPENED &&
+			    priv->child_up[i]) {
+				fd_ctx->opened_on[i] = AFR_FD_OPENING;
+				need_open[i] = 1;
+				count++;
+			} else {
+				need_open[i] = 0;
+			}
+		}
+	}
+	UNLOCK (&fd->lock);
 
-        LOCK (&local->fd->lock);
-        {
-                _ret = __fd_ctx_get (local->fd, this, &ctx);
+	return count;
+}
 
-                if (_ret < 0) {
-                        goto out;
-                }
 
-                fd_ctx = (afr_fd_ctx_t *)(long) ctx;
+void
+afr_fix_open (fd_t *fd, xlator_t *this)
+{
+        afr_private_t *priv    = NULL;
+        int           i        = 0;
+        call_frame_t  *frame   = NULL;
+        afr_local_t   *local   = NULL;
+        int           ret      = -1;
+        int32_t       op_errno = 0;
+        afr_fd_ctx_t  *fd_ctx  = NULL;
+	unsigned char *need_open = NULL;
+	int call_count = 0;
 
-                fd_ctx->down_count = priv->down_count;
-                fd_ctx->up_count   = priv->up_count;
-        }
-out:
-        UNLOCK (&local->fd->lock);
+        priv  = this->private;
 
-        afr_local_transaction_cleanup (local, this);
+        if (!afr_is_fd_fixable (fd))
+                goto out;
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "The up/down flush is over");
+        fd_ctx = afr_fd_ctx_get (fd, this);
+        if (!fd_ctx)
+                goto out;
 
-        fd_unref (local->fd);
-        local->openfd_flush_cbk (frame, this);
+	need_open = alloca0 (priv->child_count);
 
-	return 0;
-}
+	call_count = afr_fd_ctx_need_open (fd, this, need_open);
+	if (!call_count)
+		goto out;
 
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame)
+                goto out;
 
-int
-afr_openfd_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
-{
-	afr_local_t   * local = NULL;
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local)
+		goto out;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        local->loc.inode = inode_ref (fd->inode);
+        ret = loc_path (&local->loc, NULL);
+        if (ret < 0)
+                goto out;
 
-        local = frame->local;
+        local->fd = fd_ref (fd);
+	local->fd_ctx = fd_ctx;
 
-        local->op = GF_FOP_FLUSH;
+        local->call_count = call_count;
 
-        local->fd = fd_ref (fd);
+        gf_log (this->name, GF_LOG_DEBUG, "need open count: %d",
+                call_count);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (!need_open[i])
+                        continue;
 
-        local->transaction.fop          = afr_openfd_sh;
-        local->transaction.done         = afr_openfd_flush_done;
+                if (IA_IFDIR == fd->inode->ia_type) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "opening fd for dir %s on subvolume %s",
+                                local->loc.path, priv->children[i]->name);
 
-        local->transaction.start  = 0;
-        local->transaction.len    = 0;
+                        STACK_WIND_COOKIE (frame, afr_openfd_fix_open_cbk,
+                                           (void*) (long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->opendir,
+                                           &local->loc, local->fd,
+                                           NULL);
+                } else {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "opening fd for file %s on subvolume %s",
+                                local->loc.path, priv->children[i]->name);
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "doing up/down flush on fd=%p",
-                fd);
+                        STACK_WIND_COOKIE (frame, afr_openfd_fix_open_cbk,
+                                           (void *)(long) i,
+                                           priv->children[i],
+                                           priv->children[i]->fops->open,
+                                           &local->loc,
+                                           fd_ctx->flags & (~O_TRUNC),
+                                           local->fd, NULL);
+                }
 
-        afr_transaction (frame, this, AFR_DATA_TRANSACTION);
+		if (!--call_count)
+			break;
+        }
 
+	return;
 out:
-	return 0;
+        if (frame)
+                AFR_STACK_DESTROY (frame);
 }
-
diff --git a/xlators/cluster/afr/src/afr-read-txn.c b/xlators/cluster/afr/src/afr-read-txn.c
new file mode 100644
index 000000000..186f68c33
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-read-txn.c
@@ -0,0 +1,239 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "afr.h"
+#include "afr-transaction.h"
+
+int
+afr_read_txn_next_subvol (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int subvol = -1;
+
+	local = frame->local;
+	priv = this->private;
+
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->readable[i]) {
+			/* don't even bother trying here.
+			   just mark as attempted and move on. */
+			local->read_attempted[i] = 1;
+			continue;
+		}
+
+		if (!local->read_attempted[i]) {
+			subvol = i;
+			break;
+		}
+	}
+
+	/* If no more subvols were available for reading, we leave
+	   @subvol as -1, which is an indication we have run out of
+	   readable subvols. */
+	if (subvol != -1)
+		local->read_attempted[subvol] = 1;
+	local->readfn (frame, this, subvol);
+
+	return 0;
+}
+
+
+int
+afr_read_txn_refresh_done (call_frame_t *frame, xlator_t *this, int err)
+{
+	afr_local_t *local = NULL;
+	int read_subvol = 0;
+	int event_generation = 0;
+	inode_t *inode = NULL;
+	int ret = -1;
+
+	local = frame->local;
+	inode = local->inode;
+
+	if (err) {
+		local->op_errno = -err;
+		local->op_ret = -1;
+		read_subvol = -1;
+		goto readfn;
+	}
+
+	ret = afr_inode_read_subvol_type_get (inode, this, local->readable,
+					      &event_generation,
+					      local->transaction.type);
+
+	if (ret == -1 || !event_generation) {
+		/* Even after refresh, we don't have a good
+		   read subvolume. Time to bail */
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		read_subvol = -1;
+		goto readfn;
+	}
+
+	read_subvol = afr_read_subvol_select_by_policy (inode, this,
+							local->readable);
+
+	if (read_subvol == -1) {
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		goto readfn;
+	}
+
+	if (local->read_attempted[read_subvol]) {
+		afr_read_txn_next_subvol (frame, this);
+		return 0;
+	}
+
+	local->read_attempted[read_subvol] = 1;
+readfn:
+	local->readfn (frame, this, read_subvol);
+
+	return 0;
+}
+
+
+int
+afr_read_txn_continue (call_frame_t *frame, xlator_t *this, int subvol)
+{
+	afr_local_t *local = NULL;
+
+	local = frame->local;
+
+	if (!local->refreshed) {
+		local->refreshed = _gf_true;
+		afr_inode_refresh (frame, this, local->inode,
+				   afr_read_txn_refresh_done);
+	} else {
+		afr_read_txn_next_subvol (frame, this);
+	}
+
+	return 0;
+}
+
+
+/* afr_read_txn_wipe:
+
+   clean internal variables in @local in order to make
+   it possible to call afr_read_txn() multiple times from
+   the same frame
+*/
+
+void
+afr_read_txn_wipe (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
+
+	local = frame->local;
+	priv = this->private;
+
+	local->readfn = NULL;
+
+	if (local->inode)
+		inode_unref (local->inode);
+
+	for (i = 0; i < priv->child_count; i++) {
+		local->read_attempted[i] = 0;
+		local->readable[i] = 0;
+	}
+}
+
+
+/*
+  afr_read_txn:
+
+  This is the read transaction function. The way it works:
+
+  - Determine read-subvolume from inode ctx.
+
+  - If read-subvolume's generation was stale, refresh ctx once by
+    calling afr_inode_refresh()
+
+    Else make an attempt to read on read-subvolume.
+
+  - If attempted read on read-subvolume fails, refresh ctx once
+    by calling afr_inode_refresh()
+
+  - After ctx refresh, query read-subvolume freshly and attempt
+    read once.
+
+  - If read fails, try every other readable[] subvolume before
+    finally giving up. readable[] elements are set by afr_inode_refresh()
+    based on dirty and pending flags.
+
+  - If file is in split brain in the backend, generation will be
+    kept 0 by afr_inode_refresh() and readable[] will be set 0 for
+    all elements. Therefore reads always fail.
+*/
+
+int
+afr_read_txn (call_frame_t *frame, xlator_t *this, inode_t *inode,
+	      afr_read_txn_wind_t readfn, afr_transaction_type type)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int read_subvol = -1;
+	int event_generation = 0;
+	int ret = -1;
+
+	priv = this->private;
+	local = frame->local;
+
+	afr_read_txn_wipe (frame, this);
+
+	local->readfn = readfn;
+	local->inode = inode_ref (inode);
+
+	local->transaction.type = type;
+	ret = afr_inode_read_subvol_type_get (inode, this, local->readable,
+					      &event_generation, type);
+	if (ret == -1)
+		/* very first transaction on this inode */
+		goto refresh;
+
+	if (local->event_generation != event_generation)
+		/* servers have disconnected / reconnected, and possibly
+		   rebooted, very likely changing the state of freshness
+		   of copies */
+		goto refresh;
+
+	read_subvol = afr_read_subvol_select_by_policy (inode, this,
+							local->readable);
+
+	if (read_subvol < 0 || read_subvol > priv->child_count) {
+		gf_log (this->name, GF_LOG_WARNING, "Unreadable subvolume %d "
+			"found with event generation %d", read_subvol,
+			event_generation);
+		goto refresh;
+	}
+
+	if (!local->child_up[read_subvol]) {
+		/* should never happen, just in case */
+		gf_log (this->name, GF_LOG_WARNING, "subvolume %d is the "
+			"read subvolume in this generation, but is not up",
+			read_subvol);
+		goto refresh;
+	}
+
+	local->read_attempted[read_subvol] = 1;
+
+	local->readfn (frame, this, read_subvol);
+
+	return 0;
+
+refresh:
+	afr_inode_refresh (frame, this, inode, afr_read_txn_refresh_done);
+
+	return 0;
+}
diff --git a/xlators/cluster/afr/src/afr-self-heal-algorithm.c b/xlators/cluster/afr/src/afr-self-heal-algorithm.c
deleted file mode 100644
index 804783201..000000000
--- a/xlators/cluster/afr/src/afr-self-heal-algorithm.c
+++ /dev/null
@@ -1,1072 +0,0 @@
-/*
-   Copyright (c) 2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-
-#include "glusterfs.h"
-#include "afr.h"
-#include "xlator.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-#include "md5.h"
-
-#include "afr-transaction.h"
-#include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-#include "afr-self-heal-algorithm.h"
-
-/*
-  This file contains the various self-heal algorithms
-*/
-
-
-/*
-  The "full" algorithm. Copies the entire file from
-  source to sinks.
-*/
-
-
-static void
-sh_full_private_cleanup (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *               local   = NULL;
-        afr_self_heal_t *           sh      = NULL;
-        afr_sh_algo_full_private_t *sh_priv = NULL;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        sh_priv = sh->private;
-
-        if (sh_priv)
-                GF_FREE (sh_priv);
-}
-
-
-static int
-sh_full_loop_driver (call_frame_t *frame, xlator_t *this);
-
-static int
-sh_full_loop_return (call_frame_t *rw_frame, xlator_t *this, off_t offset)
-{
-        afr_local_t *               rw_local   = NULL;
-        afr_self_heal_t *           rw_sh      = NULL;
-
-        call_frame_t *sh_frame              = NULL;
-	afr_local_t * sh_local              = NULL;
-	afr_self_heal_t *sh                 = NULL;
-        afr_sh_algo_full_private_t *sh_priv = NULL;
-
-        rw_local = rw_frame->local;
-        rw_sh    = &rw_local->self_heal;
-
-        sh_frame = rw_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-        sh_priv  = sh->private;
-
-        LOCK (&sh_priv->lock);
-        {
-                sh_priv->loops_running--;
-        }
-        UNLOCK (&sh_priv->lock);
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "loop for offset %"PRId64" returned", offset);
-
-        AFR_STACK_DESTROY (rw_frame);
-
-        sh_full_loop_driver (sh_frame, this);
-
-        return 0;
-}
-
-
-static int
-sh_full_write_cbk (call_frame_t *rw_frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
-{
-	afr_private_t * priv    = NULL;
-	afr_local_t * rw_local  = NULL;
-	afr_self_heal_t *rw_sh  = NULL;
-
-        call_frame_t *sh_frame  = NULL;
-	afr_local_t * sh_local  = NULL;
-	afr_self_heal_t *sh     = NULL;
-
-	int child_index = (long) cookie;
-	int call_count = 0;
-
-	priv = this->private;
-
-	rw_local = rw_frame->local;
-	rw_sh    = &rw_local->self_heal;
-
-        sh_frame = rw_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"wrote %d bytes of data from %s to child %d, offset %"PRId64"",
-		op_ret, sh_local->loc.path, child_index,
-                rw_sh->offset - op_ret);
-
-	LOCK (&sh_frame->lock);
-	{
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"write to %s failed on subvolume %s (%s)",
-				sh_local->loc.path,
-				priv->children[child_index]->name,
-				strerror (op_errno));
-
-			sh->op_failed = 1;
-		}
-	}
-	UNLOCK (&sh_frame->lock);
-
-	call_count = afr_frame_return (rw_frame);
-
-	if (call_count == 0) {
-		sh_full_loop_return (rw_frame, this, rw_sh->offset - op_ret);
-	}
-
-	return 0;
-}
-
-
-static int
-sh_full_read_cbk (call_frame_t *rw_frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  struct iovec *vector, int32_t count, struct iatt *buf,
-                  struct iobref *iobref)
-{
-	afr_private_t * priv    = NULL;
-	afr_local_t * rw_local  = NULL;
-	afr_self_heal_t *rw_sh  = NULL;
-
-        call_frame_t *sh_frame  = NULL;
-	afr_local_t * sh_local  = NULL;
-	afr_self_heal_t *sh     = NULL;
-
-	int i = 0;
-	int call_count = 0;
-
-	off_t offset = (long) cookie;
-
-	priv = this->private;
-	rw_local = rw_frame->local;
-	rw_sh    = &rw_local->self_heal;
-
-        sh_frame = rw_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-
-	call_count = sh->active_sinks;
-
-        rw_local->call_count = call_count;
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"read %d bytes of data from %s, offset %"PRId64"",
-		op_ret, sh_local->loc.path, offset);
-
-	if (op_ret <= 0) {
-                sh->op_failed = 1;
-
-                sh_full_loop_return (rw_frame, this, offset);
-		return 0;
-	}
-
-	rw_sh->offset += op_ret;
-
-	if (sh->file_has_holes) {
-		if (iov_0filled (vector, count) == 0) {
-			/* the iter function depends on the
-			   sh->offset already being updated
-			   above
-			*/
-
-                        sh_full_loop_return (rw_frame, this, offset);
-			goto out;
-		}
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->sources[i] || !sh_local->child_up[i])
-			continue;
-
-		/* this is a sink, so write to it */
-
-		STACK_WIND_COOKIE (rw_frame, sh_full_write_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->writev,
-				   sh->healing_fd, vector, count, offset,
-                                   iobref);
-
-		if (!--call_count)
-			break;
-	}
-
-out:
-	return 0;
-}
-
-
-static int
-sh_full_read_write (call_frame_t *frame, xlator_t *this, off_t offset)
-{
-	afr_private_t * priv    = NULL;
-	afr_local_t * local     = NULL;
-	afr_local_t * rw_local  = NULL;
-	afr_self_heal_t *rw_sh  = NULL;
-	afr_self_heal_t *sh     = NULL;
-
-        call_frame_t *rw_frame = NULL;
-
-        int32_t op_errno = 0;
-
-	priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        rw_frame = copy_frame (frame);
-        if (!rw_frame)
-                goto out;
-
-        ALLOC_OR_GOTO (rw_local, afr_local_t, out);
-
-        rw_frame->local = rw_local;
-	rw_sh           = &rw_local->self_heal;
-
-        rw_sh->offset       = offset;
-        rw_sh->sh_frame     = frame;
-
-	STACK_WIND_COOKIE (rw_frame, sh_full_read_cbk,
-			   (void *) (long) offset,
-			   priv->children[sh->source],
-			   priv->children[sh->source]->fops->readv,
-			   sh->healing_fd, sh->block_size,
-			   offset);
-        return 0;
-
-out:
-        sh->op_failed = 1;
-
-        sh_full_loop_driver (frame, this);
-
-	return 0;
-}
-
-
-static int
-sh_full_loop_driver (call_frame_t *frame, xlator_t *this)
-{
-	afr_private_t * priv = NULL;
-	afr_local_t * local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-        afr_sh_algo_full_private_t *sh_priv = NULL;
-
-        int   loop    = 0;
-        int   recurse = 0;
-
-        off_t offset  = 0;
-
-	priv    = this->private;
-	local   = frame->local;
-	sh      = &local->self_heal;
-        sh_priv = sh->private;
-
-	if (sh->op_failed) {
-                if (sh_priv->loops_running == 0) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "full self-heal aborting on %s",
-                                local->loc.path);
-
-                        sh_full_private_cleanup (frame, this);
-                        local->self_heal.algo_abort_cbk (frame, this);
-                }
-
-		goto out;
-	}
-
-	if (sh_priv->offset >= sh->file_size) {
-                if (sh_priv->loops_running == 0) {
-
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "full self-heal completed on %s",
-                                local->loc.path);
-
-                        sh_full_private_cleanup (frame, this);
-                        local->self_heal.algo_completion_cbk (frame, this);
-                }
-
-		goto out;
-	}
-
-spawn:
-        loop    = 0;
-        recurse = 0;
-
-        LOCK (&sh_priv->lock);
-        {
-                if ((sh_priv->loops_running < priv->data_self_heal_window_size)
-                    && (sh_priv->offset < sh->file_size)) {
-
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "spawning a loop for offset %"PRId64,
-                                sh_priv->offset);
-
-                        offset           = sh_priv->offset;
-                        sh_priv->offset += sh->block_size;
-
-                        sh_priv->loops_running++;
-
-                        loop = 1;
-
-                        if (sh_priv->offset < sh->file_size)
-                                recurse = 1;
-                }
-        }
-        UNLOCK (&sh_priv->lock);
-
-        if (loop) {
-                sh_full_read_write (frame, this, offset);
-                if (recurse)
-                        goto spawn;
-        }
-
-out:
-	return 0;
-}
-
-
-int
-afr_sh_algo_full (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t *               local   = NULL;
-        afr_self_heal_t *           sh      = NULL;
-        afr_sh_algo_full_private_t *sh_priv = NULL;
-
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        sh_priv = GF_CALLOC (1, sizeof (*sh_priv),
-                             gf_afr_mt_afr_private_t);
-
-        LOCK_INIT (&sh_priv->lock);
-
-        sh->private = sh_priv;
-
-        local->call_count = 0;
-
-        sh_full_loop_driver (frame, this);
-        return 0;
-}
-
-
-/*
- * The "diff" algorithm. Copies only those blocks whose checksums
- * don't match with those of source.
- */
-
-
-static void
-sh_diff_private_cleanup (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *             priv    = NULL;
-        afr_local_t *               local   = NULL;
-        afr_self_heal_t *           sh      = NULL;
-        afr_sh_algo_diff_private_t *sh_priv = NULL;
-
-        int i;
-
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        sh_priv = sh->private;
-
-        for (i = 0; i < priv->data_self_heal_window_size; i++) {
-                if (sh_priv->loops[i]) {
-                        if (sh_priv->loops[i]->write_needed)
-                                GF_FREE (sh_priv->loops[i]->write_needed);
-
-                        if (sh_priv->loops[i]->checksum)
-                                GF_FREE (sh_priv->loops[i]->checksum);
-
-                        GF_FREE (sh_priv->loops[i]);
-                }
-        }
-
-        if (sh_priv) {
-                if (sh_priv->loops)
-                        GF_FREE (sh_priv->loops);
-
-                GF_FREE (sh_priv);
-        }
-
-
-}
-
-
-static uint32_t
-__make_cookie (int loop_index, int child_index)
-{
-        uint32_t ret = (loop_index << 16) | child_index;
-        return ret;
-}
-
-
-static int
-__loop_index (uint32_t cookie)
-{
-        return (cookie & 0xFFFF0000) >> 16;
-}
-
-
-static int
-__child_index (uint32_t cookie)
-{
-        return (cookie & 0x0000FFFF);
-}
-
-
-static void
-sh_diff_loop_state_reset (struct sh_diff_loop_state *loop_state, int child_count)
-{
-        loop_state->active = _gf_false;
-//        loop_state->offset = 0;
-
-        memset (loop_state->write_needed,
-                0, sizeof (*loop_state->write_needed) * child_count);
-
-        memset (loop_state->checksum,
-                0, MD5_DIGEST_LEN * child_count);
-}
-
-
-static int
-sh_diff_number_of_writes_needed (unsigned char *write_needed, int child_count)
-{
-        int writes = 0;
-        int i;
-
-        for (i = 0; i < child_count; i++) {
-                if (write_needed[i])
-                        writes++;
-        }
-
-        return writes;
-}
-
-
-static int
-sh_diff_loop_driver (call_frame_t *frame, xlator_t *this);
-
-
-static int
-sh_diff_loop_return (call_frame_t *rw_frame, xlator_t *this,
-                     struct sh_diff_loop_state *loop_state)
-{
-        afr_private_t *             priv       = NULL;
-        afr_local_t *               rw_local   = NULL;
-        afr_self_heal_t *           rw_sh      = NULL;
-
-        call_frame_t *sh_frame              = NULL;
-	afr_local_t * sh_local              = NULL;
-	afr_self_heal_t *sh                 = NULL;
-        afr_sh_algo_diff_private_t *sh_priv = NULL;
-
-        priv  = this->private;
-
-        rw_local = rw_frame->local;
-        rw_sh    = &rw_local->self_heal;
-
-        sh_frame = rw_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-        sh_priv  = sh->private;
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "loop for offset %"PRId64" returned", loop_state->offset);
-
-        LOCK (&sh_priv->lock);
-        {
-                sh_priv->loops_running--;
-                sh_diff_loop_state_reset (loop_state, priv->child_count);
-        }
-        UNLOCK (&sh_priv->lock);
-
-        AFR_STACK_DESTROY (rw_frame);
-
-        sh_diff_loop_driver (sh_frame, this);
-
-        return 0;
-}
-
-
-static int
-sh_diff_write_cbk (call_frame_t *rw_frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
-                   struct iatt *postbuf)
-{
-	afr_private_t *   priv     = NULL;
-	afr_local_t *     rw_local = NULL;
-	afr_self_heal_t * rw_sh    = NULL;
-
-        call_frame_t *sh_frame  = NULL;
-	afr_local_t * sh_local  = NULL;
-	afr_self_heal_t *sh     = NULL;
-
-        afr_sh_algo_diff_private_t *sh_priv;
-        struct sh_diff_loop_state *loop_state;
-
-	int call_count  = 0;
-        int child_index = 0;
-        int loop_index  = 0;
-
-	priv     = this->private;
-	rw_local = rw_frame->local;
-	rw_sh    = &rw_local->self_heal;
-
-        sh_frame = rw_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-        sh_priv  = sh->private;
-
-        child_index = __child_index ((uint32_t) (long) cookie);
-        loop_index  = __loop_index ((uint32_t) (long) cookie);
-        loop_state  = sh_priv->loops[loop_index];
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"wrote %d bytes of data from %s to child %d, offset %"PRId64"",
-		op_ret, sh_local->loc.path, child_index,
-                loop_state->offset);
-
-	LOCK (&sh_frame->lock);
-	{
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"write to %s failed on subvolume %s (%s)",
-				sh_local->loc.path,
-				priv->children[child_index]->name,
-				strerror (op_errno));
-
-			sh->op_failed = 1;
-		}
-	}
-	UNLOCK (&sh_frame->lock);
-
-	call_count = afr_frame_return (rw_frame);
-
-	if (call_count == 0) {
-		sh_diff_loop_return (rw_frame, this, loop_state);
-	}
-
-	return 0;
-}
-
-
-static int
-sh_diff_read_cbk (call_frame_t *rw_frame, void *cookie,
-                  xlator_t *this, int32_t op_ret, int32_t op_errno,
-                  struct iovec *vector, int32_t count, struct iatt *buf,
-                  struct iobref *iobref)
-{
-	afr_private_t *   priv     = NULL;
-	afr_local_t *     rw_local = NULL;
-	afr_self_heal_t * rw_sh    = NULL;
-
-        afr_sh_algo_diff_private_t * sh_priv = NULL;
-
-        call_frame_t *sh_frame  = NULL;
-	afr_local_t * sh_local  = NULL;
-	afr_self_heal_t *sh     = NULL;
-
-        int loop_index;
-        struct sh_diff_loop_state *loop_state;
-
-        uint32_t wcookie;
-
-	int i = 0;
-	int call_count = 0;
-
-	priv     = this->private;
-	rw_local = rw_frame->local;
-	rw_sh    = &rw_local->self_heal;
-
-        sh_frame = rw_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-        sh_priv  = sh->private;
-
-        loop_index = __loop_index ((uint32_t) (long) cookie);
-        loop_state = sh_priv->loops[loop_index];
-
-	call_count = sh_diff_number_of_writes_needed (loop_state->write_needed,
-                                                      priv->child_count);
-
-	rw_local->call_count = call_count;
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"read %d bytes of data from %s, offset %"PRId64"",
-		op_ret, sh_local->loc.path, loop_state->offset);
-
-	if ((op_ret <= 0) ||
-            (call_count == 0)) {
-                sh_diff_loop_return (rw_frame, this, loop_state);
-
-		return 0;
-	}
-
-	if (sh->file_has_holes) {
-		if (iov_0filled (vector, count) == 0) {
-
-                        sh_diff_loop_return (rw_frame, this, loop_state);
-			goto out;
-		}
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-                if (loop_state->write_needed[i]) {
-                        wcookie = __make_cookie (loop_index, i);
-
-                        STACK_WIND_COOKIE (rw_frame, sh_diff_write_cbk,
-                                           (void *) (long) wcookie,
-                                           priv->children[i],
-                                           priv->children[i]->fops->writev,
-                                           sh->healing_fd, vector, count,
-                                           loop_state->offset, iobref);
-
-                        if (!--call_count)
-                                break;
-                }
-        }
-
-out:
-	return 0;
-}
-
-
-static int
-sh_diff_read (call_frame_t *rw_frame, xlator_t *this,
-              int loop_index)
-{
-	afr_private_t *   priv     = NULL;
-	afr_local_t *     rw_local = NULL;
-	afr_self_heal_t * rw_sh    = NULL;
-
-        afr_sh_algo_diff_private_t * sh_priv = NULL;
-        struct sh_diff_loop_state *loop_state;
-
-        call_frame_t *sh_frame  = NULL;
-	afr_local_t * sh_local  = NULL;
-	afr_self_heal_t *sh     = NULL;
-
-        uint32_t cookie;
-
-	priv     = this->private;
-	rw_local = rw_frame->local;
-	rw_sh    = &rw_local->self_heal;
-
-        sh_frame = rw_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-        sh_priv  = sh->private;
-
-        loop_state = sh_priv->loops[loop_index];
-
-        cookie = __make_cookie (loop_index, sh->source);
-
-	STACK_WIND_COOKIE (rw_frame, sh_diff_read_cbk,
-			   (void *) (long) cookie,
-			   priv->children[sh->source],
-			   priv->children[sh->source]->fops->readv,
-			   sh->healing_fd, sh_priv->block_size,
-			   loop_state->offset);
-
-	return 0;
-}
-
-
-static int
-sh_diff_checksum_cbk (call_frame_t *rw_frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno,
-                      uint32_t weak_checksum, uint8_t *strong_checksum)
-{
-	afr_private_t * priv    = NULL;
-	afr_local_t * rw_local  = NULL;
-	afr_self_heal_t *rw_sh  = NULL;
-
-        call_frame_t *sh_frame  = NULL;
-	afr_local_t * sh_local  = NULL;
-	afr_self_heal_t *sh     = NULL;
-
-        afr_sh_algo_diff_private_t * sh_priv = NULL;
-
-        int loop_index = 0;
-        int child_index = 0;
-        struct sh_diff_loop_state *loop_state;
-
-        int call_count   = 0;
-        int i            = 0;
-        int write_needed = 0;
-
-	priv  = this->private;
-
-	rw_local = rw_frame->local;
-	rw_sh    = &rw_local->self_heal;
-
-        sh_frame = rw_sh->sh_frame;
-        sh_local = sh_frame->local;
-        sh       = &sh_local->self_heal;
-
-        sh_priv = sh->private;
-
-        child_index = __child_index ((uint32_t) (long) cookie);
-        loop_index  = __loop_index ((uint32_t) (long) cookie);
-
-        loop_state  = sh_priv->loops[loop_index];
-
-        if (op_ret < 0) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "checksum on %s failed on subvolume %s (%s)",
-                        sh_local->loc.path, priv->children[child_index]->name,
-                        strerror (op_errno));
-
-                sh->op_failed = 1;
-        } else {
-                memcpy (loop_state->checksum + child_index * MD5_DIGEST_LEN,
-                        strong_checksum,
-                        MD5_DIGEST_LEN);
-        }
-
-        call_count = afr_frame_return (rw_frame);
-
-        if (call_count == 0) {
-                for (i = 0; i < priv->child_count; i++) {
-                        if (sh->sources[i] || !sh_local->child_up[i])
-                                continue;
-
-                        if (memcmp (loop_state->checksum + (i * MD5_DIGEST_LEN),
-                                    loop_state->checksum + (sh->source * MD5_DIGEST_LEN),
-                                    MD5_DIGEST_LEN)) {
-                                /*
-                                   Checksums differ, so this block
-                                   must be written to this sink
-                                */
-
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "checksum on subvolume %s at offset %"
-                                        PRId64" differs from that on source",
-                                        priv->children[i]->name, loop_state->offset);
-
-                                write_needed = loop_state->write_needed[i] = 1;
-                        }
-                }
-
-                LOCK (&sh_priv->lock);
-                {
-                        sh_priv->total_blocks++;
-                        if (write_needed)
-                                sh_priv->diff_blocks++;
-                }
-                UNLOCK (&sh_priv->lock);
-
-                if (write_needed && !sh->op_failed) {
-                        sh_diff_read (rw_frame, this, loop_index);
-                } else {
-                        sh->offset += sh_priv->block_size;
-
-                        sh_diff_loop_return (rw_frame, this, loop_state);
-                }
-        }
-
-        return 0;
-}
-
-
-static int
-sh_diff_find_unused_loop (afr_sh_algo_diff_private_t *sh_priv, int max)
-{
-        int i;
-
-        LOCK (&sh_priv->lock);
-        {
-                for (i = 0; i < max; i++) {
-                        if (sh_priv->loops[i]->active == _gf_false) {
-                                sh_priv->loops[i]->active = _gf_true;
-                                break;
-                        }
-                }
-        }
-        UNLOCK (&sh_priv->lock);
-
-        if (i == max) {
-                gf_log ("[sh-diff]", GF_LOG_ERROR,
-                        "no free loops found! This shouldn't happen. Please"
-                        " report this to gluster-devel@nongnu.org");
-        }
-
-        return i;
-}
-
-
-static int
-sh_diff_checksum (call_frame_t *frame, xlator_t *this, off_t offset)
-{
-	afr_private_t *   priv     = NULL;
-	afr_local_t *     local    = NULL;
-	afr_local_t *     rw_local = NULL;
-	afr_self_heal_t * sh       = NULL;
-	afr_self_heal_t * rw_sh    = NULL;
-
-        afr_sh_algo_diff_private_t * sh_priv = NULL;
-
-        call_frame_t *rw_frame = NULL;
-
-        uint32_t cookie;
-        int loop_index = 0;
-        struct sh_diff_loop_state *loop_state = NULL;
-
-        int32_t op_errno = 0;
-
-        int call_count = 0;
-        int i          = 0;
-
-	priv    = this->private;
-	local   = frame->local;
-	sh      = &local->self_heal;
-
-        sh_priv = sh->private;
-
-        rw_frame = copy_frame (frame);
-        if (!rw_frame)
-                goto out;
-
-        ALLOC_OR_GOTO (rw_local, afr_local_t, out);
-
-        rw_frame->local = rw_local;
-	rw_sh           = &rw_local->self_heal;
-
-        rw_sh->offset       = sh->offset;
-        rw_sh->sh_frame     = frame;
-
-        call_count = sh->active_sinks + 1;  /* sinks and source */
-
-        rw_local->call_count = call_count;
-
-        loop_index = sh_diff_find_unused_loop (sh_priv, priv->data_self_heal_window_size);
-
-        loop_state = sh_priv->loops[loop_index];
-        loop_state->offset       = offset;
-
-        /* we need to send both the loop index and child index,
-           so squeeze them both into a 32-bit number */
-
-        cookie = __make_cookie (loop_index, sh->source);
-
-        STACK_WIND_COOKIE (rw_frame, sh_diff_checksum_cbk,
-                           (void *) (long) cookie,
-                           priv->children[sh->source],
-                           priv->children[sh->source]->fops->rchecksum,
-                           sh->healing_fd,
-                           offset, sh_priv->block_size);
-
-        for (i = 0; i < priv->child_count; i++) {
-                if (sh->sources[i] || !local->child_up[i])
-                        continue;
-
-                cookie = __make_cookie (loop_index, i);
-
-                STACK_WIND_COOKIE (rw_frame, sh_diff_checksum_cbk,
-                                   (void *) (long) cookie,
-                                   priv->children[i],
-                                   priv->children[i]->fops->rchecksum,
-                                   sh->healing_fd,
-                                   offset, sh_priv->block_size);
-
-                if (!--call_count)
-                        break;
-        }
-
-        return 0;
-
-out:
-        sh->op_failed = 1;
-
-        sh_diff_loop_driver (frame, this);
-
-        return 0;
-}
-
-
-static int
-sh_diff_loop_driver (call_frame_t *frame, xlator_t *this)
-{
-	afr_private_t *   priv  = NULL;
-	afr_local_t *     local = NULL;
-	afr_self_heal_t * sh    = NULL;
-        afr_sh_algo_diff_private_t *sh_priv = NULL;
-
-        int   loop    = 0;
-        int   recurse = 0;
-
-        off_t offset = 0;
-        char  sh_type_str[256] = {0,};
- 
-	priv    = this->private;
-	local   = frame->local;
-	sh      = &local->self_heal;
-        sh_priv = sh->private;
-
-        afr_self_heal_type_str_get(sh, sh_type_str, sizeof(sh_type_str));
-
-	if (sh->op_failed) {
-                if (sh_priv->loops_running == 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "diff %s self-heal aborting on %s",
-                                sh_type_str, local->loc.path);
-
-                        sh_diff_private_cleanup (frame, this);
-                        local->self_heal.algo_abort_cbk (frame, this);
-                }
-
-		goto out;
-	}
-
-	if (sh_priv->offset >= sh->file_size) {
-                if (sh_priv->loops_running == 0) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "diff %s self-heal completed on %s",
-                                sh_type_str, local->loc.path);
-
-
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "diff %s self-heal on %s: %d blocks of %d were different (%.2f%%)",
-                                sh_type_str, local->loc.path,
-                                sh_priv->diff_blocks, sh_priv->total_blocks,
-                                ((sh_priv->diff_blocks * 1.0)/sh_priv->total_blocks) * 100);
-
-                        sh_diff_private_cleanup (frame, this);
-                        local->self_heal.algo_completion_cbk (frame, this);
-                }
-
-		goto out;
-	}
-
-spawn:
-        loop    = 0;
-        recurse = 0;
-
-        LOCK (&sh_priv->lock);
-        {
-                if ((sh_priv->loops_running < priv->data_self_heal_window_size)
-                    && (sh_priv->offset < sh->file_size)) {
-
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "spawning a loop for offset %"PRId64,
-                                sh_priv->offset);
-
-                        offset           = sh_priv->offset;
-                        sh_priv->offset += sh_priv->block_size;
-
-                        sh_priv->loops_running++;
-
-                        loop = 1;
-
-                        if (sh_priv->offset < sh->file_size)
-                                recurse = 1;
-                }
-        }
-        UNLOCK (&sh_priv->lock);
-
-        if (loop) {
-                sh_diff_checksum (frame, this, offset);
-                if (recurse)
-                    goto spawn;
-        }
-
-out:
-        return 0;
-}
-
-
-int
-afr_sh_algo_diff (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *             priv    = NULL;
-        afr_local_t *               local   = NULL;
-        afr_self_heal_t *           sh      = NULL;
-        afr_sh_algo_diff_private_t *sh_priv = NULL;
-
-        int i;
-
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-
-        sh_priv = GF_CALLOC (1, sizeof (*sh_priv),
-                             gf_afr_mt_afr_private_t);
-
-        sh_priv->block_size = this->ctx->page_size;
-
-        sh->private = sh_priv;
-
-        LOCK_INIT (&sh_priv->lock);
-
-        local->call_count = 0;
-
-        sh_priv->loops = GF_CALLOC (priv->data_self_heal_window_size,
-                                    sizeof (*sh_priv->loops),
-                                    gf_afr_mt_sh_diff_loop_state);
-
-        for (i = 0; i < priv->data_self_heal_window_size; i++) {
-                sh_priv->loops[i]               = GF_CALLOC (1, sizeof (*sh_priv->loops[i]),
-                                                             gf_afr_mt_sh_diff_loop_state);
-
-                sh_priv->loops[i]->checksum     = GF_CALLOC (priv->child_count,
-                                                             MD5_DIGEST_LEN, gf_afr_mt_uint8_t);
-                sh_priv->loops[i]->write_needed = GF_CALLOC (priv->child_count,
-                                                             sizeof (*sh_priv->loops[i]->write_needed),
-                                                             gf_afr_mt_char);
-        }
-
-        sh_diff_loop_driver (frame, this);
-
-        return 0;
-}
-
-
-struct afr_sh_algorithm afr_self_heal_algorithms[] = {
-        {.name = "full",  .fn = afr_sh_algo_full},
-        {.name = "diff",  .fn = afr_sh_algo_diff},
-        {0, 0},
-};
diff --git a/xlators/cluster/afr/src/afr-self-heal-algorithm.h b/xlators/cluster/afr/src/afr-self-heal-algorithm.h
deleted file mode 100644
index 4b27d2afe..000000000
--- a/xlators/cluster/afr/src/afr-self-heal-algorithm.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
-   Copyright (c) 2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef __AFR_SELF_HEAL_ALGORITHM_H__
-#define __AFR_SELF_HEAL_ALGORITHM_H__
-
-
-typedef int (*afr_sh_algo_fn) (call_frame_t *frame,
-                               xlator_t *this);
-
-struct afr_sh_algorithm {
-        const char *name;
-        afr_sh_algo_fn fn;
-};
-
-extern struct afr_sh_algorithm afr_self_heal_algorithms[3];
-
-typedef struct {
-        gf_lock_t lock;
-        unsigned int loops_running;
-        off_t offset;
-} afr_sh_algo_full_private_t;
-
-struct sh_diff_loop_state {
-        off_t   offset;
-        unsigned char *write_needed;
-        uint8_t *checksum;
-        gf_boolean_t active;
-};
-
-typedef struct {
-        size_t block_size;
-
-        gf_lock_t lock;
-        unsigned int loops_running;
-        off_t offset;
-
-        int32_t total_blocks;
-        int32_t diff_blocks;
-
-        struct sh_diff_loop_state **loops;
-} afr_sh_algo_diff_private_t;
-
-#endif /* __AFR_SELF_HEAL_ALGORITHM_H__ */
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.c b/xlators/cluster/afr/src/afr-self-heal-common.c
index 83be5ed8c..4dac83113 100644
--- a/xlators/cluster/afr/src/afr-self-heal-common.c
+++ b/xlators/cluster/afr/src/afr-self-heal-common.c
@@ -1,1657 +1,1009 @@
 /*
-  Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#include "glusterfs.h"
-#include "xlator.h"
-#include "byte-order.h"
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
 
 #include "afr.h"
-#include "afr-transaction.h"
-#include "afr-self-heal-common.h"
 #include "afr-self-heal.h"
-#include "pump.h"
+#include "byte-order.h"
 
-/**
- * select_source - select a source and return it
- */
 
 int
-afr_sh_select_source (int sources[], int child_count)
+afr_selfheal_post_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			  int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-	int i;
-	for (i = 0; i < child_count; i++)
-		if (sources[i])
-			return i;
-
-	return -1;
-}
+	afr_local_t *local = NULL;
 
+	local = frame->local;
 
-/**
- * sink_count - return number of sinks in sources array
- */
+	syncbarrier_wake (&local->barrier);
 
-int
-afr_sh_sink_count (int sources[], int child_count)
-{
-	int i;
-	int sinks = 0;
-	for (i = 0; i < child_count; i++)
-		if (!sources[i])
-			sinks++;
-	return sinks;
+	return 0;
 }
 
+
 int
-afr_sh_source_count (int sources[], int child_count)
+afr_selfheal_post_op (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		      int subvol, dict_t *xattr)
 {
-	int i;
-	int nsource = 0;
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
+	loc_t loc = {0, };
 
-	for (i = 0; i < child_count; i++)
-		if (sources[i])
-			nsource++;
-	return nsource;
-}
+	priv = this->private;
+	local = frame->local;
 
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-int
-afr_sh_supress_errenous_children (int sources[], int child_errno[],
-				  int child_count)
-{
-	int i = 0;
+	STACK_WIND (frame, afr_selfheal_post_op_cbk, priv->children[subvol],
+		    priv->children[subvol]->fops->xattrop, &loc,
+		    GF_XATTROP_ADD_ARRAY, xattr, NULL);
 
-	for (i = 0; i < child_count; i++) {
-		if (child_errno[i] && sources[i]) {
-			sources[i] = 0;
-		}
-	}
+	syncbarrier_wait (&local->barrier, 1);
 
 	return 0;
 }
 
 
-void
-afr_sh_print_pending_matrix (int32_t *pending_matrix[], xlator_t *this)
+dict_t *
+afr_selfheal_output_xattr (xlator_t *this, afr_transaction_type type,
+			   int *output_dirty, int **output_matrix, int subvol)
 {
-	afr_private_t * priv = this->private;
+	dict_t *xattr = NULL;
+	afr_private_t *priv = NULL;
+	int j = 0;
+	int idx = 0;
+	int ret = 0;
+	int *raw = 0;
+
+	priv = this->private;
+	idx = afr_index_for_transaction_type (type);
+
+	xattr = dict_new ();
+	if (!xattr)
+		return NULL;
+
+	if (output_dirty[subvol]) {
+		/* clear dirty */
+		raw = GF_CALLOC (sizeof(int), AFR_NUM_CHANGE_LOGS, gf_afr_mt_int32_t);
+		if (!raw)
+			goto err;
+
+		raw[idx] = hton32 (output_dirty[subvol]);
+		ret = dict_set_bin (xattr, AFR_DIRTY, raw,
+				    sizeof(int) * AFR_NUM_CHANGE_LOGS);
+		if (ret)
+			goto err;
+	}
 
-	char *buf = NULL;
-	char *ptr = NULL;
+	/* clear/set pending */
+	for (j = 0; j < priv->child_count; j++) {
+		if (!output_matrix[subvol][j])
+			continue;
 
-	int i, j;
+		raw = GF_CALLOC (sizeof(int), AFR_NUM_CHANGE_LOGS,
+				 gf_afr_mt_int32_t);
+		if (!raw)
+			goto err;
 
-        /* 10 digits per entry + 1 space + '[' and ']' */
-	buf = GF_MALLOC (priv->child_count * 11 + 8, gf_afr_mt_char);
+		raw[idx] = hton32 (output_matrix[subvol][j]);
 
-	for (i = 0; i < priv->child_count; i++) {
-		ptr = buf;
-		ptr += sprintf (ptr, "[ ");
-		for (j = 0; j < priv->child_count; j++) {
-			ptr += sprintf (ptr, "%d ", pending_matrix[i][j]);
-		}
-		sprintf (ptr, "]");
-		gf_log (this->name, GF_LOG_TRACE,
-			"pending_matrix: %s", buf);
+		ret = dict_set_bin (xattr, priv->pending_key[j],
+				    raw, sizeof(int) * AFR_NUM_CHANGE_LOGS);
+		if (ret)
+			goto err;
 	}
 
-	GF_FREE (buf);
+	return xattr;
+err:
+	if (xattr)
+		dict_unref (xattr);
+	return NULL;
 }
 
 
-void
-afr_sh_build_pending_matrix (afr_private_t *priv,
-                             int32_t *pending_matrix[], dict_t *xattr[],
-			     int child_count, afr_transaction_type type)
+int
+afr_selfheal_undo_pending (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			   unsigned char *sources, unsigned char *sinks,
+			   unsigned char *healed_sinks, afr_transaction_type type,
+			   struct afr_reply *replies, unsigned char *locked_on)
 {
-	int i, j, k;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int j = 0;
+	unsigned char *pending = NULL;
+	int *input_dirty = NULL;
+	int **input_matrix = NULL;
+	int *output_dirty = NULL;
+	int **output_matrix = NULL;
+	dict_t *xattr = NULL;
 
-	/* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-	int32_t pending[3];
-	void *pending_raw = NULL;
-	int ret = -1;
+	priv = this->private;
+
+	pending = alloca0 (priv->child_count);
+
+	input_dirty = alloca0 (priv->child_count * sizeof (int));
+	input_matrix = ALLOC_MATRIX (priv->child_count, int);
+	output_dirty = alloca0 (priv->child_count * sizeof (int));
+	output_matrix = ALLOC_MATRIX (priv->child_count, int);
 
-        unsigned char *ignorant_subvols = NULL;
+	afr_selfheal_extract_xattr (this, replies, type, input_dirty,
+				    input_matrix);
 
-        ignorant_subvols = GF_CALLOC (sizeof (*ignorant_subvols), child_count,
-                                      gf_afr_mt_char);
+	for (i = 0; i < priv->child_count; i++)
+		if (sinks[i] && !healed_sinks[i])
+			pending[i] = 1;
 
-	/* start clean */
-	for (i = 0; i < child_count; i++) {
-		for (j = 0; j < child_count; j++) {
-			pending_matrix[i][j] = 0;
+	for (i = 0; i < priv->child_count; i++) {
+		for (j = 0; j < priv->child_count; j++) {
+			if (pending[j])
+				output_matrix[i][j] = 1;
+			else
+				output_matrix[i][j] = -input_matrix[i][j];
 		}
 	}
 
-	for (i = 0; i < child_count; i++) {
-		pending_raw = NULL;
-
-                for (j = 0; j < child_count; j++) {
-                        ret = dict_get_ptr (xattr[i], priv->pending_key[j],
-                                            &pending_raw);
-                        
-                        if (ret != 0) {
-                                /*
-                                 * There is no xattr present. This means this
-                                 * subvolume should be considered an 'ignorant'
-                                 * subvolume.
-                                 */
-
-                                ignorant_subvols[i] = 1;
-                                continue;
-                        }
-
-			memcpy (pending, pending_raw, sizeof(pending));
-                        k = afr_index_for_transaction_type (type);
-                        
-                        pending_matrix[i][j] = ntoh32 (pending[k]);
-                }
+	for (i = 0; i < priv->child_count; i++) {
+		if (!pending[i])
+			output_dirty[i] = -input_dirty[i];
 	}
 
-        /*
-         * Make all non-ignorant subvols point towards the ignorant
-         * subvolumes.
-         */
-
-        for (i = 0; i < child_count; i++) {
-                if (ignorant_subvols[i]) {
-                        for (j = 0; j < child_count; j++) {
-                                if (!ignorant_subvols[j])
-                                        pending_matrix[j][i] += 1;
-                        }
-                }
-        }
-
-        GF_FREE (ignorant_subvols);
-}
-
-
-/**
- * mark_sources: Mark all 'source' nodes and return number of source
- * nodes found
- *
- * A node (a row in the pending matrix) belongs to one of
- * three categories:
- *
- * M is the pending matrix.
- *
- * 'innocent' - M[i] is all zeroes
- * 'fool'     - M[i] has i'th element = 1 (self-reference)
- * 'wise'     - M[i] has i'th element = 0, others are 1 or 0.
- *
- * All 'innocent' nodes are sinks. If all nodes are innocent, no self-heal is
- * needed.
- *
- * A 'wise' node can be a source. If two 'wise' nodes conflict, it is 
- * a split-brain. If one wise node refers to the other but the other doesn't
- * refer back, the referrer is a source.
- *
- * All fools are sinks, unless there are no 'wise' nodes. In that case,
- * one of the fools is made a source.
- */
-
-typedef enum {
-        AFR_NODE_INNOCENT,
-        AFR_NODE_FOOL,
-        AFR_NODE_WISE
-} afr_node_type;
-
-typedef struct {
-        afr_node_type type;
-        int           wisdom;
-} afr_node_character;
-
-
-static int
-afr_sh_is_innocent (int32_t *array, int child_count)
-{
-        int i   = 0;
-        int ret = 1;   /* innocent until proven guilty */
+	for (i = 0; i < priv->child_count; i++) {
+		if (!locked_on[i])
+			/* perform post-op only on subvols we had locked
+			   and inspected on.
+			*/
+			continue;
 
-        for (i = 0; i < child_count; i++) {
-                if (array[i]) {
-                        ret = 0;
-                        break;
-                }
-        }
+		xattr = afr_selfheal_output_xattr (this, type, output_dirty,
+						   output_matrix, i);
+		if (!xattr) {
+			gf_log (this->name, GF_LOG_ERROR,
+				"unable to allocate xdata for subvol %d", i);
+			continue;
+		}
 
-        return ret;
-}
+		afr_selfheal_post_op (frame, this, inode, i, xattr);
 
+		dict_unref (xattr);
+	}
 
-static int
-afr_sh_is_fool (int32_t *array, int i, int child_count)
-{
-        return array[i];   /* fool if accuses itself */
+	return 0;
 }
 
 
-static int
-afr_sh_is_wise (int32_t *array, int i, int child_count)
+void
+afr_replies_copy (struct afr_reply *dst, struct afr_reply *src, int count)
 {
-        return !array[i];  /* wise if does not accuse itself */
+	int i = 0;
+	dict_t *xdata = NULL;
+
+	if (dst == src)
+		return;
+
+	for (i = 0; i < count; i++) {
+		dst[i].valid = src[i].valid;
+		dst[i].op_ret = src[i].op_ret;
+		dst[i].op_errno = src[i].op_errno;
+		dst[i].prestat = src[i].prestat;
+		dst[i].poststat = src[i].poststat;
+		dst[i].preparent = src[i].preparent;
+		dst[i].postparent = src[i].postparent;
+		dst[i].preparent2 = src[i].preparent2;
+		dst[i].postparent2 = src[i].postparent2;
+		if (src[i].xdata)
+			xdata = dict_ref (src[i].xdata);
+		else
+			xdata = NULL;
+		if (dst[i].xdata)
+			dict_unref (dst[i].xdata);
+		dst[i].xdata = xdata;
+		memcpy (dst[i].checksum, src[i].checksum,
+			MD5_DIGEST_LENGTH);
+	}
 }
 
 
-static int
-afr_sh_all_nodes_innocent (afr_node_character *characters, 
-                           int child_count)
+int
+afr_selfheal_fill_dirty (xlator_t *this, int *dirty, int subvol,
+			 int idx, dict_t *xdata)
 {
-        int i   = 0;
-        int ret = 1;
-
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type != AFR_NODE_INNOCENT) {
-                        ret = 0;
-                        break;
-                }
-        }
+	void *pending_raw = NULL;
+	int pending[3] = {0, };
 
-        return ret;
-}
+	if (dict_get_ptr (xdata, AFR_DIRTY, &pending_raw))
+		return -1;
 
+	if (!pending_raw)
+		return -1;
 
-static int
-afr_sh_wise_nodes_exist (afr_node_character *characters, int child_count)
-{
-        int i   = 0;
-        int ret = 0;
+	memcpy (pending, pending_raw, sizeof(pending));
 
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type == AFR_NODE_WISE) {
-                        ret = 1;
-                        break;
-                }
-        }
+	dirty[subvol] = ntoh32 (pending[idx]);
 
-        return ret;
+	return 0;
 }
 
 
-/*
- * The 'wisdom' of a wise node is 0 if any other wise node accuses it.
- * It is 1 if no other wise node accuses it. 
- * Only wise nodes with wisdom 1 are sources.
- *
- * If no nodes with wisdom 1 exist, a split-brain has occured.
- */
-
-static void
-afr_sh_compute_wisdom (int32_t *pending_matrix[],
-                       afr_node_character characters[], int child_count)
+int
+afr_selfheal_fill_matrix (xlator_t *this, int **matrix, int subvol,
+			  int idx, dict_t *xdata)
 {
-        int i = 0;
-        int j = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type == AFR_NODE_WISE) {
-                        characters[i].wisdom = 1;
-
-                        for (j = 0; j < child_count; j++) {
-                                if ((characters[j].type == AFR_NODE_WISE)
-                                    && pending_matrix[j][i]) {
-                                        
-                                        characters[i].wisdom = 0;
-                                }
-                        }
-                }
-        }
-}
-
+	int i = 0;
+	void *pending_raw = NULL;
+	int pending[3] = {0, };
+	afr_private_t *priv = NULL;
 
-static int
-afr_sh_wise_nodes_conflict (afr_node_character *characters, 
-                            int child_count)
-{
-        int i   = 0;
-        int ret = 1;
+	priv = this->private;
 
-        for (i = 0; i < child_count; i++) {
-                if ((characters[i].type == AFR_NODE_WISE)
-                    && characters[i].wisdom == 1) {
+	for (i = 0; i < priv->child_count; i++) {
+		if (dict_get_ptr (xdata, priv->pending_key[i], &pending_raw))
+			continue;
 
-                        /* There is atleast one bona-fide wise node */
-                        ret = 0;
-                        break;
-                }
-        }
+		if (!pending_raw)
+			continue;
 
-        return ret;
-}
+		memcpy (pending, pending_raw, sizeof(pending));
 
+		matrix[subvol][i] = ntoh32 (pending[idx]);
+	}
 
-static int
-afr_sh_mark_wisest_as_sources (int sources[], 
-                               afr_node_character *characters, 
-                               int child_count)
-{
-        int nsources = 0;
-        
-        int i = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].wisdom == 1) {
-                        sources[i] = 1;
-                        nsources++;
-                }
-        }
-
-        return nsources;
+	return 0;
 }
 
 
-static int
-afr_sh_mark_if_size_differs (afr_self_heal_t *sh, int child_count)
+int
+afr_selfheal_extract_xattr (xlator_t *this, struct afr_reply *replies,
+			    afr_transaction_type type, int *dirty, int **matrix)
 {
-        int32_t ** pending_matrix;
-        int i, j;
-
-        int size_differs = 0;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	dict_t *xdata = NULL;
+	int idx = -1;
 
-        pending_matrix = sh->pending_matrix;
+	idx = afr_index_for_transaction_type (type);
 
-        for (i = 0; i < child_count; i++) {
-                for (j = 0; j < child_count; j++) {
-                        if (!sh->buf)
-                                break;
+	priv = this->private;
 
-                        if (SIZE_DIFFERS (&sh->buf[i], &sh->buf[j])
-                            && (pending_matrix[i][j] == 0)
-                            && (pending_matrix[j][i] == 0)) {
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].xdata)
+			continue;
 
-                                pending_matrix[i][j] = 1;
-                                pending_matrix[j][i] = 1;
+		xdata = replies[i].xdata;
 
-                                size_differs = 1;
-                        }
-                }
-        }
+		afr_selfheal_fill_dirty (this, dirty, i, idx, xdata);
+		afr_selfheal_fill_matrix (this, matrix, i, idx, xdata);
+	}
 
-        return size_differs;
+	return 0;
 }
 
-        
-static int
-afr_sh_mark_biggest_fool_as_source (afr_self_heal_t *sh,
-                                    afr_node_character *characters, 
-                                    int child_count)
-{
-        int i = 0;
-        int biggest = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type == AFR_NODE_FOOL) {
-                        biggest = i;
-                        break;
-                }
-        }
-
-        for (i = 0; i < child_count; i++) {
-                if (characters[i].type != AFR_NODE_FOOL)
-                        continue;
 
-                if (!sh->buf)
-                        break;
-
-                if (SIZE_GREATER (&sh->buf[i], &sh->buf[biggest])) {
-                        biggest = i;
-                }
-        }
-
-        sh->sources[biggest] = 1;
-
-        return 1;
-}
 
+/*
+ * This function determines if a self-heal is required for a given inode,
+ * and if needed, in what direction.
+ *
+ * locked_on[] is the array representing servers which have been locked and
+ * from which xattrs have been fetched for analysis.
+ *
+ * The output of the function is by filling the arrays sources[] and sinks[].
+ *
+ * sources[i] is set if i'th server is an eligible source for a selfheal.
+ *
+ * sinks[i] is set if i'th server needs to be healed.
+ *
+ * if sources[0..N] are all set, there is no need for a selfheal.
+ *
+ * if sinks[0..N] are all set, the inode is in split brain.
+ *
+ */
 
-static int
-afr_sh_mark_biggest_as_source (afr_self_heal_t *sh, int child_count)
+int
+afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
+			     struct afr_reply *replies,
+			     afr_transaction_type type, unsigned char *locked_on,
+			     unsigned char *sources, unsigned char *sinks)
 {
-        int biggest = 0;
-        int i;
-
-        for (i = 0; i < child_count; i++) {
-                if (!sh->buf)
-                        break;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int j = 0;
+	int *dirty = NULL;
+	int **matrix = NULL;
+	char *accused = NULL;
 
-                if (SIZE_GREATER (&sh->buf[i], &sh->buf[biggest])) {
-                        biggest = i;
-                }
-        }
+	priv = this->private;
 
-        sh->sources[biggest] = 1;
+	dirty = alloca0 (priv->child_count * sizeof (int));
+	accused = alloca0 (priv->child_count);
+	matrix = ALLOC_MATRIX(priv->child_count, int);
 
-        return 1;
-}
+	/* First construct the pending matrix for further analysis */
+	afr_selfheal_extract_xattr (this, replies, type, dirty, matrix);
 
+	/* Next short list all accused to exclude them from being sources */
+	for (i = 0; i < priv->child_count; i++) {
+		for (j = 0; j < priv->child_count; j++) {
+			if (matrix[i][j])
+				accused[j] = 1;
+		}
+	}
 
-static int
-afr_sh_mark_loweia_uid_as_source (afr_self_heal_t *sh, int child_count)
-{
-        uid_t smallest = 0;
-        int i;
+	/* Short list all non-accused as sources */
+	memset (sources, 0, priv->child_count);
+	for (i = 0; i < priv->child_count; i++) {
+		if (!accused[i] && locked_on[i])
+			sources[i] = 1;
+	}
 
-        for (i = 0; i < child_count; i++) {
-                if (!sh->buf)
-                        break;
+	/* Everyone accused by sources are sinks */
+	memset (sinks, 0, priv->child_count);
+	for (i = 0; i < priv->child_count; i++) {
+		if (!sources[i])
+			continue;
+		for (j = 0; j < priv->child_count; j++) {
+			if (matrix[i][j])
+				sinks[j] = 1;
+		}
+	}
 
-                if (sh->buf[i].ia_uid < sh->buf[smallest].ia_uid) {
-                        smallest = i;
-                }
-        }
+	/* If any source has 'dirty' bit, pick first
+	   'dirty' source and make everybody else sinks */
+	for (i = 0; i < priv->child_count; i++) {
+		if (sources[i] && dirty[i]) {
+			for (j = 0; j < priv->child_count; j++) {
+				if (j != i) {
+					sources[j] = 0;
+					sinks[j] = 1;
+				}
+			}
+			break;
+		}
+	}
 
-        sh->sources[smallest] = 1;
+	/* If no sources, all locked nodes are sinks - split brain */
+	if (AFR_COUNT (sources, priv->child_count) == 0) {
+		for (i = 0; i < priv->child_count; i++) {
+			if (locked_on[i])
+				sinks[i] = 1;
+		}
+	}
 
-        return 1;
+	return 0;
 }
 
 
 int
-afr_sh_mark_sources (afr_self_heal_t *sh, int child_count,
-                     afr_self_heal_type type)
+afr_selfheal_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			   int op_ret, int op_errno, inode_t *inode,
+			   struct iatt *buf, dict_t *xdata, struct iatt *parbuf)
 {
-	int i = 0;
-
-        int32_t ** pending_matrix;
-        int *      sources;
+	afr_local_t *local = NULL;
+	int i = -1;
 
-        int size_differs = 0;
-
-        pending_matrix = sh->pending_matrix;
-        sources        = sh->sources;
-
-	int nsources = 0;
-
-        /* stores the 'characters' (innocent, fool, wise) of the nodes */
-        afr_node_character *
-                characters = GF_CALLOC (sizeof (afr_node_character), 
-                                        child_count,
-                                        gf_afr_mt_afr_node_character) ;
+	local = frame->local;
+	i = (long) cookie;
 
-	/* start clean */
-	for (i = 0; i < child_count; i++) {
-		sources[i] = 0;
-	}
-        
-        for (i = 0; i < child_count; i++) {
-                if (afr_sh_is_innocent (pending_matrix[i], child_count)) {
-                        characters[i].type = AFR_NODE_INNOCENT;
-
-                } else if (afr_sh_is_fool (pending_matrix[i], i, child_count)) {
-                        characters[i].type = AFR_NODE_FOOL;
-
-                } else if (afr_sh_is_wise (pending_matrix[i], i, child_count)) {
-                        characters[i].type = AFR_NODE_WISE;
-
-                } else {
-                        gf_log ("[module:replicate]", GF_LOG_ERROR,
-                                "Could not determine the state of subvolume %d!"
-                                " (This message should never appear."
-                                " Please file a bug report to "
-                                "<gluster-devel@nongnu.org>.)", i);
-                }
-        }
-
-        if (type == AFR_SELF_HEAL_DATA) {
-                size_differs = afr_sh_mark_if_size_differs (sh, child_count);
-        }
-
-        if ((type == AFR_SELF_HEAL_METADATA)
-            && afr_sh_all_nodes_innocent (characters, child_count)) {
-
-                nsources = afr_sh_mark_loweia_uid_as_source (sh, child_count);
-                goto out;
-        }
-
-        if (afr_sh_all_nodes_innocent (characters, child_count)) {
-                if (size_differs) {
-                        nsources = afr_sh_mark_biggest_as_source (sh,
-                                                                  child_count);
-                }
-
-        } else if (afr_sh_wise_nodes_exist (characters, child_count)) {
-                afr_sh_compute_wisdom (pending_matrix, characters, child_count);
-
-                if (afr_sh_wise_nodes_conflict (characters, child_count)) {
-                        /* split-brain */
-
-                        nsources = -1;
-                        goto out;
-
-                } else {
-                        nsources = afr_sh_mark_wisest_as_sources (sources, 
-                                                                  characters,
-                                                                  child_count);
-                }
-        } else {
-                nsources = afr_sh_mark_biggest_fool_as_source (sh, characters,
-                                                               child_count);
-        }
+	local->replies[i].valid = 1;
+	local->replies[i].op_ret = op_ret;
+	local->replies[i].op_errno = op_errno;
+	if (buf)
+		local->replies[i].poststat = *buf;
+	if (parbuf)
+		local->replies[i].postparent = *parbuf;
+	if (xdata)
+		local->replies[i].xdata = dict_ref (xdata);
 
-out:
-        GF_FREE (characters);
+	syncbarrier_wake (&local->barrier);
 
-	return nsources;
+	return 0;
 }
 
 
-void
-afr_sh_pending_to_delta (afr_private_t *priv, dict_t **xattr,
-                         int32_t *delta_matrix[], int success[],
-                         int child_count, afr_transaction_type type)
+inode_t *
+afr_selfheal_unlocked_lookup_on (call_frame_t *frame, inode_t *parent,
+				 const char *name, struct afr_reply *replies,
+				 unsigned char *lookup_on)
 {
-	int i = 0;
-	int j = 0;
-        int k = 0;
+	loc_t loc = {0, };
+	dict_t *xattr_req = NULL;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	inode_t *inode = NULL;
 
-	/* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-	int32_t   pending[3];
-	void    * pending_raw = NULL;
-        int       ret         = 0;
+	local = frame->local;
+	priv = frame->this->private;
 
-	/* start clean */
-	for (i = 0; i < child_count; i++) {
-		for (j = 0; j < child_count; j++) {
-			delta_matrix[i][j] = 0;
-		}
+	xattr_req = dict_new ();
+	if (!xattr_req)
+		return NULL;
+
+	if (afr_xattr_req_prepare (frame->this, xattr_req) != 0) {
+		dict_destroy (xattr_req);
+		return NULL;
 	}
 
-	for (i = 0; i < child_count; i++) {
-                pending_raw = NULL;
+	inode = inode_new (parent->table);
+	if (!inode) {
+		dict_destroy (xattr_req);
+		return NULL;
+	}
 
-                for (j = 0; j < child_count; j++) {
-                        ret = dict_get_ptr (xattr[i], priv->pending_key[j],
-                                            &pending_raw);
-                        if (ret < 0)
-                                gf_log ("afr_sh_pending_to_delta",
-                                        GF_LOG_WARNING,
-                                        "Unable to get dict value.");
+	loc.parent = inode_ref (parent);
+	uuid_copy (loc.pargfid, parent->gfid);
+	loc.name = name;
+	loc.inode = inode_ref (inode);
 
-                        if (!success[j])
-                                continue;
+	AFR_ONLIST (lookup_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+		    xattr_req);
 
-                        k = afr_index_for_transaction_type (type);
+	afr_replies_copy (replies, local->replies, priv->child_count);
 
-                        if (pending_raw) {
-				memcpy (pending, pending_raw, sizeof(pending));
-                                delta_matrix[i][j] = -(ntoh32 (pending[k]));
-                        } else {
-                                delta_matrix[i][j]  = 0;
-                        }
+	loc_wipe (&loc);
+	dict_unref (xattr_req);
 
-                }
-	}
+	return inode;
 }
 
 
 int
-afr_sh_delta_to_xattr (afr_private_t *priv,
-                       int32_t *delta_matrix[], dict_t *xattr[],
-		       int child_count, afr_transaction_type type)
+afr_selfheal_unlocked_discover_on (call_frame_t *frame, inode_t *inode,
+				   uuid_t gfid, struct afr_reply *replies,
+				   unsigned char *discover_on)
 {
-	int i = 0;
-	int j = 0;
-        int k = 0;
+	loc_t loc = {0, };
+	dict_t *xattr_req = NULL;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
 
-	int ret = 0;
+	local = frame->local;
+	priv = frame->this->private;
 
-	int32_t *pending = 0;
+	xattr_req = dict_new ();
+	if (!xattr_req)
+		return -ENOMEM;
 
-	for (i = 0; i < child_count; i++) {
-		if (!xattr[i])
-			continue;
+	if (afr_xattr_req_prepare (frame->this, xattr_req) != 0) {
+		dict_destroy (xattr_req);
+		return -ENOMEM;
+	}
 
-		for (j = 0; j < child_count; j++) {
-                        pending = GF_CALLOC (sizeof (int32_t), 3,
-                                             gf_afr_mt_int32_t);
-                        /* 3 = data+metadata+entry */
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, gfid);
 
-                        k = afr_index_for_transaction_type (type);
+	AFR_ONLIST (discover_on, frame, afr_selfheal_discover_cbk, lookup, &loc,
+		    xattr_req);
 
-			pending[k] = hton32 (delta_matrix[i][j]);
+	afr_replies_copy (replies, local->replies, priv->child_count);
+
+	loc_wipe (&loc);
+	dict_unref (xattr_req);
 
-                        ret = dict_set_bin (xattr[i], priv->pending_key[j],
-                                            pending,
-                                            3 * sizeof (int32_t));
-                        if (ret < 0)
-                                gf_log ("afr_sh_delta_to_xattr",
-                                        GF_LOG_WARNING,
-                                        "Unable to set dict value.");
-		}
-	}
 	return 0;
 }
 
-
 int
-afr_sh_has_metadata_pending (dict_t *xattr, int child_count, xlator_t *this)
+afr_selfheal_unlocked_discover (call_frame_t *frame, inode_t *inode,
+				uuid_t gfid, struct afr_reply *replies)
 {
 	afr_private_t *priv = NULL;
-	/* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-	int32_t       pending[3];
-	void          *pending_raw = NULL;
 
-	int           ret = -1;
-	int            i  = 0;
-        int            j  = 0;
+	priv = frame->this->private;
 
-	priv = this->private;
+	return afr_selfheal_unlocked_discover_on (frame, inode, gfid, replies,
+						  priv->child_up);
+}
 
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
 
-                if (ret != 0)
-                        return 0;
+int
+afr_selfheal_lock_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		       int op_ret, int op_errno, dict_t *xdata)
+{
+	afr_local_t *local = NULL;
+	int i = 0;
 
-		memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION);
+	local = frame->local;
+	i = (long) cookie;
+
+	local->replies[i].valid = 1;
+	local->replies[i].op_ret = op_ret;
+	local->replies[i].op_errno = op_errno;
 
-                if (pending[j])
-                        return 1;
-        }
+	syncbarrier_wake (&local->barrier);
 
 	return 0;
 }
 
 
 int
-afr_sh_has_data_pending (dict_t *xattr, int child_count, xlator_t *this)
+afr_selfheal_locked_fill (call_frame_t *frame, xlator_t *this,
+			  unsigned char *locked_on)
 {
+	int i = 0;
 	afr_private_t *priv = NULL;
-	/* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-	int32_t       pending[3];
-	void          *pending_raw = NULL;
-
-	int           ret = -1;
-	int            i  = 0;
-        int            j  = 0;
+	afr_local_t *local = NULL;
+	int count = 0;
 
+	local = frame->local;
 	priv = this->private;
 
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
-
-                if (ret != 0)
-                        return 0;
-                
-		memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
-
-                if (pending[j])
-                        return 1;
-        }
+	for (i = 0; i < priv->child_count; i++) {
+		if (local->replies[i].valid && local->replies[i].op_ret == 0) {
+			locked_on[i] = 1;
+			count++;
+		} else {
+			locked_on[i] = 0;
+		}
+	}
 
-	return 0;
+	return count;
 }
 
 
 int
-afr_sh_has_entry_pending (dict_t *xattr, int child_count, xlator_t *this)
+afr_selfheal_tryinodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			 char *dom, off_t off, size_t size,
+			 unsigned char *locked_on)
 {
-        afr_private_t *priv = NULL;
-	/* Indexable by result of afr_index_for_transaction_type(): 0 -- 2. */
-	int32_t       pending[3];
-	void          *pending_raw = NULL;
-
-	int           ret = -1;
-	int            i  = 0;
-        int            j  = 0;
+	loc_t loc = {0,};
+	struct gf_flock flock = {0, };
 
-	priv = this->private;
-
-        for (i = 0; i < priv->child_count; i++) {
-                ret = dict_get_ptr (xattr, priv->pending_key[i],
-                                    &pending_raw);
-
-                if (ret != 0)
-                        return 0;
-                
-		memcpy (pending, pending_raw, sizeof(pending));
-                j = afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION);
-
-                if (pending[j])
-                        return 1;
-        }
-
-	return 0;
-}
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
+	flock.l_type = F_WRLCK;
+	flock.l_start = off;
+	flock.l_len = size;
 
-/**
- * is_matrix_zero - return true if pending matrix is all zeroes
- */
+	AFR_ONALL (frame, afr_selfheal_lock_cbk, inodelk, dom,
+		    &loc, F_SETLK, &flock, NULL);
 
-int
-afr_sh_is_matrix_zero (int32_t *pending_matrix[], int child_count)
-{
-	int i, j;
+	loc_wipe (&loc);
 
-	for (i = 0; i < child_count; i++) 
-		for (j = 0; j < child_count; j++) 
-			if (pending_matrix[i][j]) 
-				return 0;
-	return 1;
+	return afr_selfheal_locked_fill (frame, this, locked_on);
 }
 
 
 int
-afr_sh_missing_entries_done (call_frame_t *frame, xlator_t *this)
+afr_selfheal_inodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		      char *dom, off_t off, size_t size,
+		      unsigned char *locked_on)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              i = 0;
+	loc_t loc = {0,};
+	struct gf_flock flock = {0, };
+	afr_local_t *local = NULL;
+	int i = 0;
+	afr_private_t *priv = NULL;
 
-	local = frame->local;
-	sh = &local->self_heal;
 	priv = this->private;
+	local = frame->local;
 
-//	memset (sh->child_errno, 0, sizeof (int) * priv->child_count);
-	memset (sh->buf, 0, sizeof (struct iatt) * priv->child_count);
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-        for (i = 0; i < priv->child_count; i++) {
-                sh->locked_nodes[i] = 0;
-        }
+	flock.l_type = F_WRLCK;
+	flock.l_start = off;
+	flock.l_len = size;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->xattr[i])
-			dict_unref (sh->xattr[i]);
-		sh->xattr[i] = NULL;
-	}
+	AFR_ONALL (frame, afr_selfheal_lock_cbk, inodelk, dom,
+		    &loc, F_SETLK, &flock, NULL);
 
-	if (local->govinda_gOvinda) {
-		gf_log (this->name, GF_LOG_TRACE,
-			"aborting selfheal of %s",
-			local->loc.path);
-		sh->completion_cbk (frame, this);
-	} else {
-		gf_log (this->name, GF_LOG_TRACE,
-			"proceeding to metadata check on %s",
-			local->loc.path);
-		afr_self_heal_metadata (frame, this);
+	for (i = 0; i < priv->child_count; i++) {
+		if (local->replies[i].op_ret == -1 &&
+		    local->replies[i].op_errno == EAGAIN) {
+			afr_selfheal_locked_fill (frame, this, locked_on);
+			afr_selfheal_uninodelk (frame, this, inode, dom, off,
+						size, locked_on);
+
+			AFR_SEQ (frame, afr_selfheal_lock_cbk, inodelk, dom,
+				 &loc, F_SETLKW, &flock, NULL);
+			break;
+		}
 	}
 
-	return 0;
-}
-
-
-static int
-sh_missing_entries_finish (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-	local = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->lock_cbk = afr_sh_missing_entries_done;
-        afr_unlock (frame, this);
+	loc_wipe (&loc);
 
-	return 0;
+	return afr_selfheal_locked_fill (frame, this, locked_on);
 }
 
 
-static int
-sh_destroy_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int32_t op_ret, int op_errno,
-                struct iatt *preop, struct iatt *postop)
+int
+afr_selfheal_uninodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			char *dom, off_t off, size_t size,
+			const unsigned char *locked_on)
 {
-        afr_local_t *local = NULL;
+	loc_t loc = {0,};
+	struct gf_flock flock = {0, };
 
-        loc_t *parent_loc = cookie;
 
-        int call_count = 0;
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-        local = frame->local;
+	flock.l_type = F_UNLCK;
+	flock.l_start = off;
+	flock.l_len = size;
 
-        if (op_ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "setattr on %s failed: %s",
-                        local->loc.path, strerror (op_errno));
-        }
+	AFR_ONLIST (locked_on, frame, afr_selfheal_lock_cbk, inodelk,
+		    dom, &loc, F_SETLK, &flock, NULL);
 
-        if (parent_loc) {
-                loc_wipe (parent_loc);
-                GF_FREE (parent_loc);
-        }
+	loc_wipe (&loc);
 
-        call_count = afr_frame_return (frame);
-        
-        if (call_count == 0) {
-                STACK_DESTROY (frame->root);
-        }
-        
 	return 0;
 }
 
 
-static int
-sh_missing_entries_newentry_cbk (call_frame_t *frame, void *cookie,
-				 xlator_t *this,
-				 int32_t op_ret, int32_t op_errno,
-				 inode_t *inode, struct iatt *buf,
-                                 struct iatt *preparent,
-                                 struct iatt *postparent)
+int
+afr_selfheal_tryentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			 char *dom, const char *name, unsigned char *locked_on)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	call_frame_t    *setattr_frame = NULL;
-	int              call_count = 0;
-	int              child_index = 0;
+	loc_t loc = {0,};
 
-        loc_t *parent_loc = NULL;
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-	struct iatt     stbuf;
-        int32_t valid;
+	AFR_ONALL (frame, afr_selfheal_lock_cbk, entrylk, dom,
+		   &loc, name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
 
-	local = frame->local;
-	sh    = &local->self_heal;
-	priv  = this->private;
-
-	child_index = (long) cookie;
-
-	stbuf.ia_atime = sh->buf[sh->source].ia_atime;
-	stbuf.ia_atime_nsec = sh->buf[sh->source].ia_atime_nsec;
-	stbuf.ia_mtime = sh->buf[sh->source].ia_mtime;
-	stbuf.ia_mtime_nsec = sh->buf[sh->source].ia_mtime_nsec;
-
-        stbuf.ia_uid = sh->buf[sh->source].ia_uid;
-        stbuf.ia_gid = sh->buf[sh->source].ia_gid;
-        
-        valid = GF_SET_ATTR_UID   | GF_SET_ATTR_GID |
-                GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
-        
-	if (op_ret == 0) {
-		setattr_frame = copy_frame (frame);
-                
-                setattr_frame->local = GF_CALLOC (1, sizeof (afr_local_t),
-                                                  gf_afr_mt_afr_local_t);
-
-                ((afr_local_t *)setattr_frame->local)->call_count = 2;
-
-		gf_log (this->name, GF_LOG_TRACE,
-			"setattr (%s) on subvolume %s",
-			local->loc.path, priv->children[child_index]->name);
-
-		STACK_WIND_COOKIE (setattr_frame, sh_destroy_cbk,
-                                   (void *) (long) 0,
-                                   priv->children[child_index],
-                                   priv->children[child_index]->fops->setattr,
-                                   &local->loc, &stbuf, valid);
-
-                valid      = GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
-                parent_loc = GF_CALLOC (1, sizeof (*parent_loc), 
-                                        gf_afr_mt_loc_t);
-                afr_build_parent_loc (parent_loc, &local->loc);
-
-                STACK_WIND_COOKIE (setattr_frame, sh_destroy_cbk,
-                                   (void *) (long) parent_loc,
-                                   priv->children[child_index],
-                                   priv->children[child_index]->fops->setattr,
-                                   parent_loc, &sh->parentbuf, valid);
-	}
-
-	call_count = afr_frame_return (frame);
+	loc_wipe (&loc);
 
-	if (call_count == 0) {
-		sh_missing_entries_finish (frame, this);
-	}
-
-	return 0;
+	return afr_selfheal_locked_fill (frame, this, locked_on);
 }
 
 
-static int
-sh_missing_entries_mknod (call_frame_t *frame, xlator_t *this)
+int
+afr_selfheal_entrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		      char *dom, const char *name, unsigned char *locked_on)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              i = 0;
-        int              ret = 0;
-	int              enoent_count = 0;
-	int              call_count = 0;
-	mode_t           st_mode = 0;
-	dev_t            ia_dev = 0;
-        dict_t          *dict = NULL;
+	loc_t loc = {0,};
+	afr_local_t *local = NULL;
+	int i = 0;
+	afr_private_t *priv = NULL;
 
-	local = frame->local;
-	sh = &local->self_heal;
 	priv = this->private;
+	local = frame->local;
 
-	for (i = 0; i < priv->child_count; i++)
-		if (sh->child_errno[i] == ENOENT)
-			enoent_count++;
-
-	call_count = enoent_count;
-	local->call_count = call_count;
-
-	st_mode = st_mode_from_ia (sh->buf[sh->source].ia_prot,
-                                   sh->buf[sh->source].ia_type);
-	ia_dev  = sh->buf[sh->source].ia_dev;
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"mknod %s mode 0%o on %d subvolumes",
-		local->loc.path, st_mode, enoent_count);
-
-        dict = dict_new ();
-        if (!dict)
-                gf_log (this->name, GF_LOG_ERROR, "out of memory");
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-        ret = afr_set_dict_gfid (dict, sh->buf[sh->source].ia_gfid);
-        if (ret)
-                gf_log (this->name, GF_LOG_DEBUG, "gfid set failed");
+	AFR_ONALL (frame, afr_selfheal_lock_cbk, entrylk, dom, &loc,
+		   name, ENTRYLK_LOCK_NB, ENTRYLK_WRLCK, NULL);
 
 	for (i = 0; i < priv->child_count; i++) {
-		if (sh->child_errno[i] == ENOENT) {
-			STACK_WIND_COOKIE (frame,
-					   sh_missing_entries_newentry_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->mknod,
-					   &local->loc, st_mode, ia_dev, dict);
-			if (!--call_count)
-				break;
+		if (local->replies[i].op_ret == -1 &&
+		    local->replies[i].op_errno == EAGAIN) {
+			afr_selfheal_locked_fill (frame, this, locked_on);
+			afr_selfheal_unentrylk (frame, this, inode, dom, name,
+						locked_on);
+
+			AFR_SEQ (frame, afr_selfheal_lock_cbk, entrylk, dom,
+				 &loc, name, ENTRYLK_LOCK, ENTRYLK_WRLCK, NULL);
+			break;
 		}
 	}
 
-        if (dict)
-                dict_unref (dict);
+	loc_wipe (&loc);
 
-	return 0;
+	return afr_selfheal_locked_fill (frame, this, locked_on);
 }
 
 
-static int
-sh_missing_entries_mkdir (call_frame_t *frame, xlator_t *this)
+int
+afr_selfheal_unentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			char *dom, const char *name, unsigned char *locked_on)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-        dict_t          *dict = NULL;
-	int              i = 0;
-        int              ret = 0;
-	int              enoent_count = 0;
-	int              call_count = 0;
-	mode_t           st_mode = 0;
+	loc_t loc = {0,};
 
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++)
-		if (sh->child_errno[i] == ENOENT)
-			enoent_count++;
-
-	call_count = enoent_count;
-	local->call_count = call_count;
-
-	st_mode = st_mode_from_ia (sh->buf[sh->source].ia_prot,
-                                   sh->buf[sh->source].ia_type);
+	AFR_ONLIST (locked_on, frame, afr_selfheal_lock_cbk, entrylk,
+		    dom, &loc, name, ENTRYLK_UNLOCK, ENTRYLK_WRLCK, NULL);
 
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                sh_missing_entries_finish (frame, this);
-                return 0;
-        }
-
-        ret = afr_set_dict_gfid (dict, sh->buf[sh->source].ia_gfid);
-        if (ret)
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "inode gfid set failed");
-
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"mkdir %s mode 0%o on %d subvolumes",
-		local->loc.path, st_mode, enoent_count);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->child_errno[i] == ENOENT) {
-                        if (!strcmp (local->loc.path, "/")) {
-                                /* We shouldn't try to create "/" */
-
-                                sh_missing_entries_finish (frame, this);
-
-                                return 0;
-                        } else {
-                                STACK_WIND_COOKIE (frame,
-                                                   sh_missing_entries_newentry_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->mkdir,
-                                                   &local->loc, st_mode, dict);
-                                if (!--call_count)
-                                        break;
-                        }
-		}
-	}
-
-        if (dict)
-                dict_unref (dict);
+	loc_wipe (&loc);
 
 	return 0;
 }
 
 
-static int
-sh_missing_entries_symlink (call_frame_t *frame, xlator_t *this,
-			    const char *link, struct iatt *buf)
+gf_boolean_t
+afr_is_pending_set (xlator_t *this, dict_t *xdata, int type)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-        dict_t          *dict = NULL;
-	int              i = 0;
-        int              ret = 0;
-	int              enoent_count = 0;
-	int              call_count = 0;
-
+	int idx = -1;
+	afr_private_t *priv = NULL;
+	void *pending_raw = NULL;
+	int *pending_int = NULL;
+	int i = 0;
 
-	local = frame->local;
-	sh = &local->self_heal;
 	priv = this->private;
+	idx = afr_index_for_transaction_type (type);
 
-	for (i = 0; i < priv->child_count; i++)
-		if (sh->child_errno[i] == ENOENT)
-			enoent_count++;
-
-	call_count = enoent_count;
-	local->call_count = call_count;
+	if (dict_get_ptr (xdata, AFR_DIRTY, &pending_raw) == 0) {
+		if (pending_raw) {
+			pending_int = pending_raw;
 
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                sh_missing_entries_finish (frame, this);
-                return 0;
-        }
-
-        ret = afr_set_dict_gfid (dict, buf->ia_gfid);
-        if (ret)
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict gfid set failed");
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"symlink %s -> %s on %d subvolumes",
-		local->loc.path, link, enoent_count);
+			if (ntoh32 (pending_int[idx]))
+				return _gf_true;
+		}
+	}
 
 	for (i = 0; i < priv->child_count; i++) {
-		if (sh->child_errno[i] == ENOENT) {
-			STACK_WIND_COOKIE (frame,
-					   sh_missing_entries_newentry_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->symlink,
-					   link, &local->loc, dict);
-			if (!--call_count)
-				break;
-		}
+		if (dict_get_ptr (xdata, priv->pending_key[i],
+				  &pending_raw))
+			continue;
+		if (!pending_raw)
+			continue;
+		pending_int = pending_raw;
+
+		if (ntoh32 (pending_int[idx]))
+			return _gf_true;
 	}
 
-	return 0;
+	return _gf_false;
 }
 
 
-static int
-sh_missing_entries_readlink_cbk (call_frame_t *frame, void *cookie,
-				 xlator_t *this,
-				 int32_t op_ret, int32_t op_errno,
-				 const char *link, struct iatt *sbuf)
+gf_boolean_t
+afr_is_data_set (xlator_t *this, dict_t *xdata)
 {
-	if (op_ret > 0)
-		sh_missing_entries_symlink (frame, this, link, sbuf);
-	else
-		sh_missing_entries_finish (frame, this);
-
-	return 0;
+	return afr_is_pending_set (this, xdata, AFR_DATA_TRANSACTION);
 }
 
-
-static int
-sh_missing_entries_readlink (call_frame_t *frame, xlator_t *this)
+gf_boolean_t
+afr_is_metadata_set (xlator_t *this, dict_t *xdata)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	STACK_WIND (frame, sh_missing_entries_readlink_cbk,
-		    priv->children[sh->source],
-		    priv->children[sh->source]->fops->readlink,
-		    &local->loc, 4096);
-
-	return 0;
+	return afr_is_pending_set (this, xdata, AFR_METADATA_TRANSACTION);
 }
 
-
-static int
-sh_missing_entries_create (call_frame_t *frame, xlator_t *this)
+gf_boolean_t
+afr_is_entry_set (xlator_t *this, dict_t *xdata)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	int              type = 0;
-	int              i = 0;
-	afr_private_t   *priv = NULL;
-	int              enoent_count = 0;
-	int              govinda_gOvinda = 0;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	for (i = 0; i < priv->child_count; i++) {
-                if (!local->child_up[i])
-                        continue;
-
-		if (sh->child_errno[i]) {
-			if (sh->child_errno[i] == ENOENT)
-				enoent_count++;
-		} else {
-			if (type) {
-				if (type != sh->buf[i].ia_type) {
-                                        gf_log (this->name, GF_LOG_TRACE,
-                                                "file %s is govinda!",
-                                                local->loc.path);
-
-					govinda_gOvinda = 1;
-                                }
-			} else {
-				sh->source = i;
-				type = sh->buf[i].ia_type;
-			}
-		}
-	}
+	return afr_is_pending_set (this, xdata, AFR_ENTRY_TRANSACTION);
+}
 
-	if (govinda_gOvinda) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"conflicting filetypes exist for path %s. returning.",
-			local->loc.path);
 
-		local->govinda_gOvinda = 1;
-		sh_missing_entries_finish (frame, this);
-		return 0;
-	}
+void
+afr_inode_link (inode_t *inode, struct iatt *iatt)
+{
+	inode_t *linked_inode = NULL;
 
-	if (!type) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no source found for %s. all nodes down?. returning.",
-			local->loc.path);
-		/* subvolumes down and/or file does not exist */
-		sh_missing_entries_finish (frame, this);
-		return 0;
-	}
+	linked_inode = inode_link (inode, NULL, NULL, iatt);
 
-	if (enoent_count == 0) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"no missing files - %s. proceeding to metadata check",
-			local->loc.path);
-		/* proceed to next step - metadata self-heal */
-		sh_missing_entries_finish (frame, this);
-		return 0;
-	}
+	uuid_copy (inode->gfid, iatt->ia_gfid);
+	inode->ia_type = iatt->ia_type;
 
-	switch (type) {
-	case IA_IFSOCK:
-	case IA_IFREG:
-	case IA_IFBLK:
-	case IA_IFCHR:
-	case IA_IFIFO:
-		sh_missing_entries_mknod (frame, this);
-		break;
-	case IA_IFLNK:
-		sh_missing_entries_readlink (frame, this);
-		break;
-	case IA_IFDIR:
-		sh_missing_entries_mkdir (frame, this);
-		break;
-	default:
-		gf_log (this->name, GF_LOG_ERROR,
-			"unknown file type: 0%o", type);
-		local->govinda_gOvinda = 1;
-		sh_missing_entries_finish (frame, this);
+	if (linked_inode) {
+		inode_lookup (linked_inode);
+		inode_unref (linked_inode);
 	}
-
-	return 0;
 }
 
 
-static int
-sh_missing_entries_lookup_cbk (call_frame_t *frame, void *cookie,
-			       xlator_t *this,
-			       int32_t op_ret, int32_t op_errno,
-                               inode_t *inode, struct iatt *buf, dict_t *xattr,
-                               struct iatt *postparent)
-{
-	int              child_index = 0;
-	afr_local_t     *local = NULL;
-	int              call_count = 0;
-	afr_private_t   *priv = NULL;
+/*
+ * This function inspects the looked up replies (in an unlocked manner)
+ * and decides whether a locked verification and possible healing is
+ * required or not. It updates the three booleans for each type
+ * of healing. If the boolean flag gets set to FALSE, then we are sure
+ * no healing is required. If the boolean flag gets set to TRUE then
+ * we have to proceed with locked reinspection.
+ */
 
+int
+afr_selfheal_unlocked_inspect (call_frame_t *frame, xlator_t *this,
+			       inode_t *inode, uuid_t gfid,
+			       gf_boolean_t *data_selfheal,
+			       gf_boolean_t *metadata_selfheal,
+			       gf_boolean_t *entry_selfheal)
+{
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int valid_cnt = 0;
+	struct iatt first = {0, };
+	struct afr_reply *replies = NULL;
+	int ret = -1;
 
-	local = frame->local;
 	priv = this->private;
 
-	child_index = (long) cookie;
+	replies = alloca0 (sizeof (*replies) * priv->child_count);
 
-	LOCK (&frame->lock);
-	{
-		if (op_ret == 0) {
-			gf_log (this->name, GF_LOG_TRACE,
-				"path %s on subvolume %s is of mode 0%o",
-				local->loc.path,
-				priv->children[child_index]->name,
-				buf->ia_type);
+	ret = afr_selfheal_unlocked_discover (frame, inode, gfid, replies);
+	if (ret)
+		return ret;
 
-			local->self_heal.buf[child_index] = *buf;
-                        local->self_heal.parentbuf        = *postparent;
-		} else {
-			gf_log (this->name, GF_LOG_TRACE,
-				"path %s on subvolume %s => -1 (%s)",
-				local->loc.path,
-				priv->children[child_index]->name,
-				strerror (op_errno));
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+		if (replies[i].op_ret == -1)
+			continue;
 
-			local->self_heal.child_errno[child_index] = op_errno;
-		}
+		if (afr_is_data_set (this, replies[i].xdata))
+			*data_selfheal = _gf_true;
 
-	}
-	UNLOCK (&frame->lock);
+		if (afr_is_metadata_set (this, replies[i].xdata))
+			*metadata_selfheal = _gf_true;
 
-	call_count = afr_frame_return (frame);
+		if (afr_is_entry_set (this, replies[i].xdata))
+			*entry_selfheal = _gf_true;
 
-	if (call_count == 0) {
-		sh_missing_entries_create (frame, this);
-	}
+		valid_cnt ++;
+		if (valid_cnt == 1) {
+			first = replies[i].poststat;
+			continue;
+		}
 
-	return 0;
-}
+		if (!IA_EQUAL (first, replies[i].poststat, type)) {
+			gf_log (this->name, GF_LOG_ERROR,
+				"TYPE mismatch %d vs %d on %s for gfid:%s",
+				(int) first.ia_type,
+				(int) replies[i].poststat.ia_type,
+				priv->children[i]->name,
+				uuid_utoa (replies[i].poststat.ia_gfid));
+			return -EIO;
+		}
 
+		if (!IA_EQUAL (first, replies[i].poststat, uid)) {
+			gf_log (this->name, GF_LOG_DEBUG,
+				"UID mismatch %d vs %d on %s for gfid:%s",
+				(int) first.ia_uid,
+				(int) replies[i].poststat.ia_uid,
+				priv->children[i]->name,
+				uuid_utoa (replies[i].poststat.ia_gfid));
 
-static int
-sh_missing_entries_lookup (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t    *local = NULL;
-	int             i = 0;
-	int             call_count = 0;
-	afr_private_t  *priv = NULL;
-	dict_t         *xattr_req = NULL;
-	int             ret = -1;
+			*metadata_selfheal = _gf_true;
+		}
 
-	local = frame->local;
-	priv  = this->private;
+		if (!IA_EQUAL (first, replies[i].poststat, gid)) {
+			gf_log (this->name, GF_LOG_DEBUG,
+				"GID mismatch %d vs %d on %s for gfid:%s",
+				(int) first.ia_uid,
+				(int) replies[i].poststat.ia_uid,
+				priv->children[i]->name,
+				uuid_utoa (replies[i].poststat.ia_gfid));
 
-	call_count = afr_up_children_count (priv->child_count,
-                                            local->child_up);
+			*metadata_selfheal = _gf_true;
+		}
 
-	local->call_count = call_count;
+		if (!IA_EQUAL (first, replies[i].poststat, prot)) {
+			gf_log (this->name, GF_LOG_DEBUG,
+				"MODE mismatch %d vs %d on %s for gfid:%s",
+				(int) st_mode_from_ia (first.ia_prot, 0),
+				(int) st_mode_from_ia (replies[i].poststat.ia_prot, 0),
+				priv->children[i]->name,
+				uuid_utoa (replies[i].poststat.ia_gfid));
 
-	xattr_req = dict_new();
+			*metadata_selfheal = _gf_true;
+		}
 
-	if (xattr_req) {
-                for (i = 0; i < priv->child_count; i++) {
-                        ret = dict_set_uint64 (xattr_req,
-                                               priv->pending_key[i],
-                                               3 * sizeof(int32_t));
-                        if (ret < 0)
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Unable to set dict value.");
-                }
-        }
+		if (IA_ISREG(first.ia_type) &&
+		    !IA_EQUAL (first, replies[i].poststat, size)) {
+			gf_log (this->name, GF_LOG_DEBUG,
+				"SIZE mismatch %lld vs %lld on %s for gfid:%s",
+				(long long) first.ia_size,
+				(long long) replies[i].poststat.ia_size,
+				priv->children[i]->name,
+				uuid_utoa (replies[i].poststat.ia_gfid));
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			gf_log (this->name, GF_LOG_TRACE,
-				"looking up %s on subvolume %s",
-				local->loc.path, priv->children[i]->name);
-
-			STACK_WIND_COOKIE (frame,
-					   sh_missing_entries_lookup_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->lookup,
-					   &local->loc, xattr_req);
-
-			if (!--call_count)
-				break;
+			*data_selfheal = _gf_true;
 		}
 	}
 
-	if (xattr_req)
-		dict_unref (xattr_req);
+	if (valid_cnt > 0)
+		afr_inode_link (inode, &first);
+
+	if (valid_cnt < 2)
+		return -ENOTCONN;
 
 	return 0;
 }
 
 
-
-int
-afr_sh_post_nonblocking_entrylk_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non blocking entrylks failed.");
-                afr_sh_missing_entries_done (frame, this);
-        } else {
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non blocking entrylks done. Proceeding to FOP");
-                sh_missing_entries_lookup (frame, this);
-        }
-
-        return 0;
-}
-
-static int
-afr_sh_entrylk (call_frame_t *frame, xlator_t *this)
+inode_t *
+afr_inode_find (xlator_t *this, uuid_t gfid)
 {
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        sh       = &local->self_heal;
+	inode_table_t *table = NULL;
+	inode_t *inode = NULL;
 
-        int_lock->transaction_lk_type = AFR_SELFHEAL_LK;
-        int_lock->selfheal_lk_type    = AFR_ENTRY_SELF_HEAL_LK;
+	table = this->itable;
+	if (!table)
+		return NULL;
 
-        afr_set_lock_number (frame, this);
+	inode = inode_find (table, gfid);
+	if (inode)
+		return inode;
 
-        int_lock->lk_basename = local->loc.name;
-        int_lock->lk_loc      = &sh->parent_loc;
-        int_lock->lock_cbk    = afr_sh_post_nonblocking_entrylk_cbk;
+	inode = inode_new (table);
+	if (!inode)
+		return NULL;
 
-        afr_nonblocking_entrylk (frame, this);
+	uuid_copy (inode->gfid, gfid);
 
-        return 0;
+	return inode;
 }
 
-static int
-afr_self_heal_missing_entries (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-	afr_local_t         *local    = NULL;
-	afr_self_heal_t     *sh       = NULL;
-	afr_private_t       *priv     = NULL;
-
-	local    = frame->local;
-        int_lock = &local->internal_lock;
-	sh       = &local->self_heal;
-	priv     = this->private;
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"attempting to recreate missing entries for path=%s",
-		local->loc.path);
 
-	afr_build_parent_loc (&sh->parent_loc, &local->loc);
-
-        afr_sh_entrylk (frame, this);
-	return 0;
-}
-
-afr_local_t *afr_local_copy (afr_local_t *l, xlator_t *this)
+call_frame_t *
+afr_frame_create (xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        afr_local_t   *lc     = NULL;
-        afr_self_heal_t *sh = NULL;
-        afr_self_heal_t *shc = NULL;
-
-
-        priv = this->private;
-
-        sh = &l->self_heal;
-
-        lc = GF_CALLOC (1, sizeof (afr_local_t),
-                        gf_afr_mt_afr_local_t);
-
-        shc = &lc->self_heal;
-
-        shc->unwind = sh->unwind;
-        shc->need_data_self_heal = sh->need_data_self_heal;
-        shc->need_metadata_self_heal = sh->need_metadata_self_heal;
-        shc->need_entry_self_heal = sh->need_entry_self_heal;
-        shc->forced_merge = sh->forced_merge;
-        shc->healing_fd_opened = sh->healing_fd_opened;
-        shc->data_lock_held = sh->data_lock_held;
-        if (sh->healing_fd && !sh->healing_fd_opened)
-                shc->healing_fd = fd_ref (sh->healing_fd);
-        else
-                shc->healing_fd = sh->healing_fd;
-        shc->background = sh->background;
-        shc->type = sh->type;
-
-        if (l->loc.path)
-                loc_copy (&lc->loc, &l->loc);
-
-        lc->child_up  = memdup (l->child_up, priv->child_count);
-        if (l->xattr_req)
-                lc->xattr_req = dict_ref (l->xattr_req);
-
-        if (l->cont.lookup.inode)
-                lc->cont.lookup.inode = inode_ref (l->cont.lookup.inode);
-        if (l->cont.lookup.xattr)
-                lc->cont.lookup.xattr = dict_ref (l->cont.lookup.xattr);
-        if (l->internal_lock.inode_locked_nodes)
-                lc->internal_lock.inode_locked_nodes =
-                        memdup (l->internal_lock.inode_locked_nodes,
-                                priv->child_count);
-        else
-                lc->internal_lock.inode_locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.inode_locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
-        if (l->internal_lock.entry_locked_nodes)
-                lc->internal_lock.entry_locked_nodes =
-                        memdup (l->internal_lock.entry_locked_nodes,
-                                priv->child_count);
-        else
-                lc->internal_lock.entry_locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.entry_locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
-        if (l->internal_lock.locked_nodes)
-                lc->internal_lock.locked_nodes =
-                        memdup (l->internal_lock.locked_nodes,
-                                priv->child_count);
-        else
-                lc->internal_lock.locked_nodes =
-                        GF_CALLOC (sizeof (*l->internal_lock.locked_nodes),
-                                   priv->child_count,
-                                   gf_afr_mt_char);
-
-        lc->internal_lock.inodelk_lock_count =
-                l->internal_lock.inodelk_lock_count;
-        lc->internal_lock.entrylk_lock_count =
-                l->internal_lock.entrylk_lock_count;
-
-        return lc;
-}
+	call_frame_t *frame = NULL;
+	afr_local_t *local = NULL;
+	int op_errno = 0;
+	pid_t pid = -1;
 
+	frame = create_frame (this, this->ctx->pool);
+	if (!frame)
+		return NULL;
 
-int
-afr_self_heal_completion_cbk (call_frame_t *bgsh_frame, xlator_t *this)
-{
-        afr_private_t *   priv  = NULL;
-	afr_local_t *     local = NULL;
-        afr_self_heal_t * sh    = NULL;
-        char              sh_type_str[256] = {0,};
-
-        priv  = this->private;
-	local = bgsh_frame->local;
-        sh    = &local->self_heal;
-
-	if (local->govinda_gOvinda) {
-                afr_set_split_brain (this, local->cont.lookup.inode, _gf_true);
-	} else {
-                afr_set_split_brain (this, local->cont.lookup.inode, _gf_false);
+	local = AFR_FRAME_INIT (frame, op_errno);
+	if (!local) {
+		STACK_DESTROY (frame->root);
+		return NULL;
 	}
 
-        afr_self_heal_type_str_get(sh, sh_type_str,
-                                   sizeof(sh_type_str));
-        gf_log (this->name, GF_LOG_NORMAL,
-                "background %s self-heal completed on %s", sh_type_str,
-                local->loc.path);
-
-        if (!sh->unwound) {
-                sh->unwind (sh->orig_frame, this);
-        }
+	syncopctx_setfspid (&pid);
 
-        if (sh->background) {
-                LOCK (&priv->lock);
-                {
-                        priv->background_self_heals_started--;
-                }
-                UNLOCK (&priv->lock);
-        }
+	frame->root->pid = pid;
 
-        AFR_STACK_DESTROY (bgsh_frame);
+	afr_set_lk_owner (frame, this, frame->root);
 
-	return 0;
+	return frame;
 }
 
-int
-afr_self_heal (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              i = 0;
-
-        call_frame_t *sh_frame = NULL;
-        afr_local_t  *sh_local = NULL;
 
+/*
+ * This is the entry point for healing a given GFID
+ */
 
-	local = frame->local;
-	priv  = this->private;
-
-        afr_set_lk_owner (frame, this);
-
-        if (local->self_heal.background) {
-                LOCK (&priv->lock);
-                {
-                        if (priv->background_self_heals_started
-                            > priv->background_self_heal_count) {
-
-                                local->self_heal.background = _gf_false;
-
-                        } else {
-                                priv->background_self_heals_started++;
-                        }
-                }
-                UNLOCK (&priv->lock);
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "performing self heal on %s (metadata=%d data=%d entry=%d)",
-                local->loc.path,
-                local->self_heal.need_metadata_self_heal,
-                local->self_heal.need_data_self_heal,
-                local->self_heal.need_entry_self_heal);
-
-        sh_frame        = copy_frame (frame);
-        sh_local        = afr_local_copy (local, this);
-        sh_frame->local = sh_local;
-        sh              = &sh_local->self_heal;
-
-        sh->orig_frame  = frame;
-
-        sh->completion_cbk = afr_self_heal_completion_cbk;
-
-
-	sh->buf = GF_CALLOC (priv->child_count, sizeof (struct iatt),
-                             gf_afr_mt_iatt);
-	sh->child_errno = GF_CALLOC (priv->child_count, sizeof (int),
-                                     gf_afr_mt_int);
-	sh->success = GF_CALLOC (priv->child_count, sizeof (int),
-                                gf_afr_mt_int);
-	sh->xattr = GF_CALLOC (priv->child_count, sizeof (dict_t *),
-                                gf_afr_mt_dict_t);
-	sh->sources = GF_CALLOC (sizeof (*sh->sources), priv->child_count,
-                                gf_afr_mt_int);
-	sh->locked_nodes = GF_CALLOC (sizeof (*sh->locked_nodes), 
-                                      priv->child_count,
-                                      gf_afr_mt_int);
-
-	sh->pending_matrix = GF_CALLOC (sizeof (int32_t *), priv->child_count,
-                                        gf_afr_mt_int32_t);
-
-	for (i = 0; i < priv->child_count; i++) {
-		sh->pending_matrix[i] = GF_CALLOC (sizeof (int32_t),
-					  	   priv->child_count,
-                                                   gf_afr_mt_int32_t);
-	}
+int
+afr_selfheal (xlator_t *this, uuid_t gfid)
+{
+	inode_t *inode = NULL;
+	call_frame_t *frame = NULL;
+	int ret = -1;
+	gf_boolean_t data_selfheal = _gf_false;
+	gf_boolean_t metadata_selfheal = _gf_false;
+	gf_boolean_t entry_selfheal = _gf_false;
 
-	sh->delta_matrix = GF_CALLOC (sizeof (int32_t *), priv->child_count,
-                                      gf_afr_mt_int32_t);
-	for (i = 0; i < priv->child_count; i++) {
-		sh->delta_matrix[i] = GF_CALLOC (sizeof (int32_t),
-					         priv->child_count,
-                                                 gf_afr_mt_int32_t);
-	}
+	inode = afr_inode_find (this, gfid);
+	if (!inode)
+		goto out;
 
-	if (local->success_count && local->enoent_count) {
-		afr_self_heal_missing_entries (sh_frame, this);
-	} else {
-		gf_log (this->name, GF_LOG_TRACE,
-			"proceeding to metadata check on %s",
-			local->loc.path);
+	frame = afr_frame_create (this);
+	if (!frame)
+		goto out;
 
-		afr_sh_missing_entries_done (sh_frame, this);
-	}
+	ret = afr_selfheal_unlocked_inspect (frame, this, inode, gfid,
+					     &data_selfheal,
+					     &metadata_selfheal,
+					     &entry_selfheal);
+	if (ret)
+		goto out;
 
-	return 0;
-}
+	if (data_selfheal)
+		afr_selfheal_data (frame, this, inode);
 
-void
-afr_self_heal_type_str_get (afr_self_heal_t *self_heal_p, char *str,
-                            size_t size)
-{
-        assert(str && (size > 0));
+	if (metadata_selfheal)
+		afr_selfheal_metadata (frame, this, inode);
 
-        if (self_heal_p->need_metadata_self_heal) {
-                snprintf(str, size, " meta-data");
-        }
+	if (entry_selfheal)
+		afr_selfheal_entry (frame, this, inode);
 
-        if (self_heal_p->need_data_self_heal) {
-                snprintf(str + strlen(str), size - strlen(str),
-                         " data");
-        }
+	inode_forget (inode, 1);
+out:
+	if (inode)
+		inode_unref (inode);
+	if (frame)
+		AFR_STACK_DESTROY (frame);
 
-        if (self_heal_p->need_entry_self_heal) {
-                snprintf(str + strlen(str), size - strlen(str),
-                         " entry");
-        }
+	return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-self-heal-common.h b/xlators/cluster/afr/src/afr-self-heal-common.h
deleted file mode 100644
index ecec8a5ae..000000000
--- a/xlators/cluster/afr/src/afr-self-heal-common.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
-  Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef __AFR_SELF_HEAL_COMMON_H__
-#define __AFR_SELF_HEAL_COMMON_H__
-
-#define FILE_HAS_HOLES(buf) (((buf)->ia_size) > ((buf)->ia_blocks * 512))
-
-typedef enum {
-        AFR_SELF_HEAL_ENTRY,
-        AFR_SELF_HEAL_METADATA,
-        AFR_SELF_HEAL_DATA,
-} afr_self_heal_type;
-
-int
-afr_sh_select_source (int sources[], int child_count);
-
-int
-afr_sh_sink_count (int sources[], int child_count);
-
-int
-afr_sh_source_count (int sources[], int child_count);
-
-int
-afr_sh_supress_errenous_children (int sources[], int child_errno[],
-				  int child_count);
-
-void
-afr_sh_print_pending_matrix (int32_t *pending_matrix[], xlator_t *this);
-
-void
-afr_sh_build_pending_matrix (afr_private_t *priv,
-                             int32_t *pending_matrix[], dict_t *xattr[],
-			     int child_count, afr_transaction_type type);
-
-void
-afr_sh_pending_to_delta (afr_private_t *priv, dict_t **xattr,
-                         int32_t *delta_matrix[], int success[],
-                         int child_count, afr_transaction_type type);
-
-int
-afr_sh_mark_sources (afr_self_heal_t *sh, int child_count,
-                     afr_self_heal_type type);
-
-int
-afr_sh_delta_to_xattr (afr_private_t *priv,
-                       int32_t *delta_matrix[], dict_t *xattr[],
-		       int child_count, afr_transaction_type type);
-
-int
-afr_sh_is_matrix_zero (int32_t *pending_matrix[], int child_count);
-
-void
-afr_self_heal_type_str_get (afr_self_heal_t *self_heal_p, char *str,
-                            size_t size);
-
-#endif /* __AFR_SELF_HEAL_COMMON_H__ */
diff --git a/xlators/cluster/afr/src/afr-self-heal-data.c b/xlators/cluster/afr/src/afr-self-heal-data.c
index 077738169..c0548d995 100644
--- a/xlators/cluster/afr/src/afr-self-heal-data.c
+++ b/xlators/cluster/afr/src/afr-self-heal-data.c
@@ -1,1113 +1,635 @@
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#include <libgen.h>
-#include <unistd.h>
-#include <fnmatch.h>
-#include <sys/time.h>
-#include <stdlib.h>
-#include <signal.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
 #endif
 
-#include "glusterfs.h"
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
+#include "afr-self-heal.h"
 #include "byte-order.h"
 
-#include "afr-transaction.h"
-#include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-#include "afr-self-heal-algorithm.h"
+enum {
+	AFR_SELFHEAL_DATA_FULL = 0,
+	AFR_SELFHEAL_DATA_DIFF,
+};
 
 
-int
-afr_sh_data_done (call_frame_t *frame, xlator_t *this)
+#define HAS_HOLES(i) ((i->ia_blocks * 512) < (i->ia_size))
+static int
+__checksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		int op_ret, int op_errno, uint32_t weak, uint8_t *strong,
+		dict_t *xdata)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
+	afr_local_t *local = NULL;
+	int i = (long) cookie;
 
 	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	/* 
-	   TODO: cleanup sh->* 
-	 */
-
-        if (sh->healing_fd && !sh->healing_fd_opened) {
-                /* unref only if we created the fd ourselves */
-
-		fd_unref (sh->healing_fd);
-		sh->healing_fd = NULL;
-        }
-
-/*         for (i = 0; i < priv->child_count; i++) */
-/*                 sh->locked_nodes[i] = 0; */
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"self heal of %s completed",
-		local->loc.path);
 
-	sh->completion_cbk (frame, this);
+	local->replies[i].valid = 1;
+	local->replies[i].op_ret = op_ret;
+	local->replies[i].op_errno = op_errno;
+	if (strong)
+		memcpy (local->replies[i].checksum, strong, MD5_DIGEST_LENGTH);
 
+	syncbarrier_wake (&local->barrier);
 	return 0;
 }
 
 
-int
-afr_sh_data_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno)
+static int
+attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+	  int op_ret, int op_errno, struct iatt *pre, struct iatt *post,
+	  dict_t *xdata)
 {
-	afr_local_t     *local = NULL;
-	afr_private_t   *priv  = NULL;
-	int              call_count = 0;
-
-        int child_index = (long) cookie;
+	int i = (long) cookie;
+	afr_local_t *local = NULL;
 
 	local = frame->local;
-	priv = this->private;
 
-	LOCK (&frame->lock);
-	{
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "flush or setattr failed on %s on subvolume %s: %s",
-                                local->loc.path, priv->children[child_index]->name,
-                                strerror (op_errno));
-                }
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
+	local->replies[i].valid = 1;
+	local->replies[i].op_ret = op_ret;
+	local->replies[i].op_errno = op_errno;
+	if (pre)
+		local->replies[i].prestat = *pre;
+	if (post)
+		local->replies[i].poststat = *post;
+	if (xdata)
+		local->replies[i].xdata = dict_ref (xdata);
 
-	if (call_count == 0) {
-		afr_sh_data_done (frame, this);
-	}
+	syncbarrier_wake (&local->barrier);
 
 	return 0;
 }
 
 
-int
-afr_sh_data_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *statpre, struct iatt *statpost)
+static gf_boolean_t
+__afr_selfheal_data_checksums_match (call_frame_t *frame, xlator_t *this,
+				     fd_t *fd, int source,
+				     unsigned char *healed_sinks,
+				     off_t offset, size_t size)
 {
-        afr_sh_data_flush_cbk (frame, cookie, this, op_ret, op_errno);
-
-        return 0;
-}
-
-
-int
-afr_sh_data_close (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t     *local = NULL;
-	afr_private_t   *priv  = NULL;
-	afr_self_heal_t *sh    = NULL;
-
-	int  i            = 0;
-	int  call_count   = 0;
-        int  source       = 0;
-        int32_t valid     = 0;
-
-        struct iatt stbuf = {0,};
-
-        local = frame->local;
-        sh    = &local->self_heal;
-        priv  = this->private;
-
-        source       = sh->source;
-
-        valid |= (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME);
-
-	stbuf.ia_atime = sh->buf[source].ia_atime;
-	stbuf.ia_atime_nsec = sh->buf[source].ia_atime_nsec;
-	stbuf.ia_mtime = sh->buf[source].ia_mtime;
-	stbuf.ia_mtime_nsec = sh->buf[source].ia_mtime_nsec;
-
-        if (sh->healing_fd_opened) {
-                /* not our job to close the fd */
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
+	unsigned char *wind_subvols = NULL;
+	int i = 0;
 
-                afr_sh_data_done (frame, this);
-                return 0;
-        }
+	priv = this->private;
+	local = frame->local;
 
-	if (!sh->healing_fd) {
-		afr_sh_data_done (frame, this);
-		return 0;
+	wind_subvols = alloca0 (priv->child_count);
+	for (i = 0; i < priv->child_count; i++) {
+		if (i == source || healed_sinks[i])
+			wind_subvols[i] = 1;
 	}
 
-	call_count        = (sh->active_sinks + 1) * 2;
-	local->call_count = call_count;
+	AFR_ONLIST (wind_subvols, frame, __checksum_cbk, rchecksum, fd,
+		    offset, size, NULL);
 
-	/* closed source */
-	gf_log (this->name, GF_LOG_TRACE,
-		"closing fd of %s on %s",
-		local->loc.path, priv->children[sh->source]->name);
-
-	STACK_WIND_COOKIE (frame, afr_sh_data_flush_cbk,
-			   (void *) (long) sh->source,
-			   priv->children[sh->source],
-			   priv->children[sh->source]->fops->flush,
-			   sh->healing_fd);
-	call_count--;
-
-        STACK_WIND_COOKIE (frame, afr_sh_data_setattr_cbk,
-                           (void *) (long) sh->source,
-                           priv->children[sh->source],
-                           priv->children[sh->source]->fops->setattr,
-                           &local->loc, &stbuf, valid);
-
-        call_count--;
-
-        if (call_count == 0)
-                return 0;
+	if (!local->replies[source].valid || local->replies[source].op_ret != 0)
+		return _gf_false;
 
 	for (i = 0; i < priv->child_count; i++) {
-		if (sh->sources[i] || !local->child_up[i])
+		if (i == source)
 			continue;
-
-		gf_log (this->name, GF_LOG_TRACE,
-			"closing fd of %s on %s",
-			local->loc.path, priv->children[i]->name);
-
-		STACK_WIND_COOKIE (frame, afr_sh_data_flush_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->flush,
-				   sh->healing_fd);
-
-                call_count--;
-
-                STACK_WIND_COOKIE (frame, afr_sh_data_setattr_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->setattr,
-				   &local->loc, &stbuf, valid);
-
-		if (!--call_count)
-			break;
-	}
-
-	return 0;
-}
-
-
-int
-afr_sh_data_unlck_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno)
-{
-	afr_local_t * local = NULL;
-	int           call_count = 0;
-	int           child_index = (long) cookie;
-
-	
-	local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"locking inode of %s on child %d failed: %s",
-				local->loc.path, child_index,
-				strerror (op_errno));
-		} else {
-			gf_log (this->name, GF_LOG_TRACE,
-				"inode of %s on child %d locked",
-				local->loc.path, child_index);
-		}
+		if (memcmp (local->replies[source].checksum,
+			    local->replies[i].checksum,
+			    MD5_DIGEST_LENGTH))
+			return _gf_false;
 	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		afr_sh_data_close (frame, this);
-	}
-
-	return 0;
-}
-
-
-int
-afr_sh_data_unlock (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-	afr_self_heal_t     *sh       = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-	sh       = &local->self_heal;
-
-        GF_ASSERT (!sh->data_lock_held);
-
-        int_lock->lock_cbk = afr_sh_data_close;
-        afr_unlock (frame, this);
-
-	return 0;
-}
 
-
-int
-afr_sh_data_finish (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t   *local = NULL;
-	afr_self_heal_t *sh = NULL;
-
-	local = frame->local;
-	sh = &local->self_heal;
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"finishing data selfheal of %s", local->loc.path);
-
-        if (!sh->data_lock_held)
-                afr_sh_data_unlock (frame, this);
-        else
-                afr_sh_data_close (frame, this);
-
-	return 0;
+	return _gf_true;
 }
 
 
-int
-afr_sh_data_erase_pending_cbk (call_frame_t *frame, void *cookie,
-			       xlator_t *this, int32_t op_ret,
-			       int32_t op_errno, dict_t *xattr)
-{
-	int             call_count = 0;
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		afr_sh_data_finish (frame, this);
-
-	return 0;
-}
-
-
-int
-afr_sh_data_erase_pending (call_frame_t *frame, xlator_t *this)
+static int
+__afr_selfheal_data_read_write (call_frame_t *frame, xlator_t *this, fd_t *fd,
+				int source, unsigned char *healed_sinks,
+				off_t offset, size_t size,
+				struct afr_reply *replies)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              call_count = 0;
-	int              i = 0;
-	dict_t          **erase_xattr = NULL;
-
+	struct iovec *iovec = NULL;
+	int count = 0;
+	struct iobref *iobref = NULL;
+	int ret = 0;
+	int i = 0;
+	afr_private_t *priv = NULL;
 
-	local = frame->local;
-	sh = &local->self_heal;
 	priv = this->private;
 
-	afr_sh_pending_to_delta (priv, sh->xattr, sh->delta_matrix, sh->success,
-                                 priv->child_count, AFR_DATA_TRANSACTION);
-
-	erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
-                              gf_afr_mt_dict_t);
+	ret = syncop_readv (priv->children[source], fd, size, offset, 0,
+			    &iovec, &count, &iobref);
+	if (ret <= 0)
+		return ret;
 
 	for (i = 0; i < priv->child_count; i++) {
-		if (sh->xattr[i]) {
-			call_count++;
-
-			erase_xattr[i] = get_new_dict();
-			dict_ref (erase_xattr[i]);
-		}
-	}
-
-	afr_sh_delta_to_xattr (priv, sh->delta_matrix, erase_xattr,
-			       priv->child_count, AFR_DATA_TRANSACTION);
-
-	local->call_count = call_count;
-	for (i = 0; i < priv->child_count; i++) {
-		if (!erase_xattr[i])
+		if (!healed_sinks[i])
 			continue;
 
-		gf_log (this->name, GF_LOG_TRACE,
-			"erasing pending flags from %s on %s",
-			local->loc.path, priv->children[i]->name);
-
-		STACK_WIND_COOKIE (frame, afr_sh_data_erase_pending_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->fxattrop,
-				   sh->healing_fd,
-				   GF_XATTROP_ADD_ARRAY, erase_xattr[i]);
-		if (!--call_count)
-			break;
-	}
+		/*
+		 * TODO: Use fiemap() and discard() to heal holes
+		 * in the future.
+		 *
+		 * For now,
+		 *
+		 * - if the source had any holes at all,
+		 * AND
+		 * - if we are writing past the original file size
+		 *   of the sink
+		 * AND
+		 * - is NOT the last block of the source file. if
+		 *   the block contains EOF, it has to be written
+		 *   in order to set the file size even if the
+		 *   last block is 0-filled.
+		 * AND
+		 * - if the read buffer is filled with only 0's
+		 *
+		 * then, skip writing to this source. We don't depend
+		 * on the write to happen to update the size as we
+		 * have performed an ftruncate() upfront anyways.
+		 */
+#define is_last_block(o,b,s) ((s >= o) && (s <= (o + b)))
+		if (HAS_HOLES ((&replies[source].poststat)) &&
+		    offset >= replies[i].poststat.ia_size &&
+		    !is_last_block (offset, size,
+				    replies[source].poststat.ia_size) &&
+		    (iov_0filled (iovec, count) == 0))
+			continue;
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (erase_xattr[i]) {
-			dict_unref (erase_xattr[i]);
+		ret = syncop_writev (priv->children[i], fd, iovec, count,
+				     offset, iobref, 0);
+		if (ret != iov_length (iovec, count)) {
+			/* write() failed on this sink. unset the corresponding
+			   member in sinks[] (which is healed_sinks[] in the
+			   caller) so that this server does NOT get considered
+			   as successfully healed.
+			*/
+			healed_sinks[i] = 0;
 		}
 	}
-	GF_FREE (erase_xattr);
+	if (iobref)
+		iobref_unref (iobref);
 
-	return 0;
+	return ret;
 }
 
 
-int
-afr_sh_data_trim_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                      struct iatt *postbuf)
+static int
+afr_selfheal_data_block (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			 int source, unsigned char *healed_sinks, off_t offset,
+			 size_t size, int type, struct afr_reply *replies)
 {
-	afr_private_t * priv = NULL;
-	afr_local_t * local  = NULL;
-	int              call_count = 0;
-	int              child_index = 0;
+	int ret = -1;
+	int sink_count = 0;
+	afr_private_t *priv = NULL;
+	unsigned char *data_lock = NULL;
 
 	priv = this->private;
-	local = frame->local;
-
-	child_index = (long) cookie;
+	sink_count = AFR_COUNT (healed_sinks, priv->child_count);
+	data_lock = alloca0 (priv->child_count);
 
-	LOCK (&frame->lock);
+	ret = afr_selfheal_inodelk (frame, this, fd->inode, this->name,
+				    offset, size, data_lock);
 	{
-		if (op_ret == -1)
-			gf_log (this->name, GF_LOG_DEBUG,
-				"ftruncate of %s on subvolume %s failed (%s)",
-				local->loc.path,
-				priv->children[child_index]->name,
-				strerror (op_errno));
-		else
-			gf_log (this->name, GF_LOG_TRACE,
-				"ftruncate of %s on subvolume %s completed",
-				local->loc.path,
-				priv->children[child_index]->name);
-	}
-	UNLOCK (&frame->lock);
+		if (ret < sink_count) {
+			ret = -ENOTCONN;
+			goto unlock;
+		}
 
-	call_count = afr_frame_return (frame);
+		if (type == AFR_SELFHEAL_DATA_DIFF &&
+		    __afr_selfheal_data_checksums_match (frame, this, fd, source,
+							 healed_sinks, offset, size)) {
+			ret = 0;
+			goto unlock;
+		}
 
-	if (call_count == 0) {
-		afr_sh_data_erase_pending (frame, this);
+		ret = __afr_selfheal_data_read_write (frame, this, fd, source,
+						      healed_sinks, offset, size,
+						      replies);
 	}
-
-	return 0;
+unlock:
+	afr_selfheal_uninodelk (frame, this, fd->inode, this->name,
+				offset, size, data_lock);
+	return ret;
 }
 
 
-int
-afr_sh_data_trim_sinks (call_frame_t *frame, xlator_t *this)
-{
-	afr_private_t * priv = NULL;
-	afr_local_t * local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	int             *sources = NULL;
-	int              call_count = 0;
-	int              i = 0;
 
+static int
+afr_selfheal_data_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			 unsigned char *healed_sinks)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
 
-	priv = this->private;
 	local = frame->local;
-	sh = &local->self_heal;
-
-	sources = sh->sources;
-	call_count = sh->active_sinks;
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sources[i] || !local->child_up[i])
-			continue;
-
-		STACK_WIND_COOKIE (frame, afr_sh_data_trim_cbk,
-				   (void *) (long) i,
-				   priv->children[i], 
-				   priv->children[i]->fops->ftruncate,
-				   sh->healing_fd, sh->file_size); 
+	priv = this->private;
 
-		if (!--call_count)
-			break;
-	}
+	AFR_ONLIST (healed_sinks, frame, attr_cbk, fsync, fd, 0, NULL);
 
+	for (i = 0; i < priv->child_count; i++)
+		if (healed_sinks[i] && local->replies[i].op_ret != 0)
+			/* fsync() failed. Do NOT consider this server
+			   as successfully healed. Mark it so.
+			*/
+			healed_sinks[i] = 0;
 	return 0;
 }
 
 
-static struct afr_sh_algorithm *
-sh_algo_from_name (xlator_t *this, char *name)
+static int
+afr_selfheal_data_restore_time (call_frame_t *frame, xlator_t *this,
+				inode_t *inode, int source,
+				unsigned char *healed_sinks,
+				struct afr_reply *replies)
 {
-        int i = 0;
+	loc_t loc = {0, };
 
-        while (afr_self_heal_algorithms[i].name) {
-                if (!strcmp (name, afr_self_heal_algorithms[i].name)) {
-                        return &afr_self_heal_algorithms[i];
-                }
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-                i++;
-        }
+	AFR_ONLIST (healed_sinks, frame, attr_cbk, setattr, &loc,
+		    &replies[source].poststat,
+		    (GF_SET_ATTR_ATIME|GF_SET_ATTR_MTIME), NULL);
 
-        return NULL;
-}
+	loc_wipe (&loc);
 
+	return 0;
+}
 
 static int
-sh_zero_byte_files_exist (afr_self_heal_t *sh, int child_count)
+afr_data_self_heal_type_get (afr_private_t *priv, unsigned char *healed_sinks,
+                             int source, struct afr_reply *replies)
 {
-        int i;
-        int ret = 0;
-
-        for (i = 0; i < child_count; i++) {
-                if (sh->buf[i].ia_size == 0) {
-                        ret = 1;
-                        break;
-                }
-        }
-
-        return ret;
-}
-
+        int type = AFR_SELFHEAL_DATA_FULL;
+        int i = 0;
 
-struct afr_sh_algorithm *
-afr_sh_data_pick_algo (call_frame_t *frame, xlator_t *this)
-{
-        afr_private_t *           priv  = NULL;
-        struct afr_sh_algorithm * algo  = NULL;
-        afr_local_t *             local = NULL;
-        afr_self_heal_t *         sh    = NULL;
-
-        priv  = this->private;
-        local = frame->local;
-        sh    = &local->self_heal;
-        algo  = sh_algo_from_name (this, priv->data_self_heal_algorithm);
-
-        if (algo == NULL) {
-                /* option not set, so fall back on heuristics */
-
-                if ((local->enoent_count != 0)
-                    || sh_zero_byte_files_exist (sh, priv->child_count)
-                    || (sh->file_size <= (priv->data_self_heal_window_size * this->ctx->page_size))) {
-
-                        /*
-                         * If the file does not exist on one of the subvolumes,
-                         * or a zero-byte file exists (created by entry self-heal)
-                         * the entire content has to be copied anyway, so there
-                         * is no benefit from using the "diff" algorithm.
-                         *
-                         * If the file size is about the same as page size,
-                         * the entire file can be read and written with a few
-                         * (pipelined) STACK_WINDs, which will be faster
-                         * than "diff" which has to read checksums and then
-                         * read and write.
-                         */
-
-                        algo = sh_algo_from_name (this, "full");
-
-                } else {
-                        algo = sh_algo_from_name (this, "diff");
+        if (priv->data_self_heal_algorithm == NULL) {
+                type = AFR_SELFHEAL_DATA_FULL;
+                for (i = 0; i < priv->child_count; i++) {
+                        if (!healed_sinks[i] && i != source)
+                                continue;
+                        if (replies[i].poststat.ia_size) {
+                                type = AFR_SELFHEAL_DATA_DIFF;
+                                break;
+                        }
                 }
+        } else if (strcmp (priv->data_self_heal_algorithm, "full") == 0) {
+                type = AFR_SELFHEAL_DATA_FULL;
+        } else if (strcmp (priv->data_self_heal_algorithm, "diff") == 0) {
+                type = AFR_SELFHEAL_DATA_DIFF;
         }
-
-        return algo;
-}
-
-
-int
-afr_sh_data_sync_prepare (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              active_sinks = 0;
-	int              source = 0;
-	int              i = 0;
-
-        struct afr_sh_algorithm *sh_algo = NULL;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	source = sh->source;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->sources[i] == 0 && local->child_up[i] == 1) {
-			active_sinks++;
-			sh->success[i] = 1;
-		}
-	}
-	sh->success[source] = 1;
-
-	if (active_sinks == 0) {
-		gf_log (this->name, GF_LOG_TRACE,
-			"no active sinks for performing self-heal on file %s",
-			local->loc.path);
-		afr_sh_data_finish (frame, this);
-		return 0;
-	}
-	sh->active_sinks = active_sinks;
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"self-healing file %s from subvolume %s to %d other",
-		local->loc.path, priv->children[source]->name, active_sinks);
-
-        sh->algo_completion_cbk = afr_sh_data_trim_sinks;
-        sh->algo_abort_cbk      = afr_sh_data_finish;
-
-        sh_algo = afr_sh_data_pick_algo (frame, this);
-
-        sh_algo->fn (frame, this);
-
-	return 0;
+        return type;
 }
 
-
-int
-afr_sh_data_fix (call_frame_t *frame, xlator_t *this)
+static int
+afr_selfheal_data_do (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		      int source, unsigned char *healed_sinks,
+		      struct afr_reply *replies)
 {
-	afr_local_t     *local      = NULL;
-        afr_local_t *    orig_local = NULL;
-
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              nsources = 0;
-	int              source = 0;
-	int              i = 0;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	off_t off = 0;
+	size_t block = 128 * 1024;
+	int type = AFR_SELFHEAL_DATA_FULL;
+	int ret = -1;
+	call_frame_t *iter_frame = NULL;
+	char *sinks_str = NULL;
+	char *p = NULL;
 
-	local = frame->local;
-	sh = &local->self_heal;
 	priv = this->private;
 
-	afr_sh_build_pending_matrix (priv, sh->pending_matrix, sh->xattr, 
-				     priv->child_count, AFR_DATA_TRANSACTION);
-
-	afr_sh_print_pending_matrix (sh->pending_matrix, this);
-
-	nsources = afr_sh_mark_sources (sh, priv->child_count,
-                                        AFR_SELF_HEAL_DATA);
-
-	afr_sh_supress_errenous_children (sh->sources, sh->child_errno,
-					  priv->child_count);
-
-        if (nsources == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No self-heal needed for %s",
-                        local->loc.path);
-
-                afr_sh_data_finish (frame, this);
-                return 0;
-        }
-
-	if ((nsources == -1)
-	    && (priv->favorite_child != -1)
-	    && (sh->child_errno[priv->favorite_child] == 0)) {
-
-		gf_log (this->name, GF_LOG_DEBUG,
-			"Picking favorite child %s as authentic source to resolve conflicting data of %s",
-			priv->children[priv->favorite_child]->name,
-			local->loc.path);
-
-		sh->sources[priv->favorite_child] = 1;
-
-		nsources = afr_sh_source_count (sh->sources,
-						priv->child_count);
-	}
-
-	if (nsources == -1) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Unable to self-heal contents of '%s' (possible split-brain). "
-                        "Please delete the file from all but the preferred "
-                        "subvolume.", local->loc.path);
-
-		local->govinda_gOvinda = 1;
-
-		afr_sh_data_finish (frame, this);
-		return 0;
-	}
-
-	source = afr_sh_select_source (sh->sources, priv->child_count);
-        
-        if (source == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No active sources found.");
-
-                afr_sh_data_finish (frame, this);
-                return 0;
-        }
-
-	sh->source     = source;
-	sh->block_size = 65536;
-	sh->file_size  = sh->buf[source].ia_size;
-
-	if (FILE_HAS_HOLES (&sh->buf[source]))
-		sh->file_has_holes = 1;
-
-        orig_local = sh->orig_frame->local;
-        orig_local->cont.lookup.buf.ia_size = sh->buf[source].ia_size;
-
-	/* detect changes not visible through pending flags -- JIC */
+	sinks_str = alloca0 (priv->child_count * 8);
+	p = sinks_str;
 	for (i = 0; i < priv->child_count; i++) {
-		if (i == source || sh->child_errno[i])
+		if (!healed_sinks[i])
 			continue;
-
-		if (SIZE_DIFFERS (&sh->buf[i], &sh->buf[source]))
-			sh->sources[i] = 0;
+		p += sprintf (p, "%d ", i);
 	}
 
-        afr_set_read_child (this, local->loc.inode, sh->source);
-
-	/*
-	   quick-read might have read the file, so send xattr from
-	   the source subvolume (http://bugs.gluster.com/cgi-bin/bugzilla3/show_bug.cgi?id=815)
-	*/
-
-	dict_unref (orig_local->cont.lookup.xattr);
-        if (orig_local->cont.lookup.xattrs)
-                orig_local->cont.lookup.xattr = dict_ref (orig_local->cont.lookup.xattrs[sh->source]);
-
-        if (sh->background) {
-                sh->unwind (sh->orig_frame, this);
-                sh->unwound = _gf_true;
-        }
-
-	afr_sh_data_sync_prepare (frame, this);
-
-	return 0;
-}
-
+	gf_log (this->name, GF_LOG_INFO, "performing data selfheal on %s. "
+		"source=%d sinks=%s",
+		uuid_utoa (fd->inode->gfid), source, sinks_str);
 
-int
-afr_self_heal_get_source (xlator_t *this, afr_local_t *local, dict_t **xattr)
-{
-	afr_self_heal_t *sh   = NULL;
-	afr_private_t   *priv = NULL;
+        type = afr_data_self_heal_type_get (priv, healed_sinks, source,
+                                            replies);
 
-	int              source = 0;
-	int              i = 0;
+	iter_frame = afr_copy_frame (frame);
+	if (!iter_frame)
+		return -ENOMEM;
 
-	sh   = &local->self_heal;
-	priv = this->private;
+	for (off = 0; off < replies[source].poststat.ia_size; off += block) {
+		ret = afr_selfheal_data_block (iter_frame, this, fd, source,
+					       healed_sinks, off, block, type,
+					       replies);
+		if (ret < 0)
+			goto out;
 
-	sh->pending_matrix = GF_CALLOC (sizeof (int32_t *), priv->child_count,
-                                        gf_afr_mt_int32_t);
-	for (i = 0; i < priv->child_count; i++) {
-		sh->pending_matrix[i] = GF_CALLOC (sizeof (int32_t),
-						priv->child_count,
-                                                gf_afr_mt_int32_t);
+		AFR_STACK_RESET (iter_frame);
 	}
 
-	sh->sources = GF_CALLOC (priv->child_count, sizeof (*sh->sources),
-                                 gf_afr_mt_int32_t);
-
-	afr_sh_build_pending_matrix (priv, sh->pending_matrix, xattr,
-				     priv->child_count, AFR_DATA_TRANSACTION);
+	afr_selfheal_data_restore_time (frame, this, fd->inode, source,
+					healed_sinks, replies);
 
-	(void)afr_sh_mark_sources (sh, priv->child_count, AFR_SELF_HEAL_DATA);
+	ret = afr_selfheal_data_fsync (frame, this, fd, healed_sinks);
 
-	source = afr_sh_select_source (sh->sources, priv->child_count);
-
-	return source;
+out:
+	if (iter_frame)
+		AFR_STACK_DESTROY (iter_frame);
+	return ret;
 }
 
 
-int
-afr_sh_data_fstat_cbk (call_frame_t *frame, void *cookie,
-                       xlator_t *this, int32_t op_ret, int32_t op_errno,
-                       struct iatt *buf)
+static int
+__afr_selfheal_truncate_sinks (call_frame_t *frame, xlator_t *this,
+			       fd_t *fd, unsigned char *healed_sinks,
+			       struct afr_reply *replies, uint64_t size)
 {
-	afr_private_t   *priv  = NULL;
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-
-	int call_count  = -1;
-	int child_index = (long) cookie;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	unsigned char *larger_sinks = 0;
+	int i = 0;
 
 	local = frame->local;
-	sh = &local->self_heal;
 	priv = this->private;
 
-	LOCK (&frame->lock);
-	{
-		if (op_ret != -1) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "fstat of %s on %s succeeded",
-                                local->loc.path,
-                                priv->children[child_index]->name);
-
-			sh->buf[child_index] = *buf;
-		}
+	larger_sinks = alloca0 (priv->child_count);
+	for (i = 0; i < priv->child_count; i++) {
+		if (healed_sinks[i] && replies[i].poststat.ia_size > size)
+			larger_sinks[i] = 1;
 	}
-	UNLOCK (&frame->lock);
 
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		afr_sh_data_fix (frame, this);
-	}
+	AFR_ONLIST (larger_sinks, frame, attr_cbk, ftruncate, fd, size, NULL);
 
+	for (i = 0; i < priv->child_count; i++)
+		if (healed_sinks[i] && local->replies[i].op_ret == -1)
+			/* truncate() failed. Do NOT consider this server
+			   as successfully healed. Mark it so.
+			*/
+			healed_sinks[i] = 0;
 	return 0;
 }
 
-
-int
-afr_sh_data_fstat (call_frame_t *frame, xlator_t *this)
+/*
+ * If by chance there are multiple sources with differing sizes, select
+ * the largest file as the source.
+ *
+ * This can only happen if data was directly modified in the backend.
+ */
+static int
+__afr_selfheal_data_finalize_source (xlator_t *this, unsigned char *sources,
+				     unsigned char *sinks,
+				     unsigned char *healed_sinks,
+				     unsigned char *locked_on,
+				     struct afr_reply *replies)
 {
-	afr_self_heal_t *sh    = NULL; 
-	afr_local_t     *local = NULL;
-	afr_private_t   *priv  = NULL;
-
-	int call_count = 0;
 	int i = 0;
+	afr_private_t *priv = NULL;
+	uint64_t size = 0;
+	int source = -1;
+	int locked_count = 0;
+	int sources_count = 0;
+	int healed_sinks_count = 0;
 
-	priv  = this->private;
-	local = frame->local;
-	sh    = &local->self_heal;
-
-	call_count = afr_up_children_count (priv->child_count,
-                                            local->child_up);
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_sh_data_fstat_cbk,
-					   (void *) (long) i,
-					   priv->children[i], 
-					   priv->children[i]->fops->fstat,
-					   sh->healing_fd);
-
-			if (!--call_count)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-int
-afr_sh_data_fxattrop_cbk (call_frame_t *frame, void *cookie,
-                          xlator_t *this, int32_t op_ret, int32_t op_errno,
-                          dict_t *xattr)
-{
-	afr_private_t   *priv  = NULL;
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-
-	int call_count  = -1;
-	int child_index = (long) cookie;
-
-	local = frame->local;
-	sh = &local->self_heal;
 	priv = this->private;
 
-	LOCK (&frame->lock);
-	{
-		if (op_ret != -1) {
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "fxattrop of %s on %s succeeded",
-                                local->loc.path,
-                                priv->children[child_index]->name);
+	locked_count = AFR_COUNT (locked_on, priv->child_count);
+	sources_count = AFR_COUNT (sources, priv->child_count);
+	healed_sinks_count = AFR_COUNT (healed_sinks, priv->child_count);
 
-			sh->xattr[child_index] = dict_ref (xattr);
-		}
+	if (locked_count == healed_sinks_count || !sources_count) {
+		/* split brain */
+		return -EIO;
 	}
-	UNLOCK (&frame->lock);
 
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		afr_sh_data_fstat (frame, this);
+	for (i = 0; i < priv->child_count; i++) {
+		if (!sources[i])
+			continue;
+		if (size <= replies[i].poststat.ia_size) {
+			size = replies[i].poststat.ia_size;
+			source = i;
+		}
 	}
 
-	return 0;
-}
-
-
-int
-afr_sh_data_fxattrop (call_frame_t *frame, xlator_t *this)
-{
-	afr_self_heal_t *sh    = NULL;
-	afr_local_t     *local = NULL;
-	afr_private_t   *priv  = NULL;
-	dict_t          *xattr_req = NULL;
-
-        int32_t zero_pending[3] = {0, 0, 0};
-
-	int call_count = 0;
-	int i = 0;
-	int ret = 0;
-
-	priv  = this->private;
-	local = frame->local;
-	sh    = &local->self_heal;
-
-	call_count = afr_up_children_count (priv->child_count,
-                                            local->child_up);
-
-	local->call_count = call_count;
-
-	xattr_req = dict_new();
-	if (xattr_req) {
-                for (i = 0; i < priv->child_count; i++) {
-                        ret = dict_set_static_bin (xattr_req, priv->pending_key[i],
-                                                   zero_pending, 3 * sizeof(int32_t));
-                        if (ret < 0)
-				gf_log (this->name, GF_LOG_WARNING,
-					"Unable to set dict value");
-                }
-        }
-
 	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_sh_data_fxattrop_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->fxattrop,
-					   sh->healing_fd, GF_XATTROP_ADD_ARRAY,
-                                           xattr_req);
-
-			if (!--call_count)
-				break;
+		if (!sources[i])
+			continue;
+		if (replies[i].poststat.ia_size < size) {
+			sources[i] = 0;
+			sinks[i] = 1;
 		}
 	}
 
-	if (xattr_req)
-		dict_unref (xattr_req);
-
-	return 0;
-}
-
-
-int
-afr_sh_data_lock_rec (call_frame_t *frame, xlator_t *this);
-
-int
-afr_sh_data_post_nonblocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non Blocking inodelks failed.");
-                afr_sh_data_done (frame, this);
-        } else {
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non Blocking inodelks done. Proceeding to FOP");
-                afr_sh_data_fxattrop (frame, this);
-        }
-
-        return 0;
-}
-
-int
-afr_sh_data_lock_rec (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-        afr_self_heal_t     *sh       = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-        sh       = &local->self_heal;
-
-        int_lock->transaction_lk_type = AFR_SELFHEAL_LK;
-        int_lock->selfheal_lk_type    = AFR_DATA_SELF_HEAL_LK;
-
-        afr_set_lock_number (frame, this);
-
-        int_lock->lk_flock.l_start = 0;
-        int_lock->lk_flock.l_len   = 0;
-        int_lock->lk_flock.l_type  = F_WRLCK;
-        int_lock->lock_cbk         = afr_sh_data_post_nonblocking_inodelk_cbk;
-
-        afr_nonblocking_inodelk (frame, this);
-
-
-	return 0;
+	return source;
 }
 
-
-int
-afr_sh_data_lock (call_frame_t *frame, xlator_t *this)
+/*
+ * __afr_selfheal_data_prepare:
+ *
+ * This function inspects the on-disk xattrs and determines which subvols
+ * are sources and sinks.
+ *
+ * The return value is the index of the subvolume to be used as the source
+ * for self-healing, or -1 if no healing is necessary/split brain.
+ */
+static int
+__afr_selfheal_data_prepare (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			     unsigned char *locked_on, unsigned char *sources,
+			     unsigned char *sinks, unsigned char *healed_sinks,
+			     struct afr_reply *replies)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	afr_self_heal_t * sh  = NULL;
-
-
-	local = frame->local;
-	sh    = &local->self_heal;
-	priv  = this->private;
+	int ret = -1;
+	int source = -1;
+	afr_private_t *priv = NULL;
+	int i = 0;
 
-        if (sh->data_lock_held) {
-                /* caller has held the lock already,
-                   so skip locking */
+	priv = this->private;
 
-                afr_sh_data_fxattrop (frame, this);
-                return 0;
-        }
+	ret = afr_selfheal_unlocked_discover (frame, fd->inode, fd->inode->gfid,
+					      replies);
+	if (ret)
+		return ret;
+
+	ret = afr_selfheal_find_direction (frame, this, replies,
+					   AFR_DATA_TRANSACTION,
+					   locked_on, sources, sinks);
+	if (ret)
+		return ret;
+
+	source = __afr_selfheal_data_finalize_source (this, sources, sinks,
+						      healed_sinks, locked_on,
+						      replies);
+	if (source < 0)
+		return -EIO;
+
+	for (i = 0; i < priv->child_count; i++)
+		/* Initialize the healed_sinks[] array optimistically to
+		   the intersection of to-be-healed (i.e sinks[]) and
+		   the list of servers which are up (i.e locked_on[]).
+
+		   As we encounter failures in the healing process, we
+		   will unmark the respective servers in the healed_sinks[]
+		   array.
+		*/
+		healed_sinks[i] = sinks[i] && locked_on[i];
 
-        return afr_sh_data_lock_rec (frame, this);
+	return source;
 }
 
 
-int
-afr_sh_data_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno, fd_t *fd)
+static int
+__afr_selfheal_data (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		     unsigned char *locked_on)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              call_count = 0;
-	int              child_index = 0;
+	afr_private_t *priv = NULL;
+	int ret = -1;
+	unsigned char *sources = NULL;
+	unsigned char *sinks = NULL;
+	unsigned char *data_lock = NULL;
+	unsigned char *healed_sinks = NULL;
+	struct afr_reply *locked_replies = NULL;
+	int source = -1;
+	gf_boolean_t compat = _gf_false;
+	unsigned char *compat_lock = NULL;
 
-	local = frame->local;
-	sh = &local->self_heal;
 	priv = this->private;
 
-	child_index = (long) cookie;
+	sources = alloca0 (priv->child_count);
+	sinks = alloca0 (priv->child_count);
+	healed_sinks = alloca0 (priv->child_count);
+	data_lock = alloca0 (priv->child_count);
+	compat_lock = alloca0 (priv->child_count);
 
-	/* TODO: some of the open's might fail.
-	   In that case, modify cleanup fn to send flush on those 
-	   fd's which are already open */
+	locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
 
-	LOCK (&frame->lock);
+	ret = afr_selfheal_inodelk (frame, this, fd->inode, this->name, 0, 0,
+				    data_lock);
 	{
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_TRACE,
-				"open of %s failed on child %s (%s)",
-				local->loc.path,
-				priv->children[child_index]->name,
-				strerror (op_errno));
-			sh->op_failed = 1;
+		if (ret < 2) {
+			ret = -ENOTCONN;
+			goto unlock;
 		}
 
-                gf_log (this->name, GF_LOG_TRACE,
-                        "open of %s succeeded on child %s",
-                        local->loc.path,
-                        priv->children[child_index]->name);
+		ret = __afr_selfheal_data_prepare (frame, this, fd, data_lock,
+						   sources, sinks, healed_sinks,
+						   locked_replies);
+		if (ret < 0)
+			goto unlock;
+
+		source = ret;
+
+		ret = __afr_selfheal_truncate_sinks (frame, this, fd, healed_sinks,
+						     locked_replies,
+						     locked_replies[source].poststat.ia_size);
+		if (ret < 0)
+			goto unlock;
+
+		ret = 0;
+
+		/* Locking from (LLONG_MAX - 2) to (LLONG_MAX - 1) is for
+		   compatibility with older self-heal clients which do not
+		   hold a lock in the @priv->sh_domain domain to guard
+		   against concurrent ongoing self-heals
+		*/
+		afr_selfheal_inodelk (frame, this, fd->inode, this->name,
+				      LLONG_MAX - 2, 1, compat_lock);
+		compat = _gf_true;
 	}
-	UNLOCK (&frame->lock);
+unlock:
+	afr_selfheal_uninodelk (frame, this, fd->inode, this->name, 0, 0,
+				data_lock);
+	if (ret < 0)
+		goto out;
+
+	ret = afr_selfheal_data_do (frame, this, fd, source, healed_sinks,
+				    locked_replies);
+	if (ret)
+		goto out;
+
+	ret = afr_selfheal_undo_pending (frame, this, fd->inode, sources, sinks,
+					 healed_sinks, AFR_DATA_TRANSACTION,
+					 locked_replies, data_lock);
+out:
+	if (compat)
+		afr_selfheal_uninodelk (frame, this, fd->inode, this->name,
+					LLONG_MAX - 2, 1, compat_lock);
+	return ret;
+}
 
-	call_count = afr_frame_return (frame);
 
-	if (call_count == 0) {
-		if (sh->op_failed) {
-			afr_sh_data_finish (frame, this);
-			return 0;
-		}
+static fd_t *
+afr_selfheal_data_open (xlator_t *this, inode_t *inode)
+{
+	loc_t loc = {0,};
+	int ret = 0;
+	fd_t *fd = NULL;
+
+	fd = fd_create (inode, 0);
+	if (!fd)
+		return NULL;
 
-		gf_log (this->name, GF_LOG_TRACE,
-			"fd for %s opened, commencing sync",
-			local->loc.path);
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-                afr_sh_data_lock (frame, this);
+	ret = syncop_open (this, &loc, O_RDWR|O_LARGEFILE, fd);
+	if (ret) {
+		fd_unref (fd);
+		fd = NULL;
+	} else {
+		fd_bind (fd);
 	}
 
-	return 0;
+	loc_wipe (&loc);
+
+	return fd;
 }
 
 
 int
-afr_sh_data_open (call_frame_t *frame, xlator_t *this)
+afr_selfheal_data (call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-	int i = 0;
-	int call_count = 0;
-
+	afr_private_t *priv = NULL;
+	unsigned char *locked_on = NULL;
+	int ret = 0;
 	fd_t *fd = NULL;
 
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	afr_self_heal_t *sh = NULL;
-
-	local = frame->local;
-	sh = &local->self_heal;
 	priv = this->private;
 
-        if (sh->healing_fd_opened) {
-                /* caller has opened the fd for us already, so skip open */
+	fd = afr_selfheal_data_open (this, inode);
+	if (!fd)
+		return -EIO;
 
-                afr_sh_data_lock (frame, this);
-                return 0;
-        }
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-	local->call_count = call_count;
-
-	fd = fd_create (local->loc.inode, frame->root->pid);
-	sh->healing_fd = fd;
-
-	/* open sinks */
-	for (i = 0; i < priv->child_count; i++) {
-		if(!local->child_up[i])
-			continue;
+	locked_on = alloca0 (priv->child_count);
 
-		STACK_WIND_COOKIE (frame, afr_sh_data_open_cbk,
-				   (void *) (long) i,
-				   priv->children[i], 
-				   priv->children[i]->fops->open,
-				   &local->loc, 
-				   O_RDWR|O_LARGEFILE, fd, 0); 
+	ret = afr_selfheal_tryinodelk (frame, this, inode, priv->sh_domain, 0, 0,
+				       locked_on);
+	{
+		if (ret < 2) {
+			/* Either less than two subvols available, or another
+			   selfheal (from another server) is in progress. Skip
+			   for now in any case there isn't anything to do.
+			*/
+			ret = -ENOTCONN;
+			goto unlock;
+		}
 
-		if (!--call_count)
-			break;
+		ret = __afr_selfheal_data (frame, this, fd, locked_on);
 	}
+unlock:
+	afr_selfheal_uninodelk (frame, this, inode, priv->sh_domain, 0, 0, locked_on);
 
-	return 0;
-}
-
+	if (fd)
+		fd_unref (fd);
 
-int
-afr_self_heal_data (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t   *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t *priv = this->private;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-
-	if (sh->need_data_self_heal && priv->data_self_heal) {
-		afr_sh_data_open (frame, this);
-	} else {
-		gf_log (this->name, GF_LOG_TRACE,
-			"not doing data self heal on %s",
-			local->loc.path);
-		afr_sh_data_done (frame, this);
-	}
-
-	return 0;
+	return ret;
 }
-
diff --git a/xlators/cluster/afr/src/afr-self-heal-entry.c b/xlators/cluster/afr/src/afr-self-heal-entry.c
index db12c3b83..9e714b026 100644
--- a/xlators/cluster/afr/src/afr-self-heal-entry.c
+++ b/xlators/cluster/afr/src/afr-self-heal-entry.c
@@ -1,2273 +1,629 @@
 /*
-  Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#include <libgen.h>
-#include <unistd.h>
-#include <fnmatch.h>
-#include <sys/time.h>
-#include <stdlib.h>
-#include <signal.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
 #endif
 
-#include "glusterfs.h"
-#include "inode.h"
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
+#include "afr-self-heal.h"
 #include "byte-order.h"
-
 #include "afr-transaction.h"
-#include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
-
-int
-afr_sh_post_nonblocking_entrylk_cbk (call_frame_t *frame, xlator_t *this);
-
-int
-afr_sh_entry_done (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	/* 
-	   TODO: cleanup sh->* 
-	*/
-
-        if (sh->healing_fd)
-                fd_unref (sh->healing_fd);
-        sh->healing_fd = NULL;
-
-/*         for (i = 0; i < priv->child_count; i++) { */
-/*                 sh->locked_nodes[i] = 0; */
-/*         } */
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"self heal of %s completed",
-		local->loc.path);
-
-	sh->completion_cbk (frame, this);
-
-	return 0;
-}
-
-
-int
-afr_sh_entry_unlock (call_frame_t *frame, xlator_t *this)
-{
-        afr_local_t         *local    = NULL;
-        afr_internal_lock_t *int_lock = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->lock_cbk = afr_sh_entry_done;
-        afr_unlock (frame, this);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_finish (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t   *local = NULL;
-
-	local = frame->local;
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"finishing entry selfheal of %s", local->loc.path);
-
-	afr_sh_entry_unlock (frame, this);
-
-	return 0;
-}
-
-
-int
-afr_sh_entry_erase_pending_cbk (call_frame_t *frame, void *cookie,
-				xlator_t *this, int32_t op_ret,
-				int32_t op_errno, dict_t *xattr)
-{
-	int             call_count = 0;
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		afr_sh_entry_finish (frame, this);
-
-	return 0;
-}
-
-
-int
-afr_sh_entry_erase_pending (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              call_count = 0;
-	int              i = 0;
-	dict_t          **erase_xattr = NULL;
-        int              need_unwind = 0;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	afr_sh_pending_to_delta (priv, sh->xattr, sh->delta_matrix, sh->success,
-                                 priv->child_count, AFR_ENTRY_TRANSACTION);
-
-	erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
-                                 gf_afr_mt_dict_t);
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->xattr[i]) {
-			call_count++;
-
-			erase_xattr[i] = get_new_dict();
-			dict_ref (erase_xattr[i]);
-		}
-	}
-
-        if (call_count == 0)
-                need_unwind = 1;
-
-	afr_sh_delta_to_xattr (priv, sh->delta_matrix, erase_xattr,
-			       priv->child_count, AFR_ENTRY_TRANSACTION);
-
-	local->call_count = call_count;
-	for (i = 0; i < priv->child_count; i++) {
-		if (!erase_xattr[i])
-			continue;
-
-		gf_log (this->name, GF_LOG_TRACE,
-			"erasing pending flags from %s on %s",
-			local->loc.path, priv->children[i]->name);
-
-		STACK_WIND_COOKIE (frame, afr_sh_entry_erase_pending_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->xattrop,
-				   &local->loc,
-				   GF_XATTROP_ADD_ARRAY, erase_xattr[i]);
-		if (!--call_count)
-			break;
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (erase_xattr[i]) {
-			dict_unref (erase_xattr[i]);
-		}
-	}
-	GF_FREE (erase_xattr);
-
-        if (need_unwind)
-                afr_sh_entry_finish (frame, this);
-
-	return 0;
-}
-
 
 
 static int
-next_active_source (call_frame_t *frame, xlator_t *this,
-		    int current_active_source)
+afr_selfheal_entry_delete (call_frame_t *frame, xlator_t *this, inode_t *dir,
+			   const char *name, inode_t *inode, int child,
+			   struct afr_reply *replies)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	int              source = -1;
-	int              next_active_source = -1;
-	int              i = 0;
+	afr_private_t *priv = NULL;
+	xlator_t *subvol = NULL;
+	int ret = 0;
+	loc_t loc = {0, };
+	char g[64];
 
 	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	source = sh->source;
-
-	if (source != -1) {
-		if (current_active_source != source)
-			next_active_source = source;
-		goto out;
-	}
 
-	/*
-	  the next active sink becomes the source for the
-	  'conservative decision' of merging all entries
-	*/
+	subvol = priv->children[child];
 
-	for (i = 0; i < priv->child_count; i++) {
-		if ((sh->sources[i] == 0)
-		    && (local->child_up[i] == 1)
-		    && (i > current_active_source)) {
+	loc.parent = inode_ref (dir);
+	uuid_copy (loc.pargfid, dir->gfid);
+	loc.name = name;
+	loc.inode = inode_ref (inode);
 
-			next_active_source = i;
+	if (replies[child].valid && replies[child].op_ret == 0) {
+		switch (replies[child].poststat.ia_type) {
+		case IA_IFDIR:
+			gf_log (this->name, GF_LOG_WARNING,
+				"expunging dir %s/%s (%s) on %s",
+				uuid_utoa (dir->gfid), name,
+				uuid_utoa_r (replies[child].poststat.ia_gfid, g),
+				subvol->name);
+			ret = syncop_rmdir (subvol, &loc, 1);
 			break;
-		}
-	}
-out:
-	return next_active_source;
-}
-
-
-
-static int
-next_active_sink (call_frame_t *frame, xlator_t *this,
-		  int current_active_sink)
-{
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	int              next_active_sink = -1;
-	int              i = 0;
-
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	/*
-	  the next active sink becomes the source for the
-	  'conservative decision' of merging all entries
-	*/
-
-	for (i = 0; i < priv->child_count; i++) {
-		if ((sh->sources[i] == 0)
-		    && (local->child_up[i] == 1)
-		    && (i > current_active_sink)) {
-
-			next_active_sink = i;
+		default:
+			gf_log (this->name, GF_LOG_WARNING,
+				"expunging file %s/%s (%s) on %s",
+				uuid_utoa (dir->gfid), name,
+				uuid_utoa_r (replies[child].poststat.ia_gfid, g),
+				subvol->name);
+			ret = syncop_unlink (subvol, &loc);
 			break;
 		}
 	}
 
-	return next_active_sink;
-}
-
-
-int
-build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name)
-{
-	int   ret = -1;
-
-	if (!child) {
-		goto out;
-	}
-
-	if (strcmp (parent->path, "/") == 0)
-		ret = gf_asprintf ((char **)&child->path, "/%s", name);
-	else
-		ret = gf_asprintf ((char **)&child->path, "%s/%s", parent->path, 
-                                name);
-
-        if (-1 == ret) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "asprintf failed while setting child path");
-        }
-
-	if (!child->path) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		goto out;
-	}
-
-	child->name = strrchr (child->path, '/');
-	if (child->name)
-		child->name++;
-
-	child->parent = inode_ref (parent->inode);
-	child->inode = inode_new (parent->inode->table);
-
-	if (!child->inode) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		goto out;
-	}
-
-	ret = 0;
-out:
-	if (ret == -1)
-		loc_wipe (child);
+	loc_wipe (&loc);
 
 	return ret;
 }
 
 
 int
-afr_sh_entry_expunge_all (call_frame_t *frame, xlator_t *this);
-
-int
-afr_sh_entry_expunge_subvol (call_frame_t *frame, xlator_t *this,
-			     int active_src);
-
-int
-afr_sh_entry_expunge_entry_done (call_frame_t *frame, xlator_t *this,
-				 int active_src)
-{
-	int              call_count = 0;
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		afr_sh_entry_expunge_subvol (frame, this, active_src);
-
-	return 0;
-}
-
-int
-afr_sh_entry_expunge_parent_setattr_cbk (call_frame_t *expunge_frame,
-                                         void *cookie, xlator_t *this,
-                                         int32_t op_ret, int32_t op_errno,
-                                         struct iatt *preop, struct iatt *postop)
+afr_selfheal_recreate_entry (call_frame_t *frame, xlator_t *this, int dst,
+			     int source, inode_t *dir, const char *name,
+			     inode_t *inode, struct afr_reply *replies)
 {
-	afr_private_t   *priv          = NULL;
-	afr_local_t     *expunge_local = NULL;
-	afr_self_heal_t *expunge_sh    = NULL;
-        call_frame_t    *frame         = NULL;
-
-        int active_src = (long) cookie;
-
-        priv          = this->private;
-        expunge_local = expunge_frame->local;
-	expunge_sh    = &expunge_local->self_heal;
-	frame         = expunge_sh->sh_frame;
-
-        if (op_ret != 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "setattr on parent directory of %s on subvolume %s failed: %s",
-                        expunge_local->loc.path,
-                        priv->children[active_src]->name, strerror (op_errno));
-        }
-
-	AFR_STACK_DESTROY (expunge_frame);
-	afr_sh_entry_expunge_entry_done (frame, this, active_src);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_expunge_remove_cbk (call_frame_t *expunge_frame, void *cookie,
-				 xlator_t *this,
-				 int32_t op_ret, int32_t op_errno,
-                                 struct iatt *preparent,
-                                 struct iatt *postparent)
-{
-	afr_private_t   *priv = NULL;
-	afr_local_t     *expunge_local = NULL;
-	afr_self_heal_t *expunge_sh = NULL;
-	int              active_src = 0;
-	call_frame_t    *frame = NULL;
-
-        int32_t valid = 0;
+	int ret = 0;
+	loc_t loc = {0,};
+	loc_t srcloc = {0,};
+	afr_private_t *priv = NULL;
+	dict_t *xdata = NULL;
+	struct iatt *iatt = NULL;
+	char *linkname = NULL;
+	mode_t mode = 0;
+	struct iatt newent = {0,};
 
 	priv = this->private;
-	expunge_local = expunge_frame->local;
-	expunge_sh = &expunge_local->self_heal;
-	frame = expunge_sh->sh_frame;
-
-	active_src = (long) cookie;
-
-	if (op_ret == 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"removed %s on %s",
-			expunge_local->loc.path,
-			priv->children[active_src]->name);
-	} else {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"removing %s on %s failed (%s)",
-			expunge_local->loc.path,
-			priv->children[active_src]->name,
-			strerror (op_errno));
-	}
 
-        valid = GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
-        afr_build_parent_loc (&expunge_sh->parent_loc, &expunge_local->loc);
+	xdata = dict_new();
+	if (!xdata)
+		return -ENOMEM;
 
-        STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_parent_setattr_cbk,
-                           (void *) (long) active_src,
-                           priv->children[active_src],
-                           priv->children[active_src]->fops->setattr,
-                           &expunge_sh->parent_loc,
-                           &expunge_sh->parentbuf,
-                           valid);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_expunge_unlink (call_frame_t *expunge_frame, xlator_t *this,
-                             int active_src)
-{
-	afr_private_t   *priv = NULL;
-	afr_local_t     *expunge_local = NULL;
-
-	priv          = this->private;
-	expunge_local = expunge_frame->local;
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"expunging file %s on %s",
-		expunge_local->loc.path, priv->children[active_src]->name);
-
-	STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_remove_cbk,
-			   (void *) (long) active_src,
-			   priv->children[active_src],
-			   priv->children[active_src]->fops->unlink,
-			   &expunge_local->loc);
-
-	return 0;
-}
+	loc.parent = inode_ref (dir);
+	uuid_copy (loc.pargfid, dir->gfid);
+	loc.name = name;
+	loc.inode = inode_ref (inode);
 
+	ret = afr_selfheal_entry_delete (frame, this, dir, name, inode, dst,
+					 replies);
+	if (ret)
+		goto out;
 
+	ret = dict_set_static_bin (xdata, "gfid-req",
+				   replies[source].poststat.ia_gfid, 16);
+	if (ret)
+		goto out;
 
-int
-afr_sh_entry_expunge_rmdir (call_frame_t *expunge_frame, xlator_t *this,
-                            int active_src)
-{
-	afr_private_t   *priv = NULL;
-	afr_local_t     *expunge_local = NULL;
-
-	priv          = this->private;
-	expunge_local = expunge_frame->local;
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"expunging directory %s on %s",
-		expunge_local->loc.path, priv->children[active_src]->name);
-
-	STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_remove_cbk,
-			   (void *) (long) active_src,
-			   priv->children[active_src],
-			   priv->children[active_src]->fops->rmdir,
-			   &expunge_local->loc, 1);
-
-	return 0;
-}
+	iatt = &replies[source].poststat;
 
+	srcloc.inode = inode_ref (inode);
+	uuid_copy (srcloc.gfid, iatt->ia_gfid);
 
-int
-afr_sh_entry_expunge_remove (call_frame_t *expunge_frame, xlator_t *this,
-			     int active_src, struct iatt *buf)
-{
-	afr_private_t   *priv = NULL;
-	afr_local_t     *expunge_local = NULL;
-	afr_self_heal_t *expunge_sh = NULL;
-	int              source = 0;
-	call_frame_t    *frame = NULL;
-	int              type = 0;
+	mode = st_mode_from_ia (iatt->ia_prot, iatt->ia_type);
 
-	priv = this->private;
-	expunge_local = expunge_frame->local;
-	expunge_sh = &expunge_local->self_heal;
-	frame = expunge_sh->sh_frame;
-	source = expunge_sh->source;
-
-	type = buf->ia_type;
-
-	switch (type) {
-	case IA_IFSOCK:
-	case IA_IFREG:
-	case IA_IFBLK:
-	case IA_IFCHR:
-	case IA_IFIFO:
-	case IA_IFLNK:
-		afr_sh_entry_expunge_unlink (expunge_frame, this, active_src);
-                break;
+	switch (iatt->ia_type) {
 	case IA_IFDIR:
-		afr_sh_entry_expunge_rmdir (expunge_frame, this, active_src);
+		ret = syncop_mkdir (priv->children[dst], &loc, mode, xdata, 0);
+		break;
+	case IA_IFLNK:
+		ret = syncop_lookup (priv->children[dst], &srcloc, 0, 0, 0, 0);
+		if (ret == 0) {
+			ret = syncop_link (priv->children[dst], &srcloc, &loc);
+		} else {
+			ret = syncop_readlink (priv->children[source], &srcloc,
+					       &linkname, 4096);
+			if (ret <= 0)
+				goto out;
+			ret = syncop_symlink (priv->children[dst], &loc, linkname,
+					      xdata, NULL);
+		}
 		break;
 	default:
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s has unknown file type on %s: 0%o",
-			expunge_local->loc.path,
-			priv->children[source]->name, type);
-		goto out;
+		ret = dict_set_int32 (xdata, GLUSTERFS_INTERNAL_FOP_KEY, 1);
+		if (ret)
+			goto out;
+		ret = syncop_mknod (priv->children[dst], &loc, mode,
+				    iatt->ia_rdev, xdata, &newent);
+		if (ret == 0 && iatt->ia_size && !newent.ia_size) {
+			/* New entry created. Mark @dst pending on all sources */
+			ret = 1;
+		}
 		break;
 	}
 
-	return 0;
-out:
-	AFR_STACK_DESTROY (expunge_frame);
-	afr_sh_entry_expunge_entry_done (frame, this, active_src);
-
-	return 0;
-}
-
-
-int
-afr_sh_entry_expunge_lookup_cbk (call_frame_t *expunge_frame, void *cookie,
-				xlator_t *this,
-				int32_t op_ret,	int32_t op_errno,
-                                inode_t *inode, struct iatt *buf, dict_t *x,
-                                struct iatt *postparent)
-{
-	afr_private_t   *priv = NULL;
-	afr_local_t     *expunge_local = NULL;
-	afr_self_heal_t *expunge_sh = NULL;
-	call_frame_t    *frame = NULL;
-	int              active_src = 0;
-
-	priv = this->private;
-	expunge_local = expunge_frame->local;
-	expunge_sh = &expunge_local->self_heal;
-	frame = expunge_sh->sh_frame;
-	active_src = (long) cookie;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_TRACE,
-			"lookup of %s on %s failed (%s)",
-			expunge_local->loc.path,
-			priv->children[active_src]->name,
-			strerror (op_errno));
-		goto out;
-	}
-
-	afr_sh_entry_expunge_remove (expunge_frame, this, active_src, buf);
-
-	return 0;
 out:
-	AFR_STACK_DESTROY (expunge_frame);
-	afr_sh_entry_expunge_entry_done (frame, this, active_src);
-
-	return 0;
-}
-
-
-int
-afr_sh_entry_expunge_purge (call_frame_t *expunge_frame, xlator_t *this,
-			    int active_src)
-{
-	afr_private_t   *priv = NULL;
-	afr_local_t     *expunge_local = NULL;
-
-	priv = this->private;
-	expunge_local = expunge_frame->local;
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"looking up %s on %s",
-		expunge_local->loc.path, priv->children[active_src]->name);
-	
-	STACK_WIND_COOKIE (expunge_frame, afr_sh_entry_expunge_lookup_cbk,
-			   (void *) (long) active_src,
-			   priv->children[active_src],
-			   priv->children[active_src]->fops->lookup,
-			   &expunge_local->loc, 0);
-
-	return 0;
+	if (xdata)
+		dict_unref (xdata);
+	loc_wipe (&loc);
+	loc_wipe (&srcloc);
+	return ret;
 }
 
 
-int
-afr_sh_entry_expunge_entry_cbk (call_frame_t *expunge_frame, void *cookie,
-				xlator_t *this,
-				int32_t op_ret,	int32_t op_errno,
-                                inode_t *inode, struct iatt *buf, dict_t *x,
-                                struct iatt *postparent)
+static int
+afr_selfheal_newentry_mark (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			    int source, struct afr_reply *replies,
+			    unsigned char *sources, unsigned char *newentry)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *expunge_local = NULL;
-	afr_self_heal_t *expunge_sh = NULL;
-	int              source = 0;
-	call_frame_t    *frame = NULL;
-	int              active_src = 0;
-
+	int ret = 0;
+	int i = 0;
+	afr_private_t *priv = NULL;
+	dict_t *xattr = NULL;
+	int **changelog = NULL;
+	int idx = 0;
 
 	priv = this->private;
-	expunge_local = expunge_frame->local;
-	expunge_sh = &expunge_local->self_heal;
-	frame = expunge_sh->sh_frame;
-	active_src = expunge_sh->active_source;
-	source = (long) cookie;
 
-	if (op_ret == -1 && op_errno == ENOENT && postparent) {
+	idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
 
-		gf_log (this->name, GF_LOG_TRACE,
-			"missing entry %s on %s",
-			expunge_local->loc.path,
-			priv->children[source]->name);
+	uuid_copy (inode->gfid, replies[source].poststat.ia_gfid);
 
-                expunge_sh->parentbuf = *postparent;
+	changelog = afr_matrix_create (priv->child_count, AFR_NUM_CHANGE_LOGS);
 
-		afr_sh_entry_expunge_purge (expunge_frame, this, active_src);
+	xattr = dict_new();
+	if (!xattr)
+		return -ENOMEM;
 
-		return 0;
+	for (i = 0; i < priv->child_count; i++) {
+		if (!newentry[i])
+			continue;
+		changelog[i][idx] = hton32(1);
 	}
 
-	if (op_ret == 0) {
-		gf_log (this->name, GF_LOG_TRACE,
-			"%s exists under %s",
-			expunge_local->loc.path,
-			priv->children[source]->name);
-	} else {
-		gf_log (this->name, GF_LOG_TRACE,
-			"looking up %s under %s failed (%s)",
-			expunge_local->loc.path,
-			priv->children[source]->name,
-			strerror (op_errno));
-	}
+	afr_set_pending_dict (priv, xattr, changelog);
 
-	AFR_STACK_DESTROY (expunge_frame);
-	afr_sh_entry_expunge_entry_done (frame, this, active_src);
+	for (i = 0; i < priv->child_count; i++) {
+		if (!sources[i])
+			continue;
+		afr_selfheal_post_op (frame, this, inode, i, xattr);
+	}
 
-	return 0;
+	dict_unref (xattr);
+	return ret;
 }
 
 
-int
-afr_sh_entry_expunge_entry (call_frame_t *frame, xlator_t *this,
-			    char *name)
+static int
+__afr_selfheal_heal_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			    char *name, inode_t *inode, int source,
+			    unsigned char *sources, unsigned char *healed_sinks,
+			    unsigned char *locked_on, struct afr_reply *replies)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	int              ret = -1;
-	call_frame_t    *expunge_frame = NULL;
-	afr_local_t     *expunge_local = NULL;
-	afr_self_heal_t *expunge_sh = NULL;
-	int              active_src = 0;
-	int              source = 0;
-	int              op_errno = 0;
+	int ret = 0;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	unsigned char *newentry = NULL;
 
 	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	active_src = sh->active_source;
-	source = sh->source;
-
-	if ((strcmp (name, ".") == 0)
-	    || (strcmp (name, "..") == 0)
-            || ((strcmp (local->loc.path, "/") == 0)
-                && (strcmp (name, GF_REPLICATE_TRASH_DIR) == 0))) {
-
-                gf_log (this->name, GF_LOG_TRACE,
-			"skipping inspection of %s under %s",
-			name, local->loc.path);
-		goto out;
-	}
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"inspecting existance of %s under %s",
-		name, local->loc.path);
+	newentry = alloca0 (priv->child_count);
 
-	expunge_frame = copy_frame (frame);
-	if (!expunge_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (expunge_local, afr_local_t, out);
-
-	expunge_frame->local = expunge_local;
-	expunge_sh = &expunge_local->self_heal;
-	expunge_sh->sh_frame = frame;
-	expunge_sh->active_source = active_src;
-
-	ret = build_child_loc (this, &expunge_local->loc, &local->loc, name);
-	if (ret != 0) {
-		goto out;
-	}
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"looking up %s on %s", expunge_local->loc.path,
-		priv->children[source]->name);
-
-	STACK_WIND_COOKIE (expunge_frame,
-			   afr_sh_entry_expunge_entry_cbk,
-			   (void *) (long) source,
-			   priv->children[source],
-			   priv->children[source]->fops->lookup,
-			   &expunge_local->loc, 0);
-
-	ret = 0;
-out:
-	if (ret == -1)
-		afr_sh_entry_expunge_entry_done (frame, this, active_src);
+	if (!replies[source].valid)
+		return -EIO;
 
-	return 0;
-}
-
-
-int
-afr_sh_entry_expunge_readdir_cbk (call_frame_t *frame, void *cookie,
-				  xlator_t *this,
-				  int32_t op_ret, int32_t op_errno,
-				  gf_dirent_t *entries)
-{
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	gf_dirent_t     *entry = NULL;
-	off_t            last_offset = 0;
-	int              active_src = 0;
-	int              entry_count = 0;
-
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	active_src = sh->active_source;
-
-	if (op_ret <= 0) {
-		if (op_ret < 0) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"readdir of %s on subvolume %s failed (%s)",
-				local->loc.path,
-				priv->children[active_src]->name,
-				strerror (op_errno));
+	for (i = 0; i < priv->child_count; i++) {
+		if (!healed_sinks[i])
+			continue;
+		if (replies[source].op_ret == -1 &&
+		    replies[source].op_errno == ENOENT) {
+			ret = afr_selfheal_entry_delete (frame, this, fd->inode,
+							 name, inode, i, replies);
 		} else {
-			gf_log (this->name, GF_LOG_TRACE,
-				"readdir of %s on subvolume %s complete",
-				local->loc.path,
-				priv->children[active_src]->name);
+			if (!uuid_compare (replies[i].poststat.ia_gfid,
+					   replies[source].poststat.ia_gfid))
+				continue;
+
+			ret = afr_selfheal_recreate_entry (frame, this, i, source,
+							   fd->inode, name, inode,
+							   replies);
+			if (ret > 0) {
+				newentry[i] = 1;
+				ret = 0;
+			}
 		}
-
-		afr_sh_entry_expunge_all (frame, this);
-		return 0;
-	}
-
-	list_for_each_entry (entry, &entries->list, list) {
-		last_offset = entry->d_off;
-                entry_count++;
-	}
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"readdir'ed %d entries from %s",
-		entry_count, priv->children[active_src]->name);
-
-	sh->offset = last_offset;
-	local->call_count = entry_count;
-
-	list_for_each_entry (entry, &entries->list, list) {
-                afr_sh_entry_expunge_entry (frame, this, entry->d_name);
-	}
-
-	return 0;
-}
-
-int
-afr_sh_entry_expunge_subvol (call_frame_t *frame, xlator_t *this,
-			     int active_src)
-{
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	STACK_WIND (frame, afr_sh_entry_expunge_readdir_cbk,
-		    priv->children[active_src],
-		    priv->children[active_src]->fops->readdir,
-		    sh->healing_fd, sh->block_size, sh->offset);
-
-	return 0;
-}
-
-
-int
-afr_sh_entry_expunge_all (call_frame_t *frame, xlator_t *this)
-{
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	int              active_src = -1;
-
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	sh->offset = 0;
-
-	if (sh->source == -1) {
-		gf_log (this->name, GF_LOG_TRACE,
-			"no active sources for %s to expunge entries",
-			local->loc.path);
-		goto out;
-	}
-
-	active_src = next_active_sink (frame, this, sh->active_source);
-	sh->active_source = active_src;
-
-	if (sh->op_failed) {
-		goto out;
-	}
-
-	if (active_src == -1) {
-		/* completed creating missing files on all subvolumes */
-		goto out;
-	}
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"expunging entries of %s on %s to other sinks",
-		local->loc.path, priv->children[active_src]->name);
-
-	afr_sh_entry_expunge_subvol (frame, this, active_src);
-
-	return 0;
-out:
-	afr_sh_entry_erase_pending (frame, this);
-	return 0;
-
-}
-
-
-int
-afr_sh_entry_impunge_all (call_frame_t *frame, xlator_t *this);
-
-int
-afr_sh_entry_impunge_subvol (call_frame_t *frame, xlator_t *this,
-			     int active_src);
-
-int
-afr_sh_entry_impunge_entry_done (call_frame_t *frame, xlator_t *this,
-				 int active_src)
-{
-	int              call_count = 0;
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		afr_sh_entry_impunge_subvol (frame, this, active_src);
-
-	return 0;
-}
-
-
-int
-afr_sh_entry_impunge_setattr_cbk (call_frame_t *impunge_frame, void *cookie,
-				  xlator_t *this,
-                                  int32_t op_ret, int32_t op_errno,
-                                  struct iatt *preop, struct iatt *postop)
-{
-	int              call_count = 0;
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-	call_frame_t    *frame = NULL;
-	int              active_src = 0;
-	int              child_index = 0;
-
-	priv = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh = &impunge_local->self_heal;
-	frame = impunge_sh->sh_frame;
-        local = frame->local;
-        sh    = &local->self_heal;
-        active_src = sh->active_source;
-	child_index = (long) cookie;
-
-	if (op_ret == 0) {
-		gf_log (this->name, GF_LOG_TRACE,
-			"setattr done for %s on %s",
-			impunge_local->loc.path,
-			priv->children[child_index]->name);
-	} else {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"setattr (%s) on %s failed (%s)",
-			impunge_local->loc.path,
-			priv->children[child_index]->name,
-			strerror (op_errno));
-	}
-
-	LOCK (&impunge_frame->lock);
-	{
-		call_count = --impunge_local->call_count;
-	}
-	UNLOCK (&impunge_frame->lock);
-
-	if (call_count == 0) {
-		AFR_STACK_DESTROY (impunge_frame);
-		afr_sh_entry_impunge_entry_done (frame, this, active_src);
-	}
-
-	return 0;
-}
-
-
-int
-afr_sh_entry_impunge_xattrop_cbk (call_frame_t *impunge_frame, void *cookie,
-                                  xlator_t *this,
-                                  int32_t op_ret, int32_t op_errno,
-                                  dict_t *xattr)
-{
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-	int              child_index = 0;
-
-        struct iatt stbuf;
-        int32_t     valid = 0;
-
-	priv          = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh    = &impunge_local->self_heal;
-
-	child_index = (long) cookie;
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"setting ownership of %s on %s to %d/%d",
-		impunge_local->loc.path,
-		priv->children[child_index]->name,
-		impunge_local->cont.lookup.buf.ia_uid,
-		impunge_local->cont.lookup.buf.ia_gid);
-
-	stbuf.ia_atime = impunge_local->cont.lookup.buf.ia_atime;
-	stbuf.ia_atime_nsec = impunge_local->cont.lookup.buf.ia_atime_nsec;
-	stbuf.ia_mtime = impunge_local->cont.lookup.buf.ia_mtime;
-	stbuf.ia_mtime_nsec = impunge_local->cont.lookup.buf.ia_mtime_nsec;
-
-        stbuf.ia_uid = impunge_local->cont.lookup.buf.ia_uid;
-        stbuf.ia_gid = impunge_local->cont.lookup.buf.ia_gid;
-
-        valid = GF_SET_ATTR_UID   | GF_SET_ATTR_GID |
-                GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
-
-	STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_setattr_cbk,
-			   (void *) (long) child_index,
-			   priv->children[child_index],
-			   priv->children[child_index]->fops->setattr,
-			   &impunge_local->loc,
-                           &stbuf, valid);
-        return 0;
-}
-
-
-int
-afr_sh_entry_impunge_parent_setattr_cbk (call_frame_t *setattr_frame,
-                                         void *cookie, xlator_t *this,
-                                         int32_t op_ret, int32_t op_errno,
-                                         struct iatt *preop, struct iatt *postop)
-{
-        loc_t *parent_loc = cookie;
-
-        if (op_ret != 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "setattr on parent directory failed: %s",
-                        strerror (op_errno));
-        }
-
-        loc_wipe (parent_loc);
-
-        GF_FREE (parent_loc);
-
-        AFR_STACK_DESTROY (setattr_frame);
-        return 0;
-}
-
-
-int
-afr_sh_entry_impunge_newfile_cbk (call_frame_t *impunge_frame, void *cookie,
-				  xlator_t *this,
-				  int32_t op_ret, int32_t op_errno,
-                                  inode_t *inode, struct iatt *stbuf,
-                                  struct iatt *preparent,
-                                  struct iatt *postparent)
-{
-	int              call_count = 0;
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-	call_frame_t    *frame = NULL;
-	int              active_src = 0;
-	int              child_index = 0;
-        int              pending_array[3] = {0, };
-        dict_t          *xattr = NULL;
-        int              ret = 0;
-        int              idx = 0;
-        afr_local_t     *local = NULL;
-        afr_self_heal_t *sh = NULL;
-
-        call_frame_t *setattr_frame = NULL;
-        int32_t valid = 0;
-        loc_t *parent_loc = NULL;
-        struct iatt parentbuf;
-
-	priv = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh = &impunge_local->self_heal;
-	frame = impunge_sh->sh_frame;
-        local = frame->local;
-        sh    = &local->self_heal;
-        active_src = sh->active_source;
-
-	child_index = (long) cookie;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"creation of %s on %s failed (%s)",
-			impunge_local->loc.path,
-			priv->children[child_index]->name,
-			strerror (op_errno));
-		goto out;
-	}
-
-	inode->ia_type = stbuf->ia_type;
-
-        xattr = get_new_dict ();
-        dict_ref (xattr);
-
-        idx = afr_index_for_transaction_type (AFR_METADATA_TRANSACTION);
-        pending_array[idx] = hton32 (1);
-        if (IA_ISDIR (stbuf->ia_type))
-                idx = afr_index_for_transaction_type (AFR_ENTRY_TRANSACTION);
-        else
-                idx = afr_index_for_transaction_type (AFR_DATA_TRANSACTION);
-        pending_array[idx] = hton32 (1);
-
-        ret = dict_set_static_bin (xattr, priv->pending_key[child_index],
-                                   pending_array, sizeof (pending_array));
-        if (ret < 0)
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Unable to set dict value.");
-
-        valid         = GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
-        parentbuf     = impunge_sh->parentbuf;
-        setattr_frame = copy_frame (impunge_frame);
-
-        parent_loc = GF_CALLOC (1, sizeof (*parent_loc),
-                                gf_afr_mt_loc_t);
-        afr_build_parent_loc (parent_loc, &impunge_local->loc);
-
-	STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_xattrop_cbk,
-			   (void *) (long) child_index,
-			   priv->children[active_src],
-			   priv->children[active_src]->fops->xattrop,
-			   &impunge_local->loc, GF_XATTROP_ADD_ARRAY, xattr);
-
-        STACK_WIND_COOKIE (setattr_frame, afr_sh_entry_impunge_parent_setattr_cbk,
-                           (void *) (long) parent_loc,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->setattr,
-                           parent_loc, &parentbuf, valid);
-
-        dict_unref (xattr);
-
-	return 0;
-
-out:
-	LOCK (&impunge_frame->lock);
-	{
-		call_count = --impunge_local->call_count;
-	}
-	UNLOCK (&impunge_frame->lock);
-
-	if (call_count == 0) {
-		AFR_STACK_DESTROY (impunge_frame);
-		afr_sh_entry_impunge_entry_done (frame, this, active_src);
+		if (ret < 0)
+			break;
 	}
 
-	return 0;
-}
-
-
-int
-afr_sh_entry_impunge_mknod (call_frame_t *impunge_frame, xlator_t *this,
-			    int child_index, struct iatt *stbuf)
-{
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-        dict_t          *dict = NULL;
-
-        int ret = 0;
-
-	priv = this->private;
-	impunge_local = impunge_frame->local;
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"creating missing file %s on %s",
-		impunge_local->loc.path,
-		priv->children[child_index]->name);
-
-        dict = dict_new ();
-        if (!dict)
-                gf_log (this->name, GF_LOG_ERROR, "Out of memory");
-
-        ret = afr_set_dict_gfid (dict, stbuf->ia_gfid);
-        if (ret)
-                gf_log (this->name, GF_LOG_DEBUG, "gfid set failed");
-
-	STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_newfile_cbk,
-			   (void *) (long) child_index,
-			   priv->children[child_index],
-			   priv->children[child_index]->fops->mknod,
-			   &impunge_local->loc,
-			   st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type),
-                           stbuf->ia_rdev, dict);
-
-        if (dict)
-                dict_unref (dict);
-
-	return 0;
-}
-
-
-
-int
-afr_sh_entry_impunge_mkdir (call_frame_t *impunge_frame, xlator_t *this,
-			    int child_index, struct iatt *stbuf)
-{
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-        dict_t          *dict = NULL;
-
-        int ret = 0;
-
-	priv = this->private;
-	impunge_local = impunge_frame->local;
-
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                return 0;
-        }
-
-        ret = afr_set_dict_gfid (dict, stbuf->ia_gfid);
-        if (ret)
-                gf_log (this->name, GF_LOG_DEBUG, "gfid set failed");
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"creating missing directory %s on %s",
-		impunge_local->loc.path,
-		priv->children[child_index]->name);
-
-	STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_newfile_cbk,
-			   (void *) (long) child_index,
-			   priv->children[child_index],
-			   priv->children[child_index]->fops->mkdir,
-			   &impunge_local->loc,
-                           st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type),
-                           dict);
-
-        if (dict)
-                dict_unref (dict);
-
-	return 0;
+	if (AFR_COUNT (newentry, priv->child_count))
+		afr_selfheal_newentry_mark (frame, this, inode, source, replies,
+					    sources, newentry);
+	return ret;
 }
 
 
-int
-afr_sh_entry_impunge_symlink (call_frame_t *impunge_frame, xlator_t *this,
-			      int child_index, const char *linkname)
+static int
+__afr_selfheal_merge_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			     char *name, inode_t *inode, unsigned char *sources,
+			     unsigned char *healed_sinks, unsigned char *locked_on,
+			     struct afr_reply *replies)
 {
-	afr_private_t   *priv          = NULL;
-	afr_local_t     *impunge_local = NULL;
-        dict_t          *dict          = NULL;
-        struct iatt     *buf           = NULL;
-
-        int ret = 0;
+	int ret = 0;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int source = -1;
 
 	priv = this->private;
-	impunge_local = impunge_frame->local;
-
-        buf = &impunge_local->cont.symlink.buf;
-
-        dict = dict_new ();
-        if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                afr_sh_entry_impunge_entry_done (impunge_frame, this, 0);
-        }
-
-        ret = afr_set_dict_gfid (dict, buf->ia_gfid);
-        if (ret)
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict set gfid failed");
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"creating missing symlink %s -> %s on %s",
-		impunge_local->loc.path, linkname,
-		priv->children[child_index]->name);
-
-	STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_newfile_cbk,
-			   (void *) (long) child_index,
-			   priv->children[child_index],
-			   priv->children[child_index]->fops->symlink,
-			   linkname, &impunge_local->loc, dict);
-
-        if (dict)
-                dict_unref (dict);
-
-	return 0;
-}
-
-
-int
-afr_sh_entry_impunge_symlink_unlink_cbk (call_frame_t *impunge_frame,
-                                         void *cookie, xlator_t *this,
-                                         int32_t op_ret, int32_t op_errno,
-                                         struct iatt *preparent,
-                                         struct iatt *postparent)
-{
-        afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-	int              child_index = -1;
-	call_frame_t    *frame = NULL;
-	int              call_count = -1;
-	int              active_src = -1;
-
-	priv          = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh    = &impunge_local->self_heal;
-	frame         = impunge_sh->sh_frame;
-	active_src    = impunge_sh->active_source;
-
-	child_index = (long) cookie;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"unlink of %s on %s failed (%s)",
-			impunge_local->loc.path,
-			priv->children[child_index]->name,
-			strerror (op_errno));
-		goto out;
-	}
-
-        afr_sh_entry_impunge_symlink (impunge_frame, this, child_index,
-                                      impunge_sh->linkname);
-
-        return 0;
-out:
-	LOCK (&impunge_frame->lock);
-	{
-		call_count = --impunge_local->call_count;
-	}
-	UNLOCK (&impunge_frame->lock);
 
-	if (call_count == 0) {
-		AFR_STACK_DESTROY (impunge_frame);
-		afr_sh_entry_impunge_entry_done (frame, this, active_src);
+	for (i = 0; i < priv->child_count; i++) {
+		if (replies[i].valid && replies[i].op_ret == 0) {
+			source = i;
+			break;
+		}
 	}
 
-	return 0;
-}
-
-
-int
-afr_sh_entry_impunge_symlink_unlink (call_frame_t *impunge_frame, xlator_t *this,
-                                     int child_index)
-{
-	afr_private_t   *priv          = NULL;
-	afr_local_t     *impunge_local = NULL;
-
-	priv          = this->private;
-	impunge_local = impunge_frame->local;
-
-	gf_log (this->name, GF_LOG_DEBUG,
-		"unlinking symlink %s with wrong target on %s",
-		impunge_local->loc.path,
-		priv->children[child_index]->name);
-
-        STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_symlink_unlink_cbk,
-                           (void *) (long) child_index,
-                           priv->children[child_index],
-                           priv->children[child_index]->fops->unlink,
-                           &impunge_local->loc);
-
-        return 0;
-}
-
-
-int
-afr_sh_entry_impunge_readlink_sink_cbk (call_frame_t *impunge_frame, void *cookie,
-                                        xlator_t *this,
-                                        int32_t op_ret, int32_t op_errno,
-                                        const char *linkname, struct iatt *sbuf)
-{
-        afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-	int              child_index = -1;
-	call_frame_t    *frame = NULL;
-	int              call_count = -1;
-	int              active_src = -1;
-
-	priv          = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh    = &impunge_local->self_heal;
-	frame         = impunge_sh->sh_frame;
-	active_src    = impunge_sh->active_source;
-
-	child_index = (long) cookie;
-
-	if ((op_ret == -1) && (op_errno != ENOENT)) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"readlink of %s on %s failed (%s)",
-			impunge_local->loc.path,
-			priv->children[active_src]->name,
-			strerror (op_errno));
-		goto out;
+	if (source == -1) {
+		/* entry got deleted in the mean time? */
+		return 0;
 	}
 
-        /* symlink doesn't exist on the sink */
-
-        if ((op_ret == -1) && (op_errno == ENOENT)) {
-                afr_sh_entry_impunge_symlink (impunge_frame, this,
-                                              child_index, impunge_sh->linkname);
-                return 0;
-        }
-
-
-        /* symlink exists on the sink, so check if targets match */
-
-        if (strcmp (linkname, impunge_sh->linkname) == 0) {
-                /* targets match, nothing to do */
-
-                goto out;
-        } else {
-                /*
-                 * Hah! Sneaky wolf in sheep's clothing!
-                 */
-                afr_sh_entry_impunge_symlink_unlink (impunge_frame, this,
-                                                     child_index);
-                return 0;
-        }
+	for (i = 0; i < priv->child_count; i++) {
+		if (i == source || !healed_sinks[i])
+			continue;
 
-out:
-	LOCK (&impunge_frame->lock);
-	{
-		call_count = --impunge_local->call_count;
-	}
-	UNLOCK (&impunge_frame->lock);
+		if (replies[i].op_errno != ENOENT)
+			continue;
 
-	if (call_count == 0) {
-		AFR_STACK_DESTROY (impunge_frame);
-		afr_sh_entry_impunge_entry_done (frame, this, active_src);
+		ret = afr_selfheal_recreate_entry (frame, this, i, source,
+						   fd->inode, name, inode,
+						   replies);
 	}
 
-	return 0;
+	return ret;
 }
 
 
-int
-afr_sh_entry_impunge_readlink_sink (call_frame_t *impunge_frame, xlator_t *this,
-                                    int child_index)
+static int
+__afr_selfheal_entry_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			     char *name, inode_t *inode, int source,
+			     unsigned char *sources, unsigned char *healed_sinks,
+			     unsigned char *locked_on, struct afr_reply *replies)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-
-	priv = this->private;
-	impunge_local = impunge_frame->local;
+	int ret = -1;
 
-	gf_log (this->name, GF_LOG_DEBUG,
-		"checking symlink target of %s on %s",
-		impunge_local->loc.path, priv->children[child_index]->name);
-
-	STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_readlink_sink_cbk,
-			   (void *) (long) child_index,
-			   priv->children[child_index],
-			   priv->children[child_index]->fops->readlink,
-			   &impunge_local->loc, 4096);
-
-	return 0;
+	if (source < 0)
+		ret = __afr_selfheal_merge_dirent (frame, this, fd, name, inode,
+						   sources, healed_sinks,
+						   locked_on, replies);
+	else
+		ret = __afr_selfheal_heal_dirent (frame, this, fd, name, inode,
+						  source, sources, healed_sinks,
+						  locked_on, replies);
+	return ret;
 }
 
 
-int
-afr_sh_entry_impunge_readlink_cbk (call_frame_t *impunge_frame, void *cookie,
-				   xlator_t *this,
-				   int32_t op_ret, int32_t op_errno,
-				   const char *linkname, struct iatt *sbuf)
+static int
+afr_selfheal_entry_dirent (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			   int source, unsigned char *sources,
+			   unsigned char *healed_sinks, char *name)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-	int              child_index = -1;
-	call_frame_t    *frame = NULL;
-	int              call_count = -1;
-	int              active_src = -1;
+	afr_private_t *priv = NULL;
+	int ret = 0;
+	unsigned char *locked_on = NULL;
+	struct afr_reply *replies = NULL;
+	inode_t *inode = NULL;
 
 	priv = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh = &impunge_local->self_heal;
-	frame = impunge_sh->sh_frame;
-	active_src = impunge_sh->active_source;
-
-	child_index = (long) cookie;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"readlink of %s on %s failed (%s)",
-			impunge_local->loc.path,
-			priv->children[active_src]->name,
-			strerror (op_errno));
-		goto out;
-	}
 
-        impunge_sh->linkname = gf_strdup (linkname);
-	afr_sh_entry_impunge_readlink_sink (impunge_frame, this, child_index);
+	locked_on = alloca0 (priv->child_count);
 
-	return 0;
+	replies = alloca0 (priv->child_count * sizeof(*replies));
 
-out:
-	LOCK (&impunge_frame->lock);
+	ret = afr_selfheal_entrylk (frame, this, fd->inode, this->name,
+				    name, locked_on);
 	{
-		call_count = --impunge_local->call_count;
-	}
-	UNLOCK (&impunge_frame->lock);
-
-	if (call_count == 0) {
-		AFR_STACK_DESTROY (impunge_frame);
-		afr_sh_entry_impunge_entry_done (frame, this, active_src);
-	}
-
-	return 0;
-}
-
-
-int
-afr_sh_entry_impunge_readlink (call_frame_t *impunge_frame, xlator_t *this,
-			       int child_index, struct iatt *stbuf)
-{
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-	int              active_src = -1;
-
-	priv = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh = &impunge_local->self_heal;
-	active_src = impunge_sh->active_source;
-        impunge_local->cont.symlink.buf = *stbuf;
-
-	STACK_WIND_COOKIE (impunge_frame, afr_sh_entry_impunge_readlink_cbk,
-			   (void *) (long) child_index,
-			   priv->children[active_src],
-			   priv->children[active_src]->fops->readlink,
-			   &impunge_local->loc, 4096);
-
-	return 0;
-}
-
-
-int
-afr_sh_entry_impunge_recreate_lookup_cbk (call_frame_t *impunge_frame,
-					  void *cookie, xlator_t *this,
-					  int32_t op_ret, int32_t op_errno,
-					  inode_t *inode, struct iatt *buf,
-					  dict_t *xattr,struct iatt *postparent)
-{
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-	int              active_src = 0;
-	int              type = 0;
-	int              child_index = 0;
-	call_frame_t    *frame = NULL;
-	int              call_count = 0;
-
-	priv = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh = &impunge_local->self_heal;
-	frame = impunge_sh->sh_frame;
-
-	child_index = (long) cookie;
-
-	active_src = impunge_sh->active_source;
-
-	if (op_ret != 0) {
-		gf_log (this->name, GF_LOG_TRACE,
-			"looking up %s on %s (for %s) failed (%s)",
-			impunge_local->loc.path,
-			priv->children[active_src]->name,
-			priv->children[child_index]->name,
-			strerror (op_errno));
-		goto out;
-	}
-
-        impunge_sh->parentbuf = *postparent;
-
-	impunge_local->cont.lookup.buf = *buf;
-	type = buf->ia_type;
-
-	switch (type) {
-	case IA_IFSOCK:
-	case IA_IFREG:
-	case IA_IFBLK:
-	case IA_IFCHR:
-	case IA_IFIFO:
-		afr_sh_entry_impunge_mknod (impunge_frame, this,
-					    child_index, buf);
-		break;
-	case IA_IFLNK:
-		afr_sh_entry_impunge_readlink (impunge_frame, this,
-					       child_index, buf);
-		break;
-	case IA_IFDIR:
-		afr_sh_entry_impunge_mkdir (impunge_frame, this,
-					    child_index, buf);
-		break;
-	default:
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s has unknown file type on %s: 0%o",
-			impunge_local->loc.path,
-			priv->children[active_src]->name, type);
-		goto out;
-		break;
-	}
-
-	return 0;
+		if (ret < 2) {
+			ret = -ENOTCONN;
+			goto unlock;
+		}
 
-out:
-	LOCK (&impunge_frame->lock);
-	{
-		call_count = --impunge_local->call_count;
-	}
-	UNLOCK (&impunge_frame->lock);
+		inode = afr_selfheal_unlocked_lookup_on (frame, fd->inode, name,
+							 replies, locked_on);
+		if (!inode) {
+			ret = -ENOMEM;
+			goto unlock;
+		}
 
-	if (call_count == 0) {
-		AFR_STACK_DESTROY (impunge_frame);
-		afr_sh_entry_impunge_entry_done (frame, this, active_src);
+		ret = __afr_selfheal_entry_dirent (frame, this, fd, name, inode,
+						   source, sources, healed_sinks,
+						   locked_on, replies);
 	}
-
-	return 0;
-}
-
-
-int
-afr_sh_entry_impunge_recreate (call_frame_t *impunge_frame, xlator_t *this,
-			       int child_index)
-{
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-	int              active_src = 0;
-
-
-	priv = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh = &impunge_local->self_heal;
-
-	active_src = impunge_sh->active_source;
-
-	STACK_WIND_COOKIE (impunge_frame,
-			   afr_sh_entry_impunge_recreate_lookup_cbk,
-			   (void *) (long) child_index,
-			   priv->children[active_src],
-			   priv->children[active_src]->fops->lookup,
-			   &impunge_local->loc, 0);
-
-	return 0;
+unlock:
+	afr_selfheal_unentrylk (frame, this, fd->inode, this->name, name,
+				locked_on);
+	if (inode)
+		inode_unref (inode);
+	return ret;
 }
 
 
-int
-afr_sh_entry_impunge_entry_cbk (call_frame_t *impunge_frame, void *cookie,
-				xlator_t *this,
-				int32_t op_ret,	int32_t op_errno,
-                                inode_t *inode, struct iatt *buf, dict_t *x,
-                                struct iatt *postparent)
+static int
+afr_selfheal_entry_do_subvol (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			      int child, int source, unsigned char *sources,
+			      unsigned char *healed_sinks)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-	int              call_count = 0;
-	int              child_index = 0;
-	call_frame_t    *frame = NULL;
-	int              active_src = 0;
+	int ret = 0;
+	gf_dirent_t entries;
+	gf_dirent_t *entry = NULL;
+	off_t offset = 0;
+	call_frame_t *iter_frame = NULL;
+	xlator_t *subvol = NULL;
+	afr_private_t *priv = NULL;
 
 	priv = this->private;
-	impunge_local = impunge_frame->local;
-	impunge_sh = &impunge_local->self_heal;
-	frame = impunge_sh->sh_frame;
-	child_index = (long) cookie;
-	active_src = impunge_sh->active_source;
+	subvol = priv->children[child];
 
-	if ((op_ret == -1 && op_errno == ENOENT)
-            || (IA_ISLNK (impunge_sh->impunging_entry_mode))) {
+	INIT_LIST_HEAD (&entries.list);
 
-                /*
-                 * A symlink's target might have changed, so
-                 * always go down the recreate path for them.
-                 */
+	iter_frame = afr_copy_frame (frame);
+	if (!iter_frame)
+		return -ENOMEM;
 
-		/* decrease call_count in recreate-callback */
+	while ((ret = syncop_readdir (subvol, fd, 131072, offset, &entries))) {
+		if (ret > 0)
+			ret = 0;
+		list_for_each_entry (entry, &entries.list, list) {
+			offset = entry->d_off;
 
-		gf_log (this->name, GF_LOG_TRACE,
-			"missing entry %s on %s",
-			impunge_local->loc.path,
-			priv->children[child_index]->name);
+			if (!strcmp (entry->d_name, ".") ||
+			    !strcmp (entry->d_name, ".."))
+				continue;
 
-		afr_sh_entry_impunge_recreate (impunge_frame, this,
-					       child_index);
-		return 0;
-	}
+			if (__is_root_gfid (fd->inode->gfid) &&
+			    !strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR))
+				continue;
 
-	if (op_ret == 0) {
-		gf_log (this->name, GF_LOG_TRACE,
-			"%s exists under %s",
-			impunge_local->loc.path,
-			priv->children[child_index]->name);
+			ret = afr_selfheal_entry_dirent (iter_frame, this, fd,
+							 source, sources,
+							 healed_sinks,
+							 entry->d_name);
+			AFR_STACK_RESET (iter_frame);
 
-                impunge_sh->parentbuf = *postparent;
-	} else {
-		gf_log (this->name, GF_LOG_TRACE,
-			"looking up %s under %s failed (%s)",
-			impunge_local->loc.path,
-			priv->children[child_index]->name,
-			strerror (op_errno));
-	}
-
-	LOCK (&impunge_frame->lock);
-	{
-		call_count = --impunge_local->call_count;
-	}
-	UNLOCK (&impunge_frame->lock);
+			if (ret)
+				break;
+		}
 
-	if (call_count == 0) {
-		AFR_STACK_DESTROY (impunge_frame);
-		afr_sh_entry_impunge_entry_done (frame, this, active_src);
+		gf_dirent_free (&entries);
+		if (ret)
+			break;
 	}
 
-	return 0;
+	AFR_STACK_DESTROY (iter_frame);
+	return ret;
 }
 
-
-int
-afr_sh_entry_impunge_entry (call_frame_t *frame, xlator_t *this,
-			    gf_dirent_t *entry)
+static int
+afr_selfheal_entry_do (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		       int source, unsigned char *sources,
+		       unsigned char *healed_sinks,
+		       struct afr_reply *locked_replies)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	int              ret = -1;
-	call_frame_t    *impunge_frame = NULL;
-	afr_local_t     *impunge_local = NULL;
-	afr_self_heal_t *impunge_sh = NULL;
-	int              active_src = 0;
-	int              i = 0;
-	int              call_count = 0;
-	int              op_errno = 0;
+	int i = 0;
+	afr_private_t *priv = NULL;
+	int ret = 0;
 
 	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
 
-	active_src = sh->active_source;
-
-	if ((strcmp (entry->d_name, ".") == 0)
-	    || (strcmp (entry->d_name, "..") == 0)
-            || ((strcmp (local->loc.path, "/") == 0)
-                && (strcmp (entry->d_name, GF_REPLICATE_TRASH_DIR) == 0))) {
-
-		gf_log (this->name, GF_LOG_TRACE,
-			"skipping inspection of %s under %s",
-			entry->d_name, local->loc.path);
-		goto out;
-	}
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"inspecting existance of %s under %s",
-		entry->d_name, local->loc.path);
-
-	impunge_frame = copy_frame (frame);
-	if (!impunge_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		goto out;
-	}
-
-	ALLOC_OR_GOTO (impunge_local, afr_local_t, out);
-
-	impunge_frame->local = impunge_local;
-	impunge_sh = &impunge_local->self_heal;
-	impunge_sh->sh_frame = frame;
-	impunge_sh->active_source = active_src;
-
-        impunge_sh->impunging_entry_mode =
-                st_mode_from_ia (entry->d_stat.ia_prot, entry->d_stat.ia_type);
-
-	ret = build_child_loc (this, &impunge_local->loc, &local->loc, entry->d_name);
-	if (ret != 0) {
-		goto out;
-	}
+	gf_log (this->name, GF_LOG_INFO, "performing entry selfheal on %s",
+		uuid_utoa (fd->inode->gfid));
 
 	for (i = 0; i < priv->child_count; i++) {
-		if (i == active_src)
-			continue;
-		if (local->child_up[i] == 0)
-			continue;
-		if (sh->sources[i] == 1)
-			continue;
-		call_count++;
-	}
-
-	impunge_local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (i == active_src)
-			continue;
-		if (local->child_up[i] == 0)
-			continue;
-		if (sh->sources[i] == 1)
+		if (i != source && !healed_sinks[i])
 			continue;
-
-		gf_log (this->name, GF_LOG_TRACE,
-			"looking up %s on %s", impunge_local->loc.path,
-			priv->children[i]->name);
-
-		STACK_WIND_COOKIE (impunge_frame,
-				   afr_sh_entry_impunge_entry_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->lookup,
-				   &impunge_local->loc, 0);
-
-		if (!--call_count)
+		ret = afr_selfheal_entry_do_subvol (frame, this, fd, i, source,
+						    sources, healed_sinks);
+		if (ret)
 			break;
 	}
-
-	ret = 0;
-out:
-	if (ret == -1)
-		afr_sh_entry_impunge_entry_done (frame, this, active_src);
-
-	return 0;
+	return ret;
 }
 
 
-int
-afr_sh_entry_impunge_readdir_cbk (call_frame_t *frame, void *cookie,
-				  xlator_t *this,
-				  int32_t op_ret, int32_t op_errno,
-				  gf_dirent_t *entries)
+static int
+__afr_selfheal_entry_finalize_source (xlator_t *this, unsigned char *sources,
+				      unsigned char *sinks,
+				      unsigned char *locked_on,
+				      struct afr_reply *replies)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	gf_dirent_t     *entry = NULL;
-	off_t            last_offset = 0;
-	int              active_src = 0;
-	int              entry_count = 0;
+	int i = 0;
+	afr_private_t *priv = NULL;
+	int source = -1;
+	int locked_count = 0;
+	int sources_count = 0;
+	int sinks_count = 0;
 
 	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	active_src = sh->active_source;
-
-	if (op_ret <= 0) {
-		if (op_ret < 0) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"readdir of %s on subvolume %s failed (%s)",
-				local->loc.path,
-				priv->children[active_src]->name,
-				strerror (op_errno));
-		} else {
-			gf_log (this->name, GF_LOG_TRACE,
-				"readdir of %s on subvolume %s complete",
-				local->loc.path,
-				priv->children[active_src]->name);
-		}
 
-		afr_sh_entry_impunge_all (frame, this);
-		return 0;
-	}
+	locked_count = AFR_COUNT (locked_on, priv->child_count);
+	sources_count = AFR_COUNT (sources, priv->child_count);
+	sinks_count = AFR_COUNT (sinks, priv->child_count);
 
-	list_for_each_entry (entry, &entries->list, list) {
-		last_offset = entry->d_off;
-                entry_count++;
+	if (locked_count == sinks_count || !sources_count) {
+		return -1;
 	}
 
-	gf_log (this->name, GF_LOG_TRACE,
-		"readdir'ed %d entries from %s",
-		entry_count, priv->children[active_src]->name);
-
-	sh->offset = last_offset;
-	local->call_count = entry_count;
-
-	list_for_each_entry (entry, &entries->list, list) {
-                afr_sh_entry_impunge_entry (frame, this, entry);
+	for (i = 0; i < priv->child_count; i++) {
+		if (sources[i]) {
+			source = i;
+			break;
+		}
 	}
 
-	return 0;
-}
-				  
-
-int
-afr_sh_entry_impunge_subvol (call_frame_t *frame, xlator_t *this,
-			     int active_src)
-{
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-
-	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
-
-	STACK_WIND (frame, afr_sh_entry_impunge_readdir_cbk,
-		    priv->children[active_src],
-		    priv->children[active_src]->fops->readdirp,
-		    sh->healing_fd, sh->block_size, sh->offset);
-
-	return 0;
+	return source;
 }
 
 
-int
-afr_sh_entry_impunge_all (call_frame_t *frame, xlator_t *this)
+static int
+__afr_selfheal_entry_prepare (call_frame_t *frame, xlator_t *this, fd_t *fd,
+			      unsigned char *locked_on, unsigned char *sources,
+			      unsigned char *sinks, unsigned char *healed_sinks,
+			      struct afr_reply *replies, int *source_p)
 {
-	afr_private_t   *priv = NULL;
-	afr_local_t     *local  = NULL;
-	afr_self_heal_t *sh  = NULL;
-	int              active_src = -1;
+	int ret = -1;
+	int source = -1;
+	afr_private_t *priv = NULL;
+	int i = 0;
 
 	priv = this->private;
-	local = frame->local;
-	sh = &local->self_heal;
 
-	sh->offset = 0;
+	ret = afr_selfheal_unlocked_discover (frame, fd->inode, fd->inode->gfid,
+					      replies);
+	if (ret)
+		return ret;
 
-	active_src = next_active_source (frame, this, sh->active_source);
-	sh->active_source = active_src;
+	ret = afr_selfheal_find_direction (frame, this, replies,
+					   AFR_ENTRY_TRANSACTION,
+					   locked_on, sources, sinks);
+	if (ret)
+		return ret;
 
-	if (sh->op_failed) {
-		afr_sh_entry_finish (frame, this);
-		return 0;
+	source = __afr_selfheal_entry_finalize_source (this, sources, sinks,
+						       locked_on, replies);
+	if (source < 0) {
+		/* If source is < 0 (typically split-brain), we perform a
+		   conservative merge of entries rather than erroring out */
 	}
+	*source_p = source;
 
-	if (active_src == -1) {
-		/* completed creating missing files on all subvolumes */
-		afr_sh_entry_expunge_all (frame, this);
-		return 0;
-	}
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"impunging entries of %s on %s to other sinks",
-		local->loc.path, priv->children[active_src]->name);
+	for (i = 0; i < priv->child_count; i++)
+		/* Initialize the healed_sinks[] array optimistically to
+		   the intersection of to-be-healed (i.e sinks[]) and
+		   the list of servers which are up (i.e locked_on[]).
 
-	afr_sh_entry_impunge_subvol (frame, this, active_src);
+		   As we encounter failures in the healing process, we
+		   will unmark the respective servers in the healed_sinks[]
+		   array.
+		*/
+		healed_sinks[i] = sinks[i] && locked_on[i];
 
-	return 0;
+	return ret;
 }
 
 
-int
-afr_sh_entry_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			  int32_t op_ret, int32_t op_errno, fd_t *fd)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              call_count = 0;
-	int              child_index = 0;
-
-	local = frame->local;
-	sh = &local->self_heal;
+static int
+__afr_selfheal_entry (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		      unsigned char *locked_on)
+{
+	afr_private_t *priv = NULL;
+	int ret = -1;
+	unsigned char *sources = NULL;
+	unsigned char *sinks = NULL;
+	unsigned char *data_lock = NULL;
+	unsigned char *healed_sinks = NULL;
+	struct afr_reply *locked_replies = NULL;
+	int source = -1;
+
 	priv = this->private;
 
-	child_index = (long) cookie;
+	sources = alloca0 (priv->child_count);
+	sinks = alloca0 (priv->child_count);
+	healed_sinks = alloca0 (priv->child_count);
+	data_lock = alloca0 (priv->child_count);
 
-	/* TODO: some of the open's might fail.
-	   In that case, modify cleanup fn to send flush on those 
-	   fd's which are already open */
+	locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
 
-	LOCK (&frame->lock);
+	ret = afr_selfheal_entrylk (frame, this, fd->inode, this->name, NULL,
+				    data_lock);
 	{
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"opendir of %s failed on child %s (%s)",
-				local->loc.path,
-				priv->children[child_index]->name,
-				strerror (op_errno));
-			sh->op_failed = 1;
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		if (sh->op_failed) {
-			afr_sh_entry_finish (frame, this);
-			return 0;
+		if (ret < 2) {
+			ret = -ENOTCONN;
+			goto unlock;
 		}
-		gf_log (this->name, GF_LOG_TRACE,
-			"fd for %s opened, commencing sync",
-			local->loc.path);
 
-		sh->active_source = -1;
-		afr_sh_entry_impunge_all (frame, this);
+		ret = __afr_selfheal_entry_prepare (frame, this, fd, data_lock,
+						    sources, sinks, healed_sinks,
+						    locked_replies, &source);
 	}
+unlock:
+	afr_selfheal_unentrylk (frame, this, fd->inode, this->name, NULL,
+				data_lock);
+	if (ret < 0)
+		goto out;
 
-	return 0;
+	ret = afr_selfheal_entry_do (frame, this, fd, source, sources,
+				     healed_sinks, locked_replies);
+	if (ret)
+		goto out;
+
+	ret = afr_selfheal_undo_pending (frame, this, fd->inode, sources, sinks,
+					 healed_sinks, AFR_ENTRY_TRANSACTION,
+					 locked_replies, data_lock);
+out:
+	return ret;
 }
 
 
-int
-afr_sh_entry_open (call_frame_t *frame, xlator_t *this)
+static fd_t *
+afr_selfheal_data_opendir (xlator_t *this, inode_t *inode)
 {
-	int i = 0;				
-	int call_count = 0;		     
-
-	int source = -1;
-	int *sources = NULL;
-
+	loc_t loc = {0,};
+	int ret = 0;
 	fd_t *fd = NULL;
 
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	afr_self_heal_t *sh = NULL;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	source  = local->self_heal.source;
-	sources = local->self_heal.sources;
-
-	sh->block_size = 65536; //131072
-	sh->offset = 0;
-
-	call_count = sh->active_sinks;
-	if (source != -1)
-		call_count++;
-
-	local->call_count = call_count;
-
-	fd = fd_create (local->loc.inode, frame->root->pid);
-	sh->healing_fd = fd;
-
-	if (source != -1) {
-		gf_log (this->name, GF_LOG_TRACE,
-			"opening directory %s on subvolume %s (source)",
-			local->loc.path, priv->children[source]->name);
-
-		/* open source */
-		STACK_WIND_COOKIE (frame, afr_sh_entry_opendir_cbk,
-				   (void *) (long) source,
-				   priv->children[source],
-				   priv->children[source]->fops->opendir,
-				   &local->loc, fd);
-		call_count--;
-	}
-
-	/* open sinks */
-	for (i = 0; i < priv->child_count; i++) {
-		if (sources[i] || !local->child_up[i])
-			continue;
-
-		gf_log (this->name, GF_LOG_TRACE,
-			"opening directory %s on subvolume %s (sink)",
-			local->loc.path, priv->children[i]->name);
-
-		STACK_WIND_COOKIE (frame, afr_sh_entry_opendir_cbk,
-				   (void *) (long) i,
-				   priv->children[i], 
-				   priv->children[i]->fops->opendir,
-				   &local->loc, fd);
-
-		if (!--call_count)
-			break;
-	}
-
-	return 0;
-}
-
-
-int
-afr_sh_entry_sync_prepare (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              active_sinks = 0;
-	int              source = 0;
-	int              i = 0;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
+	fd = fd_create (inode, 0);
+	if (!fd)
+		return NULL;
 
-	source = sh->source;
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->sources[i] == 0 && local->child_up[i] == 1) {
-			active_sinks++;
-			sh->success[i] = 1;
-		}
-	}
-	if (source != -1)
-		sh->success[source] = 1;
-
-	if (active_sinks == 0) {
-		gf_log (this->name, GF_LOG_TRACE,
-			"no active sinks for self-heal on dir %s",
-			local->loc.path);
-		afr_sh_entry_finish (frame, this);
-		return 0;
-	}
-	if (source == -1 && active_sinks < 2) {
-		gf_log (this->name, GF_LOG_TRACE,
-			"cannot sync with 0 sources and 1 sink on dir %s",
-			local->loc.path);
-		afr_sh_entry_finish (frame, this);
-		return 0;
+	ret = syncop_opendir (this, &loc, fd);
+	if (ret) {
+		fd_unref (fd);
+		fd = NULL;
+	} else {
+		fd_bind (fd);
 	}
-	sh->active_sinks = active_sinks;
-
-	if (source != -1)
-		gf_log (this->name, GF_LOG_DEBUG,
-			"self-healing directory %s from subvolume %s to "
-                        "%d other",
-			local->loc.path, priv->children[source]->name,
-			active_sinks);
-	else
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no active sources for %s found. "
-			"merging all entries as a conservative decision",
-			local->loc.path);
-
-	afr_sh_entry_open (frame, this);
 
-	return 0;
+	loc_wipe (&loc);
+        return fd;
 }
 
 
 int
-afr_sh_entry_fix (call_frame_t *frame, xlator_t *this)
+afr_selfheal_entry (call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              source = 0;
-
-        int nsources = 0;
+	afr_private_t *priv = NULL;
+	unsigned char *locked_on = NULL;
+	fd_t *fd = NULL;
+	int ret = 0;
 
-	local = frame->local;
-	sh = &local->self_heal;
 	priv = this->private;
 
-        if (sh->forced_merge) {
-                sh->source = -1;
-                goto heal;
-        }
-
-	afr_sh_build_pending_matrix (priv, sh->pending_matrix, sh->xattr, 
-				     priv->child_count, AFR_ENTRY_TRANSACTION);
-
-	afr_sh_print_pending_matrix (sh->pending_matrix, this);
-
-	nsources = afr_sh_mark_sources (sh, priv->child_count,
-                                        AFR_SELF_HEAL_ENTRY);
-
-        if (nsources == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No self-heal needed for %s",
-                        local->loc.path);
-
-                afr_sh_entry_finish (frame, this);
-                return 0;
-        }
+	fd = afr_selfheal_data_opendir (this, inode);
+	if (!fd)
+		return -EIO;
 
-	afr_sh_supress_errenous_children (sh->sources, sh->child_errno,
-					  priv->child_count);
+	locked_on = alloca0 (priv->child_count);
 
-	source = afr_sh_select_source (sh->sources, priv->child_count);
-
-        sh->source = source;
-
-heal:
-	afr_sh_entry_sync_prepare (frame, this);
-
-	return 0;
-}
-
-
-
-int
-afr_sh_entry_lookup_cbk (call_frame_t *frame, void *cookie,
-			 xlator_t *this, int32_t op_ret, int32_t op_errno,
-                         inode_t *inode, struct iatt *buf, dict_t *xattr,
-                         struct iatt *postparent)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-
-	int call_count  = -1;
-	int child_index = (long) cookie;
-
-	local = frame->local;
-	sh = &local->self_heal;
-
-	LOCK (&frame->lock);
+	ret = afr_selfheal_tryentrylk (frame, this, inode, priv->sh_domain, NULL,
+				       locked_on);
 	{
-		if (op_ret != -1) {
-			sh->xattr[child_index] = dict_ref (xattr);
-			sh->buf[child_index] = *buf;
+		if (ret < 2) {
+			/* Either less than two subvols available, or another
+			   selfheal (from another server) is in progress. Skip
+			   for now in any case there isn't anything to do.
+			*/
+			ret = -ENOTCONN;
+			goto unlock;
 		}
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
 
-	if (call_count == 0) {
-		afr_sh_entry_fix (frame, this);
+		ret = __afr_selfheal_entry (frame, this, fd, locked_on);
 	}
+unlock:
+	afr_selfheal_unentrylk (frame, this, inode, priv->sh_domain, NULL, locked_on);
 
-	return 0;
-}
-
-
-
-int
-afr_sh_entry_lookup (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t    *  local = NULL;
-	afr_private_t  *  priv  = NULL;
-	dict_t         *xattr_req = NULL;
-	int ret = 0;
-	int call_count = 0;
-	int i = 0;
+	if (fd)
+		fd_unref (fd);
 
-	priv  = this->private;
-	local = frame->local;
-
-	call_count = afr_up_children_count (priv->child_count,
-                                            local->child_up);
-
-	local->call_count = call_count;
-
-	xattr_req = dict_new();
-	if (xattr_req) {
-                for (i = 0; i < priv->child_count; i++) {
-                        ret = dict_set_uint64 (xattr_req,
-                                               priv->pending_key[i],
-                                               3 * sizeof(int32_t));
-                        if (ret < 0)
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Unable to set dict value.");
-                }
-        }
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame,
-					   afr_sh_entry_lookup_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->lookup,
-					   &local->loc, xattr_req);
-			if (!--call_count)
-				break;
-		}
-	}
-
-	if (xattr_req)
-		dict_unref (xattr_req);
-
-	return 0;
-}
-
-int
-afr_sh_post_nonblocking_entry_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non Blocking entrylks failed.");
-                afr_sh_entry_done (frame, this);
-        } else {
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non Blocking entrylks done. Proceeding to FOP");
-                afr_sh_entry_lookup(frame, this);
-        }
-
-        return 0;
-}
-
-int
-afr_sh_entry_lock (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->transaction_lk_type = AFR_SELFHEAL_LK;
-        int_lock->selfheal_lk_type    = AFR_ENTRY_SELF_HEAL_LK;
-
-        afr_set_lock_number (frame, this);
-
-        int_lock->lk_basename = NULL;
-        int_lock->lk_loc      = &local->loc;
-        int_lock->lock_cbk    = afr_sh_post_nonblocking_entry_cbk;
-
-        afr_nonblocking_entrylk (frame, this);
-
-
-	return 0;
-}
-
-
-int
-afr_self_heal_entry (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t   *local = NULL;
-	afr_private_t   *priv = NULL;
-
-
-	priv = this->private;
-	local = frame->local;
-
-	if (local->self_heal.need_entry_self_heal && priv->entry_self_heal) {
-		afr_sh_entry_lock (frame, this);
-	} else {
-		gf_log (this->name, GF_LOG_TRACE,
-			"proceeding to completion on %s",
-			local->loc.path);
-		afr_sh_entry_done (frame, this);
-	}
-
-	return 0;
+	return ret;
 }
-
diff --git a/xlators/cluster/afr/src/afr-self-heal-metadata.c b/xlators/cluster/afr/src/afr-self-heal-metadata.c
index 124bcdb10..83628297f 100644
--- a/xlators/cluster/afr/src/afr-self-heal-metadata.c
+++ b/xlators/cluster/afr/src/afr-self-heal-metadata.c
@@ -1,731 +1,280 @@
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#include <libgen.h>
-#include <unistd.h>
-#include <fnmatch.h>
-#include <sys/time.h>
-#include <stdlib.h>
-#include <signal.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
 #endif
 
-#include "glusterfs.h"
 #include "afr.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "list.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include "compat-errno.h"
-#include "compat.h"
-#include "byte-order.h"
-
-#include "afr-transaction.h"
 #include "afr-self-heal.h"
-#include "afr-self-heal-common.h"
+#include "byte-order.h"
 
+#define AFR_HEAL_ATTR (GF_SET_ATTR_UID|GF_SET_ATTR_GID|GF_SET_ATTR_MODE)
 
 int
-afr_sh_metadata_done (call_frame_t *frame, xlator_t *this)
+afr_selfheal_metadata_do (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			  int source, unsigned char *healed_sinks,
+			  struct afr_reply *locked_replies)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              i = 0;
+	int ret = -1;
+	loc_t loc = {0,};
+	dict_t *xattr = NULL;
+	dict_t *old_xattr = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
 
-	local = frame->local;
-	sh = &local->self_heal;
 	priv = this->private;
 
-//	memset (sh->child_errno, 0, sizeof (int) * priv->child_count);
-	memset (sh->buf, 0, sizeof (struct iatt) * priv->child_count);
-	memset (sh->success, 0, sizeof (int) * priv->child_count);
-
-/*         for (i = 0; i < priv->child_count; i++) { */
-/*                 sh->locked_nodes[i] = 1; */
-/*         } */
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->xattr[i])
-			dict_unref (sh->xattr[i]);
-		sh->xattr[i] = NULL;
-	}
-
-	if (local->govinda_gOvinda) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"aborting selfheal of %s",
-			local->loc.path);
-		sh->completion_cbk (frame, this);
-	} else {
-		if (IA_ISREG (sh->type)) {
-			gf_log (this->name, GF_LOG_TRACE,
-				"proceeding to data check on %s",
-				local->loc.path);
-			afr_self_heal_data (frame, this);
-			return 0;
-		}
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.gfid, inode->gfid);
 
-		if (IA_ISDIR (sh->type)) {
-			gf_log (this->name, GF_LOG_TRACE,
-				"proceeding to entry check on %s",
-				local->loc.path);
-			afr_self_heal_entry (frame, this);
-			return 0;
-		}
-		gf_log (this->name, GF_LOG_DEBUG,
-			"completed self heal of %s",
-			local->loc.path);
+	gf_log (this->name, GF_LOG_INFO, "performing metadata selfheal on %s",
+		uuid_utoa (inode->gfid));
 
-		sh->completion_cbk (frame, this);
+	ret = syncop_getxattr (priv->children[source], &loc, &xattr, NULL);
+	if (ret < 0) {
+		loc_wipe (&loc);
+		return -EIO;
 	}
 
-	return 0;
-}
-
-
-int
-afr_sh_metadata_unlck_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int32_t op_ret, int32_t op_errno)
-{
-	int               call_count = 0;
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		afr_sh_metadata_done (frame, this);
-
-	return 0;
-}
-
-int
-afr_sh_inode_unlock (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->lock_cbk = afr_sh_metadata_done;
-        afr_unlock (frame, this);
+	afr_filter_xattrs (xattr);
+	dict_del (xattr, GF_SELINUX_XATTR_KEY);
 
-        return 0;
-}
-
-int
-afr_sh_metadata_finish (call_frame_t *frame, xlator_t *this)
-{
-        afr_sh_inode_unlock (frame, this);
-
-	return 0;
-}
-
-
-int
-afr_sh_metadata_erase_pending_cbk (call_frame_t *frame, void *cookie,
-				   xlator_t *this, int32_t op_ret,
-				   int32_t op_errno, dict_t *xattr)
-{
-	afr_local_t     *local = NULL;
-	int             call_count = 0;
+	for (i = 0; i < priv->child_count; i++) {
+		if (!healed_sinks[i])
+			continue;
 
-	local = frame->local;
+		ret = syncop_setattr (priv->children[i], &loc,
+				      &locked_replies[source].poststat,
+				      AFR_HEAL_ATTR, NULL, NULL);
+		if (ret)
+			healed_sinks[i] = 0;
+
+		old_xattr = NULL;
+		ret = syncop_getxattr (priv->children[i], &loc, &old_xattr, 0);
+		if (old_xattr) {
+			dict_del (old_xattr, GF_SELINUX_XATTR_KEY);
+			afr_filter_xattrs (old_xattr);
+			ret = syncop_removexattr (priv->children[i], &loc, "",
+						  old_xattr);
+		}
 
-	LOCK (&frame->lock);
-	{
+		ret = syncop_setxattr (priv->children[i], &loc, xattr, 0);
+		if (ret)
+			healed_sinks[i] = 0;
 	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
 
-	if (call_count == 0)
-		afr_sh_metadata_finish (frame, this);
+	loc_wipe (&loc);
+	if (xattr)
+		dict_unref (xattr);
 
 	return 0;
 }
 
 
-int
-afr_sh_metadata_erase_pending (call_frame_t *frame, xlator_t *this)
+/*
+ * Look for mismatching uid/gid or mode even if xattrs don't say so, and
+ * pick one arbitrarily as winner.
+ */
+
+static int
+__afr_selfheal_metadata_finalize_source (xlator_t *this, unsigned char *sources,
+					 unsigned char *sinks,
+					 unsigned char *locked_on,
+					 struct afr_reply *replies)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              call_count = 0;
-	int              i = 0;
-	dict_t          **erase_xattr = NULL;
-
+	int i = 0;
+	afr_private_t *priv = NULL;
+	struct iatt first = {0, };
+	int source = -1;
+	int locked_count = 0;
+	int sources_count = 0;
+	int sinks_count = 0;
 
-	local = frame->local;
-	sh = &local->self_heal;
 	priv = this->private;
 
-	afr_sh_pending_to_delta (priv, sh->xattr, sh->delta_matrix, 
-                                 sh->success, priv->child_count,
-                                 AFR_METADATA_TRANSACTION);
-
-	erase_xattr = GF_CALLOC (sizeof (*erase_xattr), priv->child_count,
-                                 gf_afr_mt_dict_t);
+	locked_count = AFR_COUNT (locked_on, priv->child_count);
+	sources_count = AFR_COUNT (sources, priv->child_count);
+	sinks_count = AFR_COUNT (sinks, priv->child_count);
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->xattr[i]) {
-			call_count++;
-
-			erase_xattr[i] = get_new_dict();
-			dict_ref (erase_xattr[i]);
+	if (locked_count == sinks_count || !sources_count) {
+		if (!priv->metadata_splitbrain_forced_heal) {
+			return -EIO;
+		}
+		/* Metadata split brain, select one subvol
+		   arbitrarily */
+		for (i = 0; i < priv->child_count; i++) {
+			if (locked_on[i] && sinks[i]) {
+				sources[i] = 1;
+				sinks[i] = 0;
+				break;
+			}
 		}
-	}
-
-	afr_sh_delta_to_xattr (priv, sh->delta_matrix, erase_xattr,
-			       priv->child_count, AFR_METADATA_TRANSACTION);
-
-	local->call_count = call_count;
-
-	if (call_count == 0) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"metadata of %s not healed on any subvolume",
-			local->loc.path);
-
-		afr_sh_metadata_finish (frame, this);
 	}
 
 	for (i = 0; i < priv->child_count; i++) {
-		if (!erase_xattr[i])
+		if (!sources[i])
 			continue;
-
-		gf_log (this->name, GF_LOG_TRACE,
-			"erasing pending flags from %s on %s",
-			local->loc.path, priv->children[i]->name);
-
-		STACK_WIND_COOKIE (frame, afr_sh_metadata_erase_pending_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->xattrop,
-				   &local->loc,
-				   GF_XATTROP_ADD_ARRAY, erase_xattr[i]);
-		if (!--call_count)
-			break;
-	}
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (erase_xattr[i]) {
-			dict_unref (erase_xattr[i]);
+		if (source == -1) {
+			source = i;
+			first = replies[i].poststat;
 		}
 	}
-	GF_FREE (erase_xattr);
-
-	return 0;
-}
-
-
-int
-afr_sh_metadata_sync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			  int32_t op_ret, int32_t op_errno)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              call_count = 0;
-	int              child_index = 0;
-
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	child_index = (long) cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"setting attributes failed for %s on %s (%s)",
-				local->loc.path,
-				priv->children[child_index]->name,
-				strerror (op_errno));
-
-			sh->success[child_index] = 0;
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		afr_sh_metadata_erase_pending (frame, this);
-
-	return 0;
-}
-
-
-int
-afr_sh_metadata_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                             int32_t op_ret, int32_t op_errno,
-                             struct iatt *preop, struct iatt *postop)
-{
-	afr_sh_metadata_sync_cbk (frame, cookie, this, op_ret, op_errno);
-
-	return 0;
-}
-
-
-int
-afr_sh_metadata_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int32_t op_ret, int32_t op_errno)
-{
-	afr_sh_metadata_sync_cbk (frame, cookie, this, op_ret, op_errno);
-
-	return 0;
-}
-
-
-int
-afr_sh_metadata_sync (call_frame_t *frame, xlator_t *this, dict_t *xattr)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              source = 0;
-	int              active_sinks = 0;
-	int              call_count = 0;
-	int              i = 0;
-
-        struct iatt      stbuf;
-        int32_t          valid = 0;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	source = sh->source;
-	active_sinks = sh->active_sinks;
-
-	/*
-	 * 2 calls per sink - setattr, setxattr
-	 */
-	if (xattr)
-		call_count = active_sinks * 2;
-	else
-		call_count = active_sinks;
-
-	local->call_count = call_count;
-
-	stbuf.ia_atime = sh->buf[source].ia_atime;
-	stbuf.ia_atime_nsec = sh->buf[source].ia_atime_nsec;
-	stbuf.ia_mtime = sh->buf[source].ia_mtime;
-	stbuf.ia_mtime_nsec = sh->buf[source].ia_mtime_nsec;
-
-        stbuf.ia_uid = sh->buf[source].ia_uid;
-        stbuf.ia_gid = sh->buf[source].ia_gid;
-
-        stbuf.ia_type = sh->buf[source].ia_type;
-        stbuf.ia_prot = sh->buf[source].ia_prot;
-        
-        valid = GF_SET_ATTR_MODE  | 
-                GF_SET_ATTR_UID   | GF_SET_ATTR_GID |
-                GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME;
 
 	for (i = 0; i < priv->child_count; i++) {
-		if (call_count == 0) {
-			break;
-		}
-		if (sh->sources[i] || !local->child_up[i])
+		if (!sources[i])
 			continue;
-
-		gf_log (this->name, GF_LOG_DEBUG,
-			"self-healing metadata of %s from %s to %s",
-			local->loc.path, priv->children[source]->name,
-			priv->children[i]->name);
-
-		STACK_WIND_COOKIE (frame, afr_sh_metadata_setattr_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->setattr,
-				   &local->loc, &stbuf, valid);
-
-		call_count--;
-
-		if (!xattr)
-			continue;
-
-		STACK_WIND_COOKIE (frame, afr_sh_metadata_xattr_cbk,
-				   (void *) (long) i,
-				   priv->children[i],
-				   priv->children[i]->fops->setxattr,
-				   &local->loc, xattr, 0);
-		call_count--;
+		if (!IA_EQUAL (first, replies[i].poststat, type) ||
+		    !IA_EQUAL (first, replies[i].poststat, uid) ||
+		    !IA_EQUAL (first, replies[i].poststat, gid) ||
+		    !IA_EQUAL (first, replies[i].poststat, prot)) {
+			sources[i] = 0;
+			sinks[i] = 1;
+		}
 	}
 
-	return 0;
+	return source;
 }
 
 
-int
-afr_sh_metadata_getxattr_cbk (call_frame_t *frame, void *cookie,
-			      xlator_t *this,
-			      int32_t op_ret, int32_t op_errno, dict_t *xattr)
+static int
+__afr_selfheal_metadata_prepare (call_frame_t *frame, xlator_t *this, inode_t *inode,
+				 unsigned char *locked_on, unsigned char *sources,
+				 unsigned char *sinks, unsigned char *healed_sinks,
+				 struct afr_reply *replies)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              source = 0;
-
-        int i;
+	int ret = -1;
+	int source = -1;
+	afr_private_t *priv = NULL;
+	int i = 0;
 
-	local = frame->local;
-	sh = &local->self_heal;
 	priv = this->private;
 
-	source = sh->source;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"getxattr of %s failed on subvolume %s (%s). proceeding without xattr",
-			local->loc.path, priv->children[source]->name,
-			strerror (op_errno));
-
-		afr_sh_metadata_sync (frame, this, NULL);
-	} else {
-                for (i = 0; i < priv->child_count; i++) {
-                        dict_del (xattr, priv->pending_key[i]);
-                }
-
-		afr_sh_metadata_sync (frame, this, xattr);
-	}
-
-	return 0;
+	ret = afr_selfheal_unlocked_discover (frame, inode, inode->gfid,
+					      replies);
+	if (ret)
+		return ret;
+
+	ret = afr_selfheal_find_direction (frame, this, replies,
+					   AFR_METADATA_TRANSACTION,
+					   locked_on, sources, sinks);
+	if (ret)
+		return ret;
+
+	source = __afr_selfheal_metadata_finalize_source (this, sources, sinks,
+							  locked_on, replies);
+	if (source < 0)
+		return -EIO;
+
+	for (i = 0; i < priv->child_count; i++)
+		/* Initialize the healed_sinks[] array optimistically to
+		   the intersection of to-be-healed (i.e sinks[]) and
+		   the list of servers which are up (i.e locked_on[]).
+
+		   As we encounter failures in the healing process, we
+		   will unmark the respective servers in the healed_sinks[]
+		   array.
+		*/
+		healed_sinks[i] = sinks[i] && locked_on[i];
+
+	return source;
 }
 
 
-int
-afr_sh_metadata_sync_prepare (call_frame_t *frame, xlator_t *this)
+static int
+__afr_selfheal_metadata (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			 unsigned char *locked_on)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              active_sinks = 0;
-	int              source = 0;
-	int              i = 0;
-
-	local = frame->local;
-	sh = &local->self_heal;
-	priv = this->private;
-
-	source = sh->source;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (sh->sources[i] == 0 && local->child_up[i] == 1) {
-			active_sinks++;
-			sh->success[i] = 1;
-		}
-	}
-	sh->success[source] = 1;
-
-	if (active_sinks == 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no active sinks for performing self-heal on file %s",
-			local->loc.path);
-		afr_sh_metadata_finish (frame, this);
-		return 0;
-	}
-	sh->active_sinks = active_sinks;
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"syncing metadata of %s from subvolume %s to %d active sinks",
-		local->loc.path, priv->children[source]->name, active_sinks);
-
-	STACK_WIND (frame, afr_sh_metadata_getxattr_cbk,
-		    priv->children[source],
-		    priv->children[source]->fops->getxattr,
-		    &local->loc, NULL);
+	afr_private_t *priv = NULL;
+	int ret = -1;
+	unsigned char *sources = NULL;
+	unsigned char *sinks = NULL;
+	unsigned char *data_lock = NULL;
+	unsigned char *healed_sinks = NULL;
+	struct afr_reply *locked_replies = NULL;
+	int source = -1;
 
-	return 0;
-}
-
-
-int
-afr_sh_metadata_fix (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              nsources = 0;
-	int              source = 0;
-	int              i = 0;
-
-	local = frame->local;
-	sh = &local->self_heal;
 	priv = this->private;
 
-	afr_sh_build_pending_matrix (priv, sh->pending_matrix, sh->xattr, 
-				     priv->child_count, 
-                                     AFR_METADATA_TRANSACTION);
-
-	afr_sh_print_pending_matrix (sh->pending_matrix, this);
+	sources = alloca0 (priv->child_count);
+	sinks = alloca0 (priv->child_count);
+	healed_sinks = alloca0 (priv->child_count);
+	data_lock = alloca0 (priv->child_count);
 
-	nsources = afr_sh_mark_sources (sh, priv->child_count,
-                                        AFR_SELF_HEAL_METADATA);
+	locked_replies = alloca0 (sizeof (*locked_replies) * priv->child_count);
 
-	afr_sh_supress_errenous_children (sh->sources, sh->child_errno,
-					  priv->child_count);
-
-        if (nsources == 0) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No self-heal needed for %s",
-                        local->loc.path);
-
-                afr_sh_metadata_finish (frame, this);
-                return 0;
-        }
-
-	if ((nsources == -1)
-	    && (priv->favorite_child != -1)
-	    && (sh->child_errno[priv->favorite_child] == 0)) {
-
-		gf_log (this->name, GF_LOG_WARNING,
-			"Picking favorite child %s as authentic source to resolve conflicting metadata of %s",
-			priv->children[priv->favorite_child]->name,
-			local->loc.path);
-
-		sh->sources[priv->favorite_child] = 1;
-
-		nsources = afr_sh_source_count (sh->sources,
-						priv->child_count);
-	}
-
-	if (nsources == -1) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Unable to self-heal permissions/ownership of '%s' "
-                        "(possible split-brain). Please fix the file on "
-                        "all backend volumes", local->loc.path);
-
-		local->govinda_gOvinda = 1;
-
-		afr_sh_metadata_finish (frame, this);
-		return 0;
-	}
-
-	source = afr_sh_select_source (sh->sources, priv->child_count);
-
-        if (source == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No active sources found.");
-
-                afr_sh_metadata_finish (frame, this);
-                return 0;
-        }
-
-	sh->source = source;
-
-	/* detect changes not visible through pending flags -- JIC */
-	for (i = 0; i < priv->child_count; i++) {
-		if (i == source || sh->child_errno[i])
-			continue;
+	ret = afr_selfheal_inodelk (frame, this, inode, this->name,
+				    LLONG_MAX - 1, 0, data_lock);
+	{
+		if (ret < 2) {
+			ret = -ENOTCONN;
+			goto unlock;
+		}
 
-		if (PERMISSION_DIFFERS (&sh->buf[i], &sh->buf[source]))
-			sh->sources[i] = 0;
+		ret = __afr_selfheal_metadata_prepare (frame, this, inode, data_lock,
+						       sources, sinks, healed_sinks,
+						       locked_replies);
+		if (ret < 0)
+			goto unlock;
 
-		if (OWNERSHIP_DIFFERS (&sh->buf[i], &sh->buf[source]))
-			sh->sources[i] = 0;
+		source = ret;
+		ret = 0;
 	}
-
-	afr_sh_metadata_sync_prepare (frame, this);
-
-	return 0;
+unlock:
+	afr_selfheal_uninodelk (frame, this, inode, this->name,
+				LLONG_MAX -1, 0, data_lock);
+	if (ret < 0)
+		goto out;
+
+	ret = afr_selfheal_metadata_do (frame, this, inode, source, healed_sinks,
+					locked_replies);
+	if (ret)
+		goto out;
+
+	ret = afr_selfheal_undo_pending (frame, this, inode, sources, sinks,
+					 healed_sinks, AFR_METADATA_TRANSACTION,
+					 locked_replies, data_lock);
+out:
+	return ret;
 }
 
 
 int
-afr_sh_metadata_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			    int32_t op_ret, int32_t op_errno,
-                            inode_t *inode, struct iatt *buf, dict_t *xattr,
-                            struct iatt *postparent)
+afr_selfheal_metadata (call_frame_t *frame, xlator_t *this, inode_t *inode)
 {
-	afr_local_t     *local = NULL;
-	afr_self_heal_t *sh = NULL;
-	afr_private_t   *priv = NULL;
-	int              call_count = 0;
-	int              child_index = 0;
-
+	afr_private_t *priv = NULL;
+	unsigned char *locked_on = NULL;
+	int ret = 0;
 
-	local = frame->local;
-	sh = &local->self_heal;
 	priv = this->private;
 
-	child_index = (long) cookie;
+	locked_on = alloca0 (priv->child_count);
 
-	LOCK (&frame->lock);
+	ret = afr_selfheal_tryinodelk (frame, this, inode, priv->sh_domain, 0, 0,
+				       locked_on);
 	{
-		if (op_ret == 0) {
-			gf_log (this->name, GF_LOG_TRACE,
-				"path %s on subvolume %s is of mode 0%o",
-				local->loc.path,
-				priv->children[child_index]->name,
-				buf->ia_type);
-
-			sh->buf[child_index] = *buf;
-			if (xattr)
-				sh->xattr[child_index] = dict_ref (xattr);
-		} else {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"path %s on subvolume %s => -1 (%s)",
-				local->loc.path,
-				priv->children[child_index]->name,
-				strerror (op_errno));
-
-			sh->child_errno[child_index] = op_errno;
+		if (ret < 2) {
+			/* Either less than two subvols available, or another
+			   selfheal (from another server) is in progress. Skip
+			   for now in any case there isn't anything to do.
+			*/
+			ret = -ENOTCONN;
+			goto unlock;
 		}
-	}
-	UNLOCK (&frame->lock);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0)
-		afr_sh_metadata_fix (frame, this);
 
-	return 0;
-}
-
-
-int
-afr_sh_metadata_lookup (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t     *local = NULL;
-	afr_private_t   *priv = NULL;
-	int              i = 0;
-	int              call_count = 0;
-	dict_t          *xattr_req = NULL;
-	int              ret = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	call_count = afr_up_children_count (priv->child_count,
-                                            local->child_up);
-	local->call_count = call_count;
-
-	xattr_req = dict_new();
-
-	if (xattr_req) {
-                for (i = 0; i < priv->child_count; i++) {
-                        ret = dict_set_uint64 (xattr_req,
-                                               priv->pending_key[i],
-                                               3 * sizeof(int32_t));
-                        if (ret < 0)
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "Unable to set dict value.");
-                }
-        }
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			gf_log (this->name, GF_LOG_TRACE,
-				"looking up %s on %s",
-				local->loc.path, priv->children[i]->name);
-
-			STACK_WIND_COOKIE (frame, afr_sh_metadata_lookup_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->lookup,
-					   &local->loc, xattr_req);
-			if (!--call_count)
-				break;
-		}
+		ret = __afr_selfheal_metadata (frame, this, inode, locked_on);
 	}
-	
-	if (xattr_req)
-		dict_unref (xattr_req);
+unlock:
+	afr_selfheal_uninodelk (frame, this, inode, priv->sh_domain, 0, 0, locked_on);
 
-	return 0;
+	return ret;
 }
-
-int
-afr_sh_post_nonblocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non Blocking inodelks failed.");
-                afr_sh_metadata_done (frame, this);
-        } else {
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Non Blocking inodelks done. Proceeding to FOP");
-                afr_sh_metadata_lookup (frame, this);
-        }
-
-        return 0;
-}
-
-int
-afr_sh_metadata_lock (call_frame_t *frame, xlator_t *this)
-{
-        afr_internal_lock_t *int_lock = NULL;
-        afr_local_t         *local    = NULL;
-
-        local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        int_lock->transaction_lk_type = AFR_SELFHEAL_LK;
-        int_lock->selfheal_lk_type    = AFR_METADATA_SELF_HEAL_LK;
-
-        afr_set_lock_number (frame, this);
-
-        int_lock->lk_flock.l_start = 0;
-        int_lock->lk_flock.l_len   = 0;
-        int_lock->lk_flock.l_type  = F_WRLCK;
-        int_lock->lock_cbk         = afr_sh_post_nonblocking_inodelk_cbk;
-
-        afr_nonblocking_inodelk (frame, this);
-
-	return 0;
-}
-
-
-int
-afr_self_heal_metadata (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t   *local = NULL;
-	afr_private_t *priv = this->private;
-
-
-	local = frame->local;
-
-	if (local->self_heal.need_metadata_self_heal && priv->metadata_self_heal) {
-		afr_sh_metadata_lock (frame, this);
-	} else {
-		afr_sh_metadata_done (frame, this);
-	}
-
-	return 0;
-}
-
diff --git a/xlators/cluster/afr/src/afr-self-heal-name.c b/xlators/cluster/afr/src/afr-self-heal-name.c
new file mode 100644
index 000000000..ce80b8da3
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-self-heal-name.c
@@ -0,0 +1,457 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "afr.h"
+#include "afr-self-heal.h"
+
+
+int
+__afr_selfheal_assign_gfid (call_frame_t *frame, xlator_t *this, inode_t *parent,
+			    uuid_t pargfid, const char *bname, inode_t *inode,
+			    struct afr_reply *replies, int gfid_idx)
+{
+	int i = 0;
+	afr_private_t *priv = NULL;
+	dict_t *xdata = NULL;
+	int ret = 0;
+	loc_t loc = {0, };
+
+	priv = this->private;
+
+	uuid_copy (parent->gfid, pargfid);
+
+	xdata = dict_new ();
+	if (!xdata) {
+		return -ENOMEM;
+	}
+
+	ret = dict_set_static_bin (xdata, "gfid-req",
+				   replies[gfid_idx].poststat.ia_gfid, 16);
+	if (ret) {
+		dict_destroy (xdata);
+		return -ENOMEM;
+	}
+
+	loc.parent = inode_ref (parent);
+	loc.inode = inode_ref (inode);
+	uuid_copy (loc.pargfid, pargfid);
+	loc.name = bname;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (replies[i].op_ret == 0 || replies[i].op_errno != ENODATA)
+			continue;
+
+		ret = syncop_lookup (priv->children[i], &loc, xdata, 0, 0, 0);
+	}
+
+	loc_wipe (&loc);
+	dict_unref (xdata);
+
+	return ret;
+}
+
+
+int
+__afr_selfheal_name_impunge (call_frame_t *frame, xlator_t *this, inode_t *parent,
+			     uuid_t pargfid, const char *bname, inode_t *inode,
+			     struct afr_reply *replies, int gfid_idx)
+{
+	int i = 0;
+	afr_private_t *priv = NULL;
+	int ret = 0;
+
+	priv = this->private;
+
+	uuid_copy (parent->gfid, pargfid);
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (uuid_compare (replies[i].poststat.ia_gfid,
+				  replies[gfid_idx].poststat.ia_gfid) == 0)
+			continue;
+
+		ret |= afr_selfheal_recreate_entry (frame, this, i, gfid_idx,
+						    parent, bname, inode, replies);
+	}
+
+	return ret;
+}
+
+
+int
+__afr_selfheal_name_expunge (call_frame_t *frame, xlator_t *this, inode_t *parent,
+			     uuid_t pargfid, const char *bname, inode_t *inode,
+			     struct afr_reply *replies)
+{
+	loc_t loc = {0, };
+	int i = 0;
+	afr_private_t *priv = NULL;
+	char g[64];
+	int ret = 0;
+
+	priv = this->private;
+
+	loc.parent = inode_ref (parent);
+	uuid_copy (loc.pargfid, pargfid);
+	loc.name = bname;
+	loc.inode = inode_ref (inode);
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (replies[i].op_ret)
+			continue;
+
+		switch (replies[i].poststat.ia_type) {
+		case IA_IFDIR:
+			gf_log (this->name, GF_LOG_WARNING,
+				"expunging dir %s/%s (%s) on %s",
+				uuid_utoa (pargfid), bname,
+				uuid_utoa_r (replies[i].poststat.ia_gfid, g),
+				priv->children[i]->name);
+			ret |= syncop_rmdir (priv->children[i], &loc, 1);
+			break;
+		default:
+			gf_log (this->name, GF_LOG_WARNING,
+				"expunging file %s/%s (%s) on %s",
+				uuid_utoa (pargfid), bname,
+				uuid_utoa_r (replies[i].poststat.ia_gfid, g),
+				priv->children[i]->name);
+			ret |= syncop_unlink (priv->children[i], &loc);
+			break;
+		}
+	}
+
+	loc_wipe (&loc);
+
+	return ret;
+
+}
+
+
+int
+__afr_selfheal_name_do (call_frame_t *frame, xlator_t *this, inode_t *parent,
+			uuid_t pargfid, const char *bname, inode_t *inode,
+			unsigned char *sources, unsigned char *sinks,
+			unsigned char *healed_sinks, int source,
+			unsigned char *locked_on, struct afr_reply *replies)
+{
+	int i = 0;
+	afr_private_t *priv = NULL;
+	uuid_t gfid = {0, };
+	int gfid_idx = -1;
+	gf_boolean_t source_is_empty = _gf_true;
+	gf_boolean_t need_heal = _gf_false;
+	int first_idx = -1;
+	char g1[64],g2[64];
+
+	priv = this->private;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (first_idx == -1) {
+			first_idx = i;
+			continue;
+		}
+
+		if (replies[i].op_ret != replies[first_idx].op_ret)
+			need_heal = _gf_true;
+
+		if (uuid_compare (replies[i].poststat.ia_gfid,
+				  replies[first_idx].poststat.ia_gfid))
+			need_heal = _gf_true;
+	}
+
+	if (!need_heal)
+		return 0;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (!replies[i].op_ret && (source == -1 || sources[i])) {
+			source_is_empty = _gf_false;
+			break;
+		}
+	}
+
+	if (source_is_empty) {
+		return __afr_selfheal_name_expunge (frame, this, parent, pargfid,
+						    bname, inode, replies);
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (uuid_is_null (replies[i].poststat.ia_gfid))
+			continue;
+
+		if (uuid_is_null (gfid)) {
+			uuid_copy (gfid, replies[i].poststat.ia_gfid);
+			gfid_idx = i;
+			continue;
+		}
+
+		if (sources[i] || source == -1) {
+			if (gfid_idx != -1 &&
+			    (sources[gfid_idx] || source == -1) &&
+			    uuid_compare (gfid, replies[i].poststat.ia_gfid)) {
+				gf_log (this->name, GF_LOG_WARNING,
+					"GFID mismatch for <gfid:%s>/%s "
+					"%s on %s and %s on %s",
+					uuid_utoa (pargfid), bname,
+					uuid_utoa_r (replies[i].poststat.ia_gfid, g1),
+					priv->children[i]->name,
+					uuid_utoa_r (replies[gfid_idx].poststat.ia_gfid, g2),
+					priv->children[gfid_idx]->name);
+				return -1;
+			}
+
+			uuid_copy (gfid, replies[i].poststat.ia_gfid);
+			gfid_idx = i;
+			continue;
+		}
+	}
+
+	if (gfid_idx == -1)
+		return -1;
+
+	__afr_selfheal_assign_gfid (frame, this, parent, pargfid, bname, inode,
+				    replies, gfid_idx);
+
+	return __afr_selfheal_name_impunge (frame, this, parent, pargfid,
+					    bname, inode, replies, gfid_idx);
+}
+
+
+int
+__afr_selfheal_name_finalize_source (xlator_t *this, unsigned char *sources,
+				     unsigned char *sinks, unsigned char *locked_on,
+				     struct afr_reply *replies)
+{
+	int i = 0;
+	afr_private_t *priv = NULL;
+	int source = -1;
+	int locked_count = 0;
+	int sources_count = 0;
+	int sinks_count = 0;
+
+	priv = this->private;
+
+	locked_count = AFR_COUNT (locked_on, priv->child_count);
+	sources_count = AFR_COUNT (sources, priv->child_count);
+	sinks_count = AFR_COUNT (sinks, priv->child_count);
+
+	if (locked_count == sinks_count || !sources_count) {
+		return -1;
+	}
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (sources[i]) {
+			source = i;
+			break;
+		}
+	}
+
+	return source;
+}
+
+
+int
+__afr_selfheal_name_prepare (call_frame_t *frame, xlator_t *this, inode_t *parent,
+			     uuid_t pargfid, unsigned char *locked_on,
+			     unsigned char *sources, unsigned char *sinks,
+			     unsigned char *healed_sinks, struct afr_reply *replies,
+			     int *source_p)
+{
+	int ret = -1;
+	int source = -1;
+	afr_private_t *priv = NULL;
+	int i = 0;
+
+	priv = this->private;
+
+	ret = afr_selfheal_unlocked_discover (frame, parent, pargfid, replies);
+	if (ret)
+		return ret;
+
+	ret = afr_selfheal_find_direction (frame, this, replies,
+					   AFR_ENTRY_TRANSACTION,
+					   locked_on, sources, sinks);
+	if (ret)
+		return ret;
+
+	source = __afr_selfheal_name_finalize_source (this, sources, sinks,
+						      locked_on, replies);
+	if (source < 0) {
+		/* If source is < 0 (typically split-brain), we perform a
+		   conservative merge of entries rather than erroring out */
+	}
+	*source_p = source;
+
+	for (i = 0; i < priv->child_count; i++)
+		/* Initialize the healed_sinks[] array optimistically to
+		   the intersection of to-be-healed (i.e sinks[]) and
+		   the list of servers which are up (i.e locked_on[]).
+
+		   As we encounter failures in the healing process, we
+		   will unmark the respective servers in the healed_sinks[]
+		   array.
+		*/
+		healed_sinks[i] = sinks[i] && locked_on[i];
+
+	return ret;
+}
+
+
+int
+afr_selfheal_name_do (call_frame_t *frame, xlator_t *this, inode_t *parent,
+		      uuid_t pargfid, const char *bname)
+{
+	afr_private_t *priv = NULL;
+	unsigned char *sources = NULL;
+	unsigned char *sinks = NULL;
+	unsigned char *healed_sinks = NULL;
+	unsigned char *locked_on = NULL;
+	int source = -1;
+	struct afr_reply *replies = NULL;
+	int ret = -1;
+	inode_t *inode = NULL;
+
+	priv = this->private;
+
+	locked_on = alloca0 (priv->child_count);
+	sources = alloca0 (priv->child_count);
+	sinks = alloca0 (priv->child_count);
+	healed_sinks = alloca0 (priv->child_count);
+
+	replies = alloca0 (priv->child_count * sizeof(*replies));
+
+	ret = afr_selfheal_entrylk (frame, this, parent, this->name, bname,
+				    locked_on);
+	{
+		if (ret < 2) {
+			ret = -ENOTCONN;
+			goto unlock;
+		}
+
+		ret = __afr_selfheal_name_prepare (frame, this, parent, pargfid,
+						   locked_on, sources, sinks,
+						   healed_sinks, replies,
+						   &source);
+		if (ret)
+			goto unlock;
+
+		inode = afr_selfheal_unlocked_lookup_on (frame, parent, bname,
+							 replies, locked_on);
+		if (!inode) {
+			ret = -ENOMEM;
+			goto unlock;
+		}
+
+		ret = __afr_selfheal_name_do (frame, this, parent, pargfid, bname,
+					      inode, sources, sinks, healed_sinks,
+					      source, locked_on, replies);
+	}
+unlock:
+	afr_selfheal_unentrylk (frame, this, parent, this->name, bname,
+				locked_on);
+	if (inode)
+		inode_unref (inode);
+
+	return ret;
+}
+
+
+int
+afr_selfheal_name_unlocked_inspect (call_frame_t *frame, xlator_t *this,
+				    inode_t *parent, uuid_t pargfid,
+				    const char *bname, gf_boolean_t *need_heal)
+{
+	afr_private_t *priv = NULL;
+	int i = 0;
+	struct afr_reply *replies = NULL;
+	inode_t *inode = NULL;
+	int first_idx = -1;
+
+	priv = this->private;
+
+	replies = alloca0 (sizeof (*replies) * priv->child_count);
+
+	inode = afr_selfheal_unlocked_lookup_on (frame, parent, bname,
+						 replies, priv->child_up);
+	if (!inode)
+		return -ENOMEM;
+
+	for (i = 0; i < priv->child_count; i++) {
+		if (!replies[i].valid)
+			continue;
+
+		if (first_idx == -1) {
+			first_idx = i;
+			continue;
+		}
+
+		if (replies[i].op_ret != replies[first_idx].op_ret)
+			*need_heal = _gf_true;
+
+		if (uuid_compare (replies[i].poststat.ia_gfid,
+				  replies[first_idx].poststat.ia_gfid))
+			*need_heal = _gf_true;
+	}
+
+	if (inode)
+		inode_unref (inode);
+	return 0;
+}
+
+int
+afr_selfheal_name (xlator_t *this, uuid_t pargfid, const char *bname)
+{
+	inode_t *parent = NULL;
+	call_frame_t *frame = NULL;
+	int ret = -1;
+	gf_boolean_t need_heal = _gf_false;
+
+	parent = afr_inode_find (this, pargfid);
+	if (!parent)
+		goto out;
+
+	frame = afr_frame_create (this);
+	if (!frame)
+		goto out;
+
+	ret = afr_selfheal_name_unlocked_inspect (frame, this, parent, pargfid,
+						  bname, &need_heal);
+	if (ret)
+		goto out;
+
+	if (need_heal)
+		afr_selfheal_name_do (frame, this, parent, pargfid, bname);
+out:
+	if (parent)
+		inode_unref (parent);
+	if (frame)
+		AFR_STACK_DESTROY (frame);
+
+	return ret;
+}
diff --git a/xlators/cluster/afr/src/afr-self-heal.h b/xlators/cluster/afr/src/afr-self-heal.h
index 63354926b..a1b972ac3 100644
--- a/xlators/cluster/afr/src/afr-self-heal.h
+++ b/xlators/cluster/afr/src/afr-self-heal.h
@@ -1,54 +1,167 @@
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#ifndef __AFR_SELF_HEAL_H__
-#define __AFR_SELF_HEAL_H__
 
-#include <sys/stat.h>
+#ifndef _AFR_SELFHEAL_H
+#define _AFR_SELFHEAL_H
+
+
+/* Perform fop on all UP subvolumes and wait for all callbacks to return */
+
+#define AFR_ONALL(frame, rfn, fop, args ...) do {			\
+	afr_local_t *__local = frame->local;				\
+	afr_private_t *__priv = frame->this->private;			\
+	int __i = 0, __count = 0;					\
+									\
+	afr_replies_wipe (__local, __priv);				\
+									\
+	for (__i = 0; __i < __priv->child_count; __i++) {		\
+		if (!__priv->child_up[__i]) continue;			\
+		STACK_WIND_COOKIE (frame, rfn, (void *)(long) __i,	\
+				   __priv->children[__i],		\
+				   __priv->children[__i]->fops->fop, args); \
+		__count++;						\
+	}								\
+	syncbarrier_wait (&__local->barrier, __count);			\
+	} while (0)
+
+
+/* Perform fop on all subvolumes represented by list[] array and wait
+   for all callbacks to return */
+
+#define AFR_ONLIST(list, frame, rfn, fop, args ...) do {		\
+	afr_local_t *__local = frame->local;				\
+	afr_private_t *__priv = frame->this->private;			\
+	int __i = 0, __count = 0;					\
+									\
+	afr_replies_wipe (__local, __priv);				\
+									\
+	for (__i = 0; __i < __priv->child_count; __i++) {		\
+		if (!list[__i]) continue;				\
+		STACK_WIND_COOKIE (frame, rfn, (void *)(long) __i,	\
+				   __priv->children[__i],		\
+				   __priv->children[__i]->fops->fop, args); \
+		__count++;						\
+	}								\
+	syncbarrier_wait (&__local->barrier, __count);			\
+	} while (0)
+
+
+#define AFR_SEQ(frame, rfn, fop, args ...) do {				\
+	afr_local_t *__local = frame->local;				\
+	afr_private_t *__priv = frame->this->private;			\
+	int __i = 0;							\
+									\
+	afr_replies_wipe (__local, __priv);				\
+									\
+	for (__i = 0; __i < __priv->child_count; __i++) {		\
+		if (!__priv->child_up[__i]) continue;			\
+		STACK_WIND_COOKIE (frame, rfn, (void *)(long) __i,	\
+				   __priv->children[__i],		\
+				   __priv->children[__i]->fops->fop, args); \
+		syncbarrier_wait (&__local->barrier, 1);		\
+	}								\
+	} while (0)
+
+
+#define ALLOC_MATRIX(n, type) ({type **__ptr = NULL; \
+	int __i; \
+	__ptr = alloca0 (n * sizeof(type *)); \
+	for (__i = 0; __i < n; __i++) __ptr[__i] = alloca0 (n * sizeof(type)); \
+	__ptr;})
+
+
+#define IA_EQUAL(f,s,field) (memcmp (&(f.ia_##field), &(s.ia_##field), sizeof (s.ia_##field)) == 0)
+
+
+int
+afr_selfheal (xlator_t *this, uuid_t gfid);
+
+int
+afr_selfheal_name (xlator_t *this, uuid_t gfid, const char *name);
+
+int
+afr_selfheal_data (call_frame_t *frame, xlator_t *this, inode_t *inode);
+
+int
+afr_selfheal_metadata (call_frame_t *frame, xlator_t *this, inode_t *inode);
+
+int
+afr_selfheal_entry (call_frame_t *frame, xlator_t *this, inode_t *inode);
+
+
+int
+afr_selfheal_inodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		      char *dom, off_t off, size_t size,
+		      unsigned char *locked_on);
+
+int
+afr_selfheal_tryinodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			 char *dom, off_t off, size_t size,
+			 unsigned char *locked_on);
 
-#define FILETYPE_DIFFERS(buf1,buf2) ((buf1)->ia_type != (buf2)->ia_type)
-#define PERMISSION_DIFFERS(buf1,buf2) (st_mode_from_ia ((buf1)->ia_prot, (buf1)->ia_type) != st_mode_from_ia ((buf2)->ia_prot, (buf2)->ia_type))
-#define OWNERSHIP_DIFFERS(buf1,buf2) (((buf1)->ia_uid != (buf2)->ia_uid) || ((buf1)->ia_gid != (buf2)->ia_gid))
-#define SIZE_DIFFERS(buf1,buf2) ((buf1)->ia_size != (buf2)->ia_size)
+int
+afr_selfheal_uninodelk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			char *dom, off_t off, size_t size,
+			const unsigned char *locked_on);
 
-#define SIZE_GREATER(buf1,buf2) ((buf1)->ia_size > (buf2)->ia_size)
+int
+afr_selfheal_entrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		      char *dom, const char *name, unsigned char *locked_on);
 
 int
-afr_sh_has_metadata_pending (dict_t *xattr, int child_count, xlator_t *this);
+afr_selfheal_tryentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			 char *dom, const char *name, unsigned char *locked_on);
+
 int
-afr_sh_has_entry_pending (dict_t *xattr, int child_count, xlator_t *this);
+afr_selfheal_unentrylk (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			char *dom, const char *name, unsigned char *locked_on);
+
 int
-afr_sh_has_data_pending (dict_t *xattr, int child_count, xlator_t *this);
+afr_selfheal_unlocked_discover (call_frame_t *frame, inode_t *inode,
+				uuid_t gfid, struct afr_reply *replies);
+
+inode_t *
+afr_selfheal_unlocked_lookup_on (call_frame_t *frame, inode_t *parent,
+				 const char *name, struct afr_reply *replies,
+				 unsigned char *lookup_on);
 
 int
-afr_self_heal_entry (call_frame_t *frame, xlator_t *this);
+afr_selfheal_find_direction (call_frame_t *frame, xlator_t *this,
+			     struct afr_reply *replies,
+			     afr_transaction_type type, unsigned char *locked_on,
+			     unsigned char *sources, unsigned char *sinks);
 
 int
-afr_self_heal_data (call_frame_t *frame, xlator_t *this);
+afr_selfheal_extract_xattr (xlator_t *this, struct afr_reply *replies,
+			    afr_transaction_type type, int *dirty, int **matrix);
 
 int
-afr_self_heal_metadata (call_frame_t *frame, xlator_t *this);
+afr_selfheal_undo_pending (call_frame_t *frame, xlator_t *this, inode_t *inode,
+			   unsigned char *sources, unsigned char *sinks,
+			   unsigned char *healed_sinks, afr_transaction_type type,
+			   struct afr_reply *replies, unsigned char *locked_on);
 
 int
-afr_self_heal_get_source (xlator_t *this, afr_local_t *local, dict_t **xattr);
+afr_selfheal_recreate_entry (call_frame_t *frame, xlator_t *this, int dst,
+			     int source, inode_t *dir, const char *name,
+			     inode_t *inode, struct afr_reply *replies);
 
 int
-afr_self_heal (call_frame_t *frame, xlator_t *this);
+afr_selfheal_post_op (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		      int subvol, dict_t *xattr);
+
+call_frame_t *
+afr_frame_create (xlator_t *this);
+
+inode_t *
+afr_inode_find (xlator_t *this, uuid_t gfid);
 
-#endif /* __AFR_SELF_HEAL_H__ */
+#endif /* !_AFR_SELFHEAL_H */
diff --git a/xlators/cluster/afr/src/afr-self-heald.c b/xlators/cluster/afr/src/afr-self-heald.c
new file mode 100644
index 000000000..4bfe909bc
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-self-heald.c
@@ -0,0 +1,1256 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "afr.h"
+#include "afr-self-heal.h"
+#include "afr-self-heald.h"
+#include "protocol-common.h"
+
+#define SHD_INODE_LRU_LIMIT          2048
+#define AFR_EH_HEALED_LIMIT          1024
+#define AFR_EH_HEAL_FAIL_LIMIT       1024
+#define AFR_EH_SPLIT_BRAIN_LIMIT     1024
+#define AFR_STATISTICS_HISTORY_SIZE    50
+
+
+#define ASSERT_LOCAL(this, healer)				\
+	if (!afr_shd_is_subvol_local(this, healer->subvol)) {	\
+		healer->local = _gf_false;			\
+		if (safe_break (healer)) {			\
+			break;					\
+		} else {					\
+			continue;				\
+		}						\
+	} else {						\
+		healer->local = _gf_true;			\
+	}
+
+
+#define NTH_INDEX_HEALER(this, n) &((((afr_private_t *)this->private))->shd.index_healers[n])
+#define NTH_FULL_HEALER(this, n) &((((afr_private_t *)this->private))->shd.full_healers[n])
+
+int afr_shd_gfid_to_path (xlator_t *this, xlator_t *subvol, uuid_t gfid, char **path_p);
+
+char *
+afr_subvol_name (xlator_t *this, int subvol)
+{
+	afr_private_t *priv = NULL;
+
+	priv = this->private;
+	if (subvol < 0 || subvol > priv->child_count)
+		return NULL;
+
+	return priv->children[subvol]->name;
+}
+
+
+void
+afr_destroy_crawl_event_data (void *data)
+{
+        return;
+}
+
+
+void
+afr_destroy_shd_event_data (void *data)
+{
+	shd_event_t *shd_event = data;
+
+	if (!shd_event)
+		return;
+	GF_FREE (shd_event->path);
+
+        return;
+}
+
+
+gf_boolean_t
+afr_shd_is_subvol_local (xlator_t *this, int subvol)
+{
+	char *pathinfo = NULL;
+	afr_private_t *priv = NULL;
+	dict_t *xattr = NULL;
+	int ret = 0;
+	gf_boolean_t is_local = _gf_false;
+	loc_t loc = {0, };
+
+	priv = this->private;
+
+	loc.inode = this->itable->root;
+	uuid_copy (loc.gfid, loc.inode->gfid);
+
+	ret = syncop_getxattr (priv->children[subvol], &loc, &xattr,
+			       GF_XATTR_PATHINFO_KEY);
+	if (ret)
+		return _gf_false;
+	if (!xattr)
+		return _gf_false;
+
+	ret = dict_get_str (xattr, GF_XATTR_PATHINFO_KEY, &pathinfo);
+	if (ret)
+		return _gf_false;
+
+	afr_local_pathinfo (pathinfo, &is_local);
+
+	gf_log (this->name, GF_LOG_DEBUG, "subvol %s is %slocal",
+		priv->children[subvol]->name, is_local? "" : "not ");
+
+	return is_local;
+}
+
+
+int
+__afr_shd_healer_wait (struct subvol_healer *healer)
+{
+	afr_private_t *priv = NULL;
+	struct timespec wait_till = {0, };
+	int ret = 0;
+
+	priv = healer->this->private;
+
+disabled_loop:
+	wait_till.tv_sec = time (NULL) + 60;
+
+	while (!healer->rerun) {
+		ret = pthread_cond_timedwait (&healer->cond,
+					      &healer->mutex,
+					      &wait_till);
+		if (ret == ETIMEDOUT)
+			break;
+	}
+
+	ret = healer->rerun;
+	healer->rerun = 0;
+
+	if (!priv->shd.enabled)
+		goto disabled_loop;
+
+	return ret;
+}
+
+
+int
+afr_shd_healer_wait (struct subvol_healer *healer)
+{
+	int ret = 0;
+
+	pthread_mutex_lock (&healer->mutex);
+	{
+		ret = __afr_shd_healer_wait (healer);
+	}
+	pthread_mutex_unlock (&healer->mutex);
+
+	return ret;
+}
+
+
+gf_boolean_t
+safe_break (struct subvol_healer *healer)
+{
+	gf_boolean_t ret = _gf_false;
+
+	pthread_mutex_lock (&healer->mutex);
+	{
+		if (healer->rerun)
+			goto unlock;
+
+		healer->running = _gf_false;
+		ret = _gf_true;
+	}
+unlock:
+	pthread_mutex_unlock (&healer->mutex);
+
+	return ret;
+}
+
+
+inode_t *
+afr_shd_inode_find (xlator_t *this, xlator_t *subvol, uuid_t gfid)
+{
+	inode_t *inode = NULL;
+	int ret = 0;
+	loc_t loc = {0, };
+	struct iatt iatt = {0, };
+
+	inode = inode_find (this->itable, gfid);
+	if (inode)
+		goto out;
+
+	loc.inode = inode_new (this->itable);
+	if (!loc.inode)
+		goto out;
+	uuid_copy (loc.gfid, gfid);
+
+	ret = syncop_lookup (subvol, &loc, NULL, &iatt, NULL, NULL);
+	if (ret < 0)
+		goto out;
+
+	inode = inode_link (loc.inode, NULL, NULL, &iatt);
+	if (inode)
+		inode_lookup (inode);
+out:
+	loc_wipe (&loc);
+	return inode;
+}
+
+
+fd_t *
+afr_shd_index_opendir (xlator_t *this, int child)
+{
+	fd_t *fd = NULL;
+	afr_private_t *priv = NULL;
+	xlator_t *subvol = NULL;
+	loc_t rootloc = {0, };
+	inode_t *inode = NULL;
+	int ret = 0;
+	dict_t *xattr = NULL;
+	void *index_gfid = NULL;
+
+	priv = this->private;
+	subvol = priv->children[child];
+
+	rootloc.inode = inode_ref (this->itable->root);
+	uuid_copy (rootloc.gfid, rootloc.inode->gfid);
+
+	ret = syncop_getxattr (subvol, &rootloc, &xattr,
+			       GF_XATTROP_INDEX_GFID);
+	if (ret || !xattr) {
+		errno = -ret;
+		goto out;
+	}
+
+	ret = dict_get_ptr (xattr, GF_XATTROP_INDEX_GFID, &index_gfid);
+	if (ret)
+		goto out;
+
+	gf_log (this->name, GF_LOG_DEBUG, "index-dir gfid for %s: %s",
+		subvol->name, uuid_utoa (index_gfid));
+
+	inode = afr_shd_inode_find (this, subvol, index_gfid);
+	if (!inode)
+		goto out;
+	fd = fd_anonymous (inode);
+out:
+	loc_wipe (&rootloc);
+	if (xattr)
+		dict_unref (xattr);
+	return fd;
+}
+
+
+int
+afr_shd_index_purge (xlator_t *subvol, inode_t *inode, char *name)
+{
+	loc_t loc = {0, };
+	int ret = 0;
+
+	loc.parent = inode_ref (inode);
+	loc.name = name;
+
+	ret = syncop_unlink (subvol, &loc);
+
+	loc_wipe (&loc);
+	return ret;
+}
+
+
+int
+afr_shd_selfheal_name (struct subvol_healer *healer, int child, uuid_t parent,
+		       const char *bname)
+{
+	int ret = -1;
+
+	ret = afr_selfheal_name (THIS, parent, bname);
+
+	return ret;
+}
+
+int
+afr_shd_selfheal (struct subvol_healer *healer, int child, uuid_t gfid)
+{
+	int ret = 0;
+	eh_t *eh = NULL;
+	afr_private_t *priv = NULL;
+	afr_self_heald_t *shd = NULL;
+	shd_event_t *shd_event = NULL;
+	char *path = NULL;
+	xlator_t *subvol = NULL;
+	xlator_t *this = NULL;
+	crawl_event_t *crawl_event = NULL;
+
+	this = healer->this;
+	priv = this->private;
+	shd = &priv->shd;
+	crawl_event = &healer->crawl_event;
+
+	subvol = priv->children[child];
+
+	ret = afr_selfheal (this, gfid);
+
+	if (ret == -EIO) {
+		eh = shd->split_brain;
+		crawl_event->split_brain_count++;
+	} else if (ret < 0) {
+		eh = shd->heal_failed;
+		crawl_event->heal_failed_count++;
+	} else if (ret == 0) {
+		eh = shd->healed;
+		crawl_event->healed_count++;
+	}
+
+	afr_shd_gfid_to_path (this, subvol, gfid, &path);
+	if (!path)
+		return ret;
+
+	if (eh) {
+		shd_event = GF_CALLOC (1, sizeof(*shd_event),
+				       gf_afr_mt_shd_event_t);
+		if (!shd_event) {
+			GF_FREE (path);
+			return ret;
+		}
+
+		shd_event->child = child;
+		shd_event->path = path;
+
+		if (eh_save_history (eh, shd_event) < 0) {
+			GF_FREE (shd_event);
+			GF_FREE (path);
+		}
+	}
+	return ret;
+}
+
+
+void
+afr_shd_sweep_prepare (struct subvol_healer *healer)
+{
+	crawl_event_t *event = NULL;
+
+	event = &healer->crawl_event;
+
+	event->healed_count = 0;
+	event->split_brain_count = 0;
+	event->heal_failed_count = 0;
+
+	time (&event->start_time);
+	event->end_time = 0;
+}
+
+
+void
+afr_shd_sweep_done (struct subvol_healer *healer)
+{
+	crawl_event_t *event = NULL;
+	crawl_event_t *history = NULL;
+	afr_self_heald_t *shd = NULL;
+
+	event = &healer->crawl_event;
+	shd = &(((afr_private_t *)healer->this->private)->shd);
+
+	time (&event->end_time);
+	history = memdup (event, sizeof (*event));
+	event->start_time = 0;
+
+	if (!history)
+		return;
+
+	if (eh_save_history (shd->statistics[healer->subvol], history) < 0)
+		GF_FREE (history);
+}
+
+
+int
+afr_shd_index_sweep (struct subvol_healer *healer)
+{
+	xlator_t *this = NULL;
+	int child = -1;
+	fd_t *fd = NULL;
+	xlator_t *subvol = NULL;
+	afr_private_t *priv = NULL;
+	off_t offset = 0;
+	gf_dirent_t entries;
+	gf_dirent_t *entry = NULL;
+	uuid_t gfid;
+	int ret = 0;
+	int count = 0;
+
+	this = healer->this;
+	child = healer->subvol;
+	priv = this->private;
+	subvol = priv->children[child];
+
+	fd = afr_shd_index_opendir (this, child);
+	if (!fd) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"unable to opendir index-dir on %s", subvol->name);
+		return -errno;
+	}
+
+	INIT_LIST_HEAD (&entries.list);
+
+	while ((ret = syncop_readdir (subvol, fd, 131072, offset, &entries))) {
+		if (ret > 0)
+			ret = 0;
+		list_for_each_entry (entry, &entries.list, list) {
+			offset = entry->d_off;
+
+			if (!priv->shd.enabled) {
+				ret = -EBUSY;
+				break;
+			}
+
+			if (!strcmp (entry->d_name, ".") ||
+			    !strcmp (entry->d_name, ".."))
+				continue;
+
+			gf_log (this->name, GF_LOG_DEBUG, "got entry: %s",
+				entry->d_name);
+
+			ret = uuid_parse (entry->d_name, gfid);
+			if (ret)
+				continue;
+
+			ret = afr_shd_selfheal (healer, child, gfid);
+			if (ret == 0)
+				count++;
+
+			if (ret == -ENOENT || ret == -ESTALE) {
+				afr_shd_index_purge (subvol, fd->inode,
+						     entry->d_name);
+				ret = 0;
+			}
+		}
+
+		gf_dirent_free (&entries);
+		if (ret)
+			break;
+	}
+
+	if (fd)
+		fd_unref (fd);
+	if (!ret)
+		ret = count;
+	return ret;
+}
+
+
+int
+afr_shd_full_sweep (struct subvol_healer *healer, inode_t *inode)
+{
+	fd_t *fd = NULL;
+	xlator_t *this = NULL;
+	xlator_t *subvol = NULL;
+	afr_private_t *priv = NULL;
+	off_t offset = 0;
+	gf_dirent_t entries;
+	gf_dirent_t *entry = NULL;
+	int ret = 0;
+
+	this = healer->this;
+	priv = this->private;
+	subvol = priv->children[healer->subvol];
+
+	fd = fd_anonymous (inode);
+	if (!fd)
+		return -errno;
+
+	INIT_LIST_HEAD (&entries.list);
+
+	while ((ret = syncop_readdirp (subvol, fd, 131072, offset, 0, &entries))) {
+		if (ret < 0)
+			break;
+
+		ret = gf_link_inodes_from_dirent (this, fd->inode, &entries);
+		if (ret)
+			break;
+
+		list_for_each_entry (entry, &entries.list, list) {
+			offset = entry->d_off;
+
+			if (!priv->shd.enabled) {
+				ret = -EBUSY;
+				break;
+			}
+
+			if (!strcmp (entry->d_name, ".") ||
+			    !strcmp (entry->d_name, ".."))
+				continue;
+
+			afr_shd_selfheal_name (healer, healer->subvol,
+					       inode->gfid, entry->d_name);
+
+			afr_shd_selfheal (healer, healer->subvol,
+					  entry->d_stat.ia_gfid);
+
+			if (entry->d_stat.ia_type == IA_IFDIR) {
+				ret = afr_shd_full_sweep (healer, entry->inode);
+				if (ret)
+					break;
+			}
+		}
+
+		gf_dirent_free (&entries);
+		if (ret)
+			break;
+	}
+
+	if (fd)
+		fd_unref (fd);
+	return ret;
+}
+
+
+void *
+afr_shd_index_healer (void *data)
+{
+	struct subvol_healer *healer = NULL;
+	xlator_t *this = NULL;
+	int ret = 0;
+
+	healer = data;
+	THIS = this = healer->this;
+
+	for (;;) {
+		afr_shd_healer_wait (healer);
+
+		ASSERT_LOCAL(this, healer);
+
+		do {
+			gf_log (this->name, GF_LOG_DEBUG,
+				"starting index sweep on subvol %s",
+				afr_subvol_name (this, healer->subvol));
+
+			afr_shd_sweep_prepare (healer);
+
+			ret = afr_shd_index_sweep (healer);
+
+			afr_shd_sweep_done (healer);
+			/*
+			  As long as at least one gfid was
+			  healed, keep retrying. We may have
+			  just healed a directory and thereby
+			  created entries for other gfids which
+			  could not be healed thus far.
+			*/
+
+			gf_log (this->name, GF_LOG_DEBUG,
+				"finished index sweep on subvol %s",
+				afr_subvol_name (this, healer->subvol));
+			/*
+			  Give a pause before retrying to avoid a busy loop
+			  in case the only entry in index is because of
+			  an ongoing I/O.
+			*/
+			sleep (1);
+		} while (ret > 0);
+	}
+
+	return NULL;
+}
+
+
+void *
+afr_shd_full_healer (void *data)
+{
+	struct subvol_healer *healer = NULL;
+	xlator_t *this = NULL;
+	int run = 0;
+
+	healer = data;
+	THIS = this = healer->this;
+
+	for (;;) {
+		pthread_mutex_lock (&healer->mutex);
+		{
+			run = __afr_shd_healer_wait (healer);
+			if (!run)
+				healer->running = _gf_false;
+		}
+		pthread_mutex_unlock (&healer->mutex);
+
+		if (!run)
+			break;
+
+		ASSERT_LOCAL(this, healer);
+
+		gf_log (this->name, GF_LOG_INFO,
+			"starting full sweep on subvol %s",
+			afr_subvol_name (this, healer->subvol));
+
+		afr_shd_sweep_prepare (healer);
+
+		afr_shd_full_sweep (healer, this->itable->root);
+
+		afr_shd_sweep_done (healer);
+
+		gf_log (this->name, GF_LOG_INFO,
+			"finished full sweep on subvol %s",
+			afr_subvol_name (this, healer->subvol));
+	}
+
+	return NULL;
+}
+
+
+int
+afr_shd_healer_init (xlator_t *this, struct subvol_healer *healer)
+{
+	int ret = 0;
+
+	ret = pthread_mutex_init (&healer->mutex, NULL);
+	if (ret)
+		goto out;
+
+	ret = pthread_cond_init (&healer->cond, NULL);
+	if (ret)
+		goto out;
+
+	healer->this = this;
+	healer->running = _gf_false;
+	healer->rerun = _gf_false;
+	healer->local = _gf_false;
+out:
+	return ret;
+}
+
+
+int
+afr_shd_healer_spawn (xlator_t *this, struct subvol_healer *healer,
+		      void *(threadfn)(void *))
+{
+	int ret = 0;
+
+	pthread_mutex_lock (&healer->mutex);
+	{
+		if (healer->running) {
+			pthread_cond_signal (&healer->cond);
+		} else {
+			ret = gf_thread_create (&healer->thread, NULL,
+						threadfn, healer);
+			if (ret)
+				goto unlock;
+			healer->running = 1;
+		}
+
+		healer->rerun = 1;
+	}
+unlock:
+	pthread_mutex_unlock (&healer->mutex);
+
+	return ret;
+}
+
+
+int
+afr_shd_full_healer_spawn (xlator_t *this, int subvol)
+{
+	return afr_shd_healer_spawn (this, NTH_FULL_HEALER (this, subvol),
+				     afr_shd_full_healer);
+}
+
+
+int
+afr_shd_index_healer_spawn (xlator_t *this, int subvol)
+{
+	return afr_shd_healer_spawn (this, NTH_INDEX_HEALER (this, subvol),
+				     afr_shd_index_healer);
+}
+
+
+int
+afr_shd_dict_add_crawl_event (xlator_t *this, dict_t *output,
+			      crawl_event_t *crawl_event)
+{
+        int             ret = 0;
+        uint64_t        count = 0;
+        char            key[256] = {0};
+        int             xl_id = 0;
+        uint64_t        healed_count = 0;
+        uint64_t        split_brain_count = 0;
+        uint64_t        heal_failed_count = 0;
+        char            *start_time_str = 0;
+        char            *end_time_str = NULL;
+        char            *crawl_type = NULL;
+        int             progress = -1;
+	int             child = -1;
+
+	child = crawl_event->child;
+        healed_count = crawl_event->healed_count;
+        split_brain_count = crawl_event->split_brain_count;
+        heal_failed_count = crawl_event->heal_failed_count;
+        crawl_type = crawl_event->crawl_type;
+
+	if (!crawl_event->start_time)
+		goto out;
+
+        start_time_str = gf_strdup (ctime (&crawl_event->start_time));
+
+	if (crawl_event->end_time)
+		end_time_str = gf_strdup (ctime (&crawl_event->end_time));
+
+        ret = dict_get_int32 (output, this->name, &xl_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "xl does not have id");
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "statistics-%d-%d-count", xl_id, child);
+        ret = dict_get_uint64 (output, key, &count);
+
+
+        snprintf (key, sizeof (key), "statistics_healed_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64(output, key, healed_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_healed_count to outout");
+                goto out;
+	}
+
+        snprintf (key, sizeof (key), "statistics_sb_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64 (output, key, split_brain_count);
+	if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_split_brain_count to outout");
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "statistics_crawl_type-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_str (output, key, crawl_type);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_crawl_type to output");
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "statistics_heal_failed_cnt-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_uint64 (output, key, heal_failed_count);
+	if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_healed_failed_count to outout");
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "statistics_strt_time-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        ret = dict_set_dynstr (output, key, start_time_str);
+	if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_crawl_start_time to outout");
+                goto out;
+        } else {
+		start_time_str = NULL;
+	}
+
+	if (!end_time_str)
+                progress = 1;
+        else
+                progress = 0;
+
+        snprintf (key, sizeof (key), "statistics_end_time-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+        if (!end_time_str)
+                end_time_str = gf_strdup ("Could not determine the end time");
+        ret = dict_set_dynstr (output, key, end_time_str);
+	if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_crawl_end_time to outout");
+                goto out;
+        } else {
+		end_time_str = NULL;
+	}
+
+        snprintf (key, sizeof (key), "statistics_inprogress-%d-%d-%"PRIu64,
+                  xl_id, child, count);
+
+        ret = dict_set_int32 (output, key, progress);
+	if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not add statistics_inprogress to outout");
+                goto out;
+        }
+
+	snprintf (key, sizeof (key), "statistics-%d-%d-count", xl_id, child);
+	ret = dict_set_uint64 (output, key, count + 1);
+	if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+			"Could not increment the counter.");
+                goto out;
+	}
+out:
+	GF_FREE (start_time_str);
+	GF_FREE (end_time_str);
+        return ret;
+}
+
+
+int
+afr_shd_dict_add_path (xlator_t *this, dict_t *output, int child, char *path,
+		       struct timeval *tv)
+{
+        int             ret = -1;
+        uint64_t        count = 0;
+        char            key[256] = {0};
+        int             xl_id = 0;
+
+        ret = dict_get_int32 (output, this->name, &xl_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "xl does not have id");
+                goto out;
+        }
+
+        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
+        ret = dict_get_uint64 (output, key, &count);
+
+        snprintf (key, sizeof (key), "%d-%d-%"PRIu64, xl_id, child, count);
+	ret = dict_set_dynstr (output, key, path);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s: Could not add to output",
+                        path);
+                goto out;
+        }
+
+	if (tv) {
+		snprintf (key, sizeof (key), "%d-%d-%"PRIu64"-time", xl_id,
+			  child, count);
+		ret = dict_set_uint32 (output, key, tv->tv_sec);
+		if (ret) {
+			gf_log (this->name, GF_LOG_ERROR, "%s: Could not set time",
+				path);
+			goto out;
+		}
+	}
+
+        snprintf (key, sizeof (key), "%d-%d-count", xl_id, child);
+
+        ret = dict_set_uint64 (output, key, count + 1);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Could not increment count");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+int
+afr_shd_gfid_to_path (xlator_t *this, xlator_t *subvol, uuid_t gfid, char **path_p)
+{
+	loc_t loc = {0,};
+	char *path = NULL;
+	dict_t *xattr = NULL;
+	int ret = 0;
+
+	uuid_copy (loc.gfid, gfid);
+	loc.inode = inode_new (this->itable);
+
+	ret = syncop_getxattr (subvol, &loc, &xattr, GFID_TO_PATH_KEY);
+	loc_wipe (&loc);
+	if (ret)
+		return ret;
+
+	ret = dict_get_str (xattr, GFID_TO_PATH_KEY, &path);
+	if (ret || !path)
+		return -EINVAL;
+
+	*path_p = gf_strdup (path);
+	if (!*path_p)
+		return -ENOMEM;
+	return 0;
+}
+
+
+int
+afr_shd_gather_index_entries (xlator_t *this, int child, dict_t *output)
+{
+	fd_t *fd = NULL;
+	xlator_t *subvol = NULL;
+	afr_private_t *priv = NULL;
+	off_t offset = 0;
+	gf_dirent_t entries;
+	gf_dirent_t *entry = NULL;
+	uuid_t gfid;
+	int ret = 0;
+	int count = 0;
+	char *path = NULL;
+
+	priv = this->private;
+	subvol = priv->children[child];
+
+	fd = afr_shd_index_opendir (this, child);
+	if (!fd) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"unable to opendir index-dir on %s", subvol->name);
+		return -errno;
+	}
+
+	INIT_LIST_HEAD (&entries.list);
+
+	while ((ret = syncop_readdir (subvol, fd, 131072, offset, &entries))) {
+		if (ret > 0)
+			ret = 0;
+		list_for_each_entry (entry, &entries.list, list) {
+			offset = entry->d_off;
+
+			if (!strcmp (entry->d_name, ".") ||
+			    !strcmp (entry->d_name, ".."))
+				continue;
+
+			gf_log (this->name, GF_LOG_DEBUG, "got entry: %s",
+				entry->d_name);
+
+			ret = uuid_parse (entry->d_name, gfid);
+			if (ret)
+				continue;
+
+			path = NULL;
+			ret = afr_shd_gfid_to_path (this, subvol, gfid, &path);
+
+			if (ret == -ENOENT || ret == -ESTALE) {
+				afr_shd_index_purge (subvol, fd->inode,
+						     entry->d_name);
+				ret = 0;
+				continue;
+			}
+
+			ret = afr_shd_dict_add_path (this, output, child, path,
+						     NULL);
+		}
+
+		gf_dirent_free (&entries);
+		if (ret)
+			break;
+	}
+
+	if (fd)
+		fd_unref (fd);
+	if (!ret)
+		ret = count;
+	return ret;
+}
+
+
+int
+afr_add_shd_event (circular_buffer_t *cb, void *data)
+{
+	dict_t *output = NULL;
+	xlator_t *this = THIS;
+	afr_private_t *priv = NULL;
+	afr_self_heald_t *shd = NULL;
+	shd_event_t *shd_event = NULL;
+	char *path = NULL;
+
+	output = data;
+	priv = this->private;
+	shd = &priv->shd;
+	shd_event = cb->data;
+
+	if (!shd->index_healers[shd_event->child].local)
+		return 0;
+
+	path = gf_strdup (shd_event->path);
+	if (!path)
+		return -ENOMEM;
+
+	afr_shd_dict_add_path (this, output, shd_event->child, path,
+			       &cb->tv);
+	return 0;
+}
+
+int
+afr_add_crawl_event (circular_buffer_t *cb, void *data)
+{
+	dict_t *output = NULL;
+	xlator_t *this = THIS;
+	afr_private_t *priv = NULL;
+	afr_self_heald_t *shd = NULL;
+	crawl_event_t *crawl_event = NULL;
+
+	output = data;
+	priv = this->private;
+	shd = &priv->shd;
+	crawl_event = cb->data;
+
+	if (!shd->index_healers[crawl_event->child].local)
+		return 0;
+
+	afr_shd_dict_add_crawl_event (this, output, crawl_event);
+
+	return 0;
+}
+
+
+int
+afr_selfheal_daemon_init (xlator_t *this)
+{
+	afr_private_t *priv = NULL;
+	afr_self_heald_t *shd = NULL;
+	int ret = -1;
+	int i = 0;
+
+	priv = this->private;
+	shd = &priv->shd;
+
+	this->itable = inode_table_new (SHD_INODE_LRU_LIMIT, this);
+	if (!this->itable)
+		goto out;
+
+	shd->index_healers = GF_CALLOC (sizeof(*shd->index_healers),
+					priv->child_count,
+					gf_afr_mt_subvol_healer_t);
+	if (!shd->index_healers)
+		goto out;
+
+	for (i = 0; i < priv->child_count; i++) {
+		shd->index_healers[i].subvol = i;
+		ret = afr_shd_healer_init (this, &shd->index_healers[i]);
+		if (ret)
+			goto out;
+	}
+
+	shd->full_healers = GF_CALLOC (sizeof(*shd->full_healers),
+				       priv->child_count,
+				       gf_afr_mt_subvol_healer_t);
+	if (!shd->full_healers)
+		goto out;
+	for (i = 0; i < priv->child_count; i++) {
+		shd->full_healers[i].subvol = i;
+		ret = afr_shd_healer_init (this, &shd->full_healers[i]);
+		if (ret)
+			goto out;
+	}
+
+	shd->healed = eh_new (AFR_EH_HEALED_LIMIT, _gf_false,
+			      afr_destroy_shd_event_data);
+        if (!shd->healed)
+		goto out;
+
+	shd->heal_failed = eh_new (AFR_EH_HEAL_FAIL_LIMIT, _gf_false,
+				   afr_destroy_shd_event_data);
+	if (!shd->heal_failed)
+		goto out;
+
+	shd->split_brain = eh_new (AFR_EH_SPLIT_BRAIN_LIMIT, _gf_false,
+				   afr_destroy_shd_event_data);
+	if (!shd->split_brain)
+		goto out;
+
+        shd->statistics = GF_CALLOC (sizeof(eh_t *), priv->child_count,
+				     gf_common_mt_eh_t);
+        if (!shd->statistics)
+                goto out;
+
+        for (i = 0; i < priv->child_count ; i++) {
+                shd->statistics[i] = eh_new (AFR_STATISTICS_HISTORY_SIZE,
+					     _gf_false,
+					     afr_destroy_crawl_event_data);
+                if (!shd->statistics[i])
+                        goto out;
+		shd->full_healers[i].crawl_event.child = i;
+		shd->full_healers[i].crawl_event.crawl_type = "FULL";
+		shd->index_healers[i].crawl_event.child = i;
+		shd->index_healers[i].crawl_event.crawl_type = "INDEX";
+        }
+
+	ret = 0;
+out:
+	return ret;
+}
+
+
+int
+afr_selfheal_childup (xlator_t *this, int subvol)
+{
+	afr_shd_index_healer_spawn (this, subvol);
+
+	return 0;
+}
+
+
+int64_t
+afr_shd_get_index_count (xlator_t *this, int i)
+{
+	afr_private_t *priv = NULL;
+	xlator_t *subvol = NULL;
+	uint64_t count = 0;
+	loc_t rootloc = {0, };
+	dict_t *xattr = NULL;
+	int ret = -1;
+
+	priv = this->private;
+	subvol = priv->children[i];
+
+	rootloc.inode = inode_ref (this->itable->root);
+	uuid_copy (rootloc.gfid, rootloc.inode->gfid);
+
+	ret = syncop_getxattr (subvol, &rootloc, &xattr,
+			       GF_XATTROP_INDEX_COUNT);
+	loc_wipe (&rootloc);
+
+	if (ret < 0)
+		return -1;
+
+	ret = dict_get_uint64 (xattr, GF_XATTROP_INDEX_COUNT, &count);
+	if (ret)
+		return -1;
+	return count;
+}
+
+
+int
+afr_xl_op (xlator_t *this, dict_t *input, dict_t *output)
+{
+        gf_xl_afr_op_t   op = GF_AFR_OP_INVALID;
+        int              ret = 0;
+        int              xl_id = 0;
+	afr_private_t   *priv = NULL;
+	afr_self_heald_t *shd = NULL;
+	struct subvol_healer *healer = NULL;
+	int i = 0;
+	char key[64];
+	int op_ret = 0;
+	int64_t cnt = 0;
+
+	priv = this->private;
+	shd = &priv->shd;
+
+	for (i = 0; i < priv->child_count; i++)
+		if (priv->child_up[i] == -1)
+			goto out;
+
+        ret = dict_get_int32 (input, "xl-op", (int32_t*)&op);
+        if (ret)
+                goto out;
+        ret = dict_get_int32 (input, this->name, &xl_id);
+        if (ret)
+                goto out;
+        ret = dict_set_int32 (output, this->name, xl_id);
+        if (ret)
+                goto out;
+        switch (op) {
+        case GF_AFR_OP_HEAL_INDEX:
+		op_ret = -1;
+
+		for (i = 0; i < priv->child_count; i++) {
+			healer = &shd->index_healers[i];
+			snprintf (key, 64, "%d-%d-status", xl_id, i);
+
+			if (!priv->child_up[i]) {
+				ret = dict_set_str (output, key,
+						    "Brick is not connected");
+			} else if (AFR_COUNT (priv->child_up,
+					      priv->child_count) < 2) {
+				ret = dict_set_str (output, key,
+						    "< 2 bricks in replica are up");
+			} else if (!afr_shd_is_subvol_local (this, healer->subvol)) {
+				ret = dict_set_str (output, key,
+						    "Brick is remote");
+			} else {
+				ret = dict_set_str (output, key,
+						    "Started self-heal");
+				afr_shd_index_healer_spawn (this, i);
+				op_ret = 0;
+			}
+		}
+                break;
+        case GF_AFR_OP_HEAL_FULL:
+		op_ret = -1;
+
+		for (i = 0; i < priv->child_count; i++) {
+			healer = &shd->full_healers[i];
+			snprintf (key, 64, "%d-%d-status", xl_id, i);
+
+			if (!priv->child_up[i]) {
+				ret = dict_set_str (output, key,
+						    "Brick is not connected");
+			} else if (AFR_COUNT (priv->child_up,
+					      priv->child_count) < 2) {
+				ret = dict_set_str (output, key,
+						    "< 2 bricks in replica are up");
+			} else if (!afr_shd_is_subvol_local (this, healer->subvol)) {
+				ret = dict_set_str (output, key,
+						    "Brick is remote");
+			} else {
+				ret = dict_set_str (output, key,
+						    "Started self-heal");
+				afr_shd_full_healer_spawn (this, i);
+				op_ret = 0;
+			}
+		}
+                break;
+        case GF_AFR_OP_INDEX_SUMMARY:
+		for (i = 0; i < priv->child_count; i++)
+			if (shd->index_healers[i].local)
+				afr_shd_gather_index_entries (this, i, output);
+                break;
+        case GF_AFR_OP_HEALED_FILES:
+		eh_dump (shd->healed, output, afr_add_shd_event);
+                break;
+        case GF_AFR_OP_HEAL_FAILED_FILES:
+		eh_dump (shd->heal_failed, output, afr_add_shd_event);
+                break;
+        case GF_AFR_OP_SPLIT_BRAIN_FILES:
+		eh_dump (shd->split_brain, output, afr_add_shd_event);
+                break;
+        case GF_AFR_OP_STATISTICS:
+		for (i = 0; i < priv->child_count; i++) {
+			eh_dump (shd->statistics[i], output,
+				 afr_add_crawl_event);
+			afr_shd_dict_add_crawl_event (this, output,
+						      &shd->index_healers[i].crawl_event);
+			afr_shd_dict_add_crawl_event (this, output,
+						      &shd->full_healers[i].crawl_event);
+		}
+                break;
+        case GF_AFR_OP_STATISTICS_HEAL_COUNT:
+        case GF_AFR_OP_STATISTICS_HEAL_COUNT_PER_REPLICA:
+		op_ret = -1;
+
+		for (i = 0; i < priv->child_count; i++) {
+			if (!priv->child_up[i]) {
+				snprintf (key, 64, "%d-%d-status", xl_id, i);
+				ret = dict_set_str (output, key,
+						    "Brick is not connected");
+			} else {
+				snprintf (key, 64, "%d-%d-hardlinks", xl_id, i);
+				cnt = afr_shd_get_index_count (this, i);
+				if (cnt >= 0) {
+					ret = dict_set_uint64 (output, key, cnt);
+				}
+				op_ret = 0;
+			}
+		}
+
+//                ret = _do_crawl_op_on_local_subvols (this, INDEX_TO_BE_HEALED,
+//                                                     STATISTICS_TO_BE_HEALED,
+//                                                     output);
+                break;
+
+        default:
+                gf_log (this->name, GF_LOG_ERROR, "Unknown set op %d", op);
+                break;
+        }
+out:
+        dict_del (output, this->name);
+        return op_ret;
+}
diff --git a/xlators/cluster/afr/src/afr-self-heald.h b/xlators/cluster/afr/src/afr-self-heald.h
new file mode 100644
index 000000000..10e229ee7
--- /dev/null
+++ b/xlators/cluster/afr/src/afr-self-heald.h
@@ -0,0 +1,72 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _AFR_SELF_HEALD_H
+#define _AFR_SELF_HEALD_H
+
+#include <pthread.h>
+
+
+typedef struct {
+	int child;
+	char *path;
+} shd_event_t;
+
+typedef struct {
+	int      child;
+	uint64_t healed_count;
+        uint64_t split_brain_count;
+        uint64_t heal_failed_count;
+
+	/* If start_time is 0, it means crawler is not in progress
+	   and stats are not valid */
+	time_t   start_time;
+	/* If start_time is NOT 0 and end_time is 0, it means
+	   cralwer is in progress */
+        time_t   end_time;
+        char     *crawl_type;
+} crawl_event_t;
+
+struct subvol_healer {
+	xlator_t        *this;
+	int              subvol;
+	gf_boolean_t     local;
+	gf_boolean_t     running;
+	gf_boolean_t     rerun;
+	crawl_event_t    crawl_event;
+	pthread_mutex_t  mutex;
+	pthread_cond_t   cond;
+	pthread_t        thread;
+};
+
+typedef struct {
+	gf_boolean_t            iamshd;
+	gf_boolean_t            enabled;
+	struct subvol_healer   *index_healers;
+	struct subvol_healer   *full_healers;
+
+	eh_t                    *healed;
+        eh_t                    *heal_failed;
+        eh_t                    *split_brain;
+        eh_t                    **statistics;
+} afr_self_heald_t;
+
+
+int
+afr_selfheal_childup (xlator_t *this, int subvol);
+
+int
+afr_selfheal_daemon_init (xlator_t *this);
+
+int
+afr_xl_op (xlator_t *this, dict_t *input, dict_t *output);
+
+#endif /* !_AFR_SELF_HEALD_H */
diff --git a/xlators/cluster/afr/src/afr-transaction.c b/xlators/cluster/afr/src/afr-transaction.c
index 130f5bd27..205ff759e 100644
--- a/xlators/cluster/afr/src/afr-transaction.c
+++ b/xlators/cluster/afr/src/afr-transaction.c
@@ -1,265 +1,219 @@
 /*
-  Copyright (c) 2007-2009 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include "dict.h"
 #include "byte-order.h"
 #include "common-utils.h"
+#include "timer.h"
 
 #include "afr.h"
 #include "afr-transaction.h"
 
 #include <signal.h>
 
+gf_boolean_t
+afr_changelog_pre_op_uninherit (call_frame_t *frame, xlator_t *this);
 
-#define LOCKED_NO       0x0        /* no lock held */
-#define LOCKED_YES      0x1        /* for DATA, METADATA, ENTRY and higher_path
-                                      of RENAME */
-#define LOCKED_LOWER    0x2        /* for lower_path of RENAME */
-
-
-afr_fd_ctx_t *
-afr_fd_ctx_get (fd_t *fd, xlator_t *this)
-{
-        uint64_t       ctx = 0;
-        afr_fd_ctx_t  *fd_ctx = NULL;
-        int            ret = 0;
-
-        ret = fd_ctx_get (fd, this, &ctx);
+gf_boolean_t
+afr_changelog_pre_op_update (call_frame_t *frame, xlator_t *this);
 
-        if (ret < 0)
-                goto out;
-
-        fd_ctx = (afr_fd_ctx_t *)(long) ctx;
-
-out:
-        return fd_ctx;
-}
+int
+afr_changelog_do (call_frame_t *frame, xlator_t *this, dict_t *xattr,
+		  afr_changelog_resume_t changelog_resume);
 
 
-static void
-afr_pid_save (call_frame_t *frame)
+int
+__afr_txn_write_fop (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t * local = NULL;
+        afr_local_t *local = NULL;
+        afr_private_t *priv = NULL;
+        int call_count = -1;
+        int i = 0;
 
         local = frame->local;
+        priv = this->private;
 
-        local->saved_pid = frame->root->pid;
-}
+        call_count = AFR_COUNT (local->transaction.pre_op, priv->child_count);
 
+        if (call_count == 0) {
+                local->transaction.resume (frame, this);
+                return 0;
+        }
 
-static void
-afr_pid_restore (call_frame_t *frame)
-{
-        afr_local_t * local = NULL;
+        local->call_count = call_count;
 
-        local = frame->local;
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.pre_op[i]) {
+			local->transaction.wind (frame, this, i);
+
+                        if (!--call_count)
+                                break;
+                }
+        }
 
-        frame->root->pid = local->saved_pid;
+        return 0;
 }
 
 
-static void
-__mark_all_pending (int32_t *pending[], int child_count,
-                    afr_transaction_type type)
+int
+__afr_txn_write_done (call_frame_t *frame, xlator_t *this)
 {
-	int i;
-        int j;
-
-        for (i = 0; i < child_count; i++) {
-                j = afr_index_for_transaction_type (type);
-		pending[i][j] = hton32 (1);
-        }
-}
+        afr_local_t *local = NULL;
 
+        local = frame->local;
 
-static void
-__mark_child_dead (int32_t *pending[], int child_count, int child,
-                   afr_transaction_type type)
-{
-        int j;
+        local->transaction.unwind (frame, this);
 
-        j = afr_index_for_transaction_type (type);
+        AFR_STACK_DESTROY (frame);
 
-	pending[child][j] = 0;
+        return 0;
 }
 
 
-static void
-__mark_pre_op_done_on_fd (call_frame_t *frame, xlator_t *this, int child_index)
+call_frame_t*
+afr_transaction_detach_fop_frame (call_frame_t *frame)
 {
-        afr_local_t   *local = NULL;
-        afr_fd_ctx_t  *fd_ctx = NULL;
+        afr_local_t *   local = NULL;
+        call_frame_t   *fop_frame = NULL;
 
         local = frame->local;
 
-        if (!local->fd)
-                return;
-
-        fd_ctx = afr_fd_ctx_get (local->fd, this);
-
-        if (!fd_ctx)
-                goto out;
-
-        LOCK (&local->fd->lock);
+        LOCK (&frame->lock);
         {
-                if (local->transaction.type == AFR_DATA_TRANSACTION)
-                        fd_ctx->pre_op_done[child_index]++;
+                fop_frame = local->transaction.main_frame;
+                local->transaction.main_frame = NULL;
         }
-        UNLOCK (&local->fd->lock);
-out:
-        return;
+        UNLOCK (&frame->lock);
+
+        return fop_frame;
 }
 
 
 static void
-__mark_pre_op_undone_on_fd (call_frame_t *frame, xlator_t *this, int child_index)
+afr_save_lk_owner (call_frame_t *frame)
 {
-        afr_local_t   *local = NULL;
-        afr_fd_ctx_t  *fd_ctx = NULL;
+        afr_local_t * local = NULL;
 
         local = frame->local;
 
-        if (!local->fd)
-                return;
-
-        fd_ctx = afr_fd_ctx_get (local->fd, this);
-
-        if (!fd_ctx)
-                goto out;
-
-        LOCK (&local->fd->lock);
-        {
-                if (local->transaction.type == AFR_DATA_TRANSACTION)
-                        fd_ctx->pre_op_done[child_index]--;
-        }
-        UNLOCK (&local->fd->lock);
-out:
-        return;
+        local->saved_lk_owner = frame->root->lk_owner;
 }
 
 
 static void
-__mark_down_children (int32_t *pending[], int child_count,
-                      unsigned char *child_up, afr_transaction_type type)
+afr_restore_lk_owner (call_frame_t *frame)
 {
-	int i;
-	int j;
+        afr_local_t * local = NULL;
 
-	for (i = 0; i < child_count; i++) {
-                j = afr_index_for_transaction_type (type);
+        local = frame->local;
 
-		if (!child_up[i])
-			pending[i][j] = 0;
-        }
+        frame->root->lk_owner = local->saved_lk_owner;
 }
 
-
-static void
-__mark_all_success (int32_t *pending[], int child_count,
-                    afr_transaction_type type)
+void
+__mark_all_success (call_frame_t *frame, xlator_t *this)
 {
+	afr_private_t *priv = NULL;
+	afr_local_t *local = NULL;
 	int i;
-        int j;
 
-	for (i = 0; i < child_count; i++) {
-                j = afr_index_for_transaction_type (type);
-		pending[i][j] = hton32 (-1);
-        }
+	local = frame->local;
+	priv = this->private;
+
+	for (i = 0; i < priv->child_count; i++) {
+		local->transaction.failed_subvols[i] = 0;
+	}
 }
 
 
-static int
-__changelog_enabled (afr_private_t *priv, afr_transaction_type type)
+int
+afr_transaction_perform_fop (call_frame_t *frame, xlator_t *this)
 {
-	int ret = 0;
+        afr_local_t     *local = NULL;
+        fd_t            *fd   = NULL;
 
-	switch (type) {
-	case AFR_DATA_TRANSACTION:
-		if (priv->data_change_log)
-			ret = 1;
-
-		break;
-
-	case AFR_METADATA_TRANSACTION:
-		if (priv->metadata_change_log)
-			ret = 1;
-
-		break;
-
-	case AFR_ENTRY_TRANSACTION:
-	case AFR_ENTRY_RENAME_TRANSACTION:
-		if (priv->entry_change_log)
-			ret = 1;
-
-		break;
-	}
+        local = frame->local;
+        fd    = local->fd;
+
+        /*  Perform fops with the lk-owner from top xlator.
+         *  Eg: lk-owner of posix-lk and flush should be same,
+         *  flush cant clear the  posix-lks without that lk-owner.
+         */
+        afr_save_lk_owner (frame);
+        frame->root->lk_owner =
+                local->transaction.main_frame->root->lk_owner;
+
+	if (local->pre_op_compat)
+		/* old mode, pre-op was done as afr_changelog_do()
+		   just now, before OP */
+		afr_changelog_pre_op_update (frame, this);
+
+        /* The wake up needs to happen independent of
+           what type of fop arrives here. If it was
+           a write, then it has already inherited the
+           lock and changelog. If it was not a write,
+           then the presumption of the optimization (of
+           optimizing for successive write operations)
+           fails.
+        */
+        if (fd)
+                afr_delayed_changelog_wake_up (this, fd);
+        local->transaction.fop (frame, this);
 
-	return ret;
+	return 0;
 }
 
 
 static int
-__changelog_needed_pre_op (call_frame_t *frame, xlator_t *this)
+__changelog_enabled (afr_private_t *priv, afr_transaction_type type)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
+        int ret = 0;
 
-	int op_ret   = 0;
+        switch (type) {
+        case AFR_DATA_TRANSACTION:
+                if (priv->data_change_log)
+                        ret = 1;
 
-	priv  = this->private;
-	local = frame->local;
+                break;
 
-	if (__changelog_enabled (priv, local->transaction.type)) {
-		switch (local->op) {
+        case AFR_METADATA_TRANSACTION:
+                if (priv->metadata_change_log)
+                        ret = 1;
 
-		case GF_FOP_WRITE:
-		case GF_FOP_FTRUNCATE:
-                        op_ret = 1;
-			break;
+                break;
 
-		case GF_FOP_FLUSH:
-			op_ret = 0;
-			break;
+        case AFR_ENTRY_TRANSACTION:
+        case AFR_ENTRY_RENAME_TRANSACTION:
+                if (priv->entry_change_log)
+                        ret = 1;
 
-		default:
-			op_ret = 1;
-		}
-	}
+                break;
+        }
 
-	return op_ret;
+        return ret;
 }
 
 
 static int
-__changelog_needed_post_op (call_frame_t *frame, xlator_t *this)
+__fop_changelog_needed (call_frame_t *frame, xlator_t *this)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-
-	int op_ret = 0;
-	afr_transaction_type type = -1;
+        afr_private_t * priv  = NULL;
+        afr_local_t   * local = NULL;
+        int op_ret = 0;
+        afr_transaction_type type = -1;
 
-	priv  = this->private;
-	local = frame->local;
-	type  = local->transaction.type;
+        priv  = this->private;
+        local = frame->local;
+        type  = local->transaction.type;
 
-	if (__changelog_enabled (priv, type)) {
+        if (__changelog_enabled (priv, type)) {
                 switch (local->op) {
 
                 case GF_FOP_WRITE:
@@ -276,617 +230,706 @@ __changelog_needed_post_op (call_frame_t *frame, xlator_t *this)
                 }
         }
 
-	return op_ret;
+        return op_ret;
 }
 
 
-static int
-afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending)
+int
+afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int **pending)
 {
-        int i;
+        int i = 0;
         int ret = 0;
+	int pending_zero[AFR_NUM_CHANGE_LOGS] = {0, };
 
         for (i = 0; i < priv->child_count; i++) {
+		if (!memcmp (pending_zero, pending[i], sizeof (pending_zero)))
+			/* don't set xattrs for non-pending servers */
+			continue;
+
                 ret = dict_set_static_bin (xattr, priv->pending_key[i],
-                                           pending[i], 3 * sizeof (int32_t));
+					   pending[i],
+					   AFR_NUM_CHANGE_LOGS * sizeof (int));
                 /* 3 = data+metadata+entry */
 
-                if (ret < 0)
-                        goto out;
+                if (ret)
+                        break;
         }
 
-out:
         return ret;
 }
 
-
-static int
-afr_set_piggyback_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending,
-                        afr_transaction_type type)
+int
+afr_lock_server_count (afr_private_t *priv, afr_transaction_type type)
 {
-        int i;
         int ret = 0;
-        int *arr = NULL;
-        int index = 0;
-
-        index = afr_index_for_transaction_type (type);
-
-        for (i = 0; i < priv->child_count; i++) {
-                arr = GF_CALLOC (3 * sizeof (int32_t), priv->child_count,
-                                 gf_afr_mt_char);
-                if (!arr) {
-                        ret = -1;
-                        goto out;
-                }
-
-                memcpy (arr, pending[i], 3 * sizeof (int32_t));
 
-                arr[index]++;
+        switch (type) {
+        case AFR_DATA_TRANSACTION:
+                ret = priv->child_count;
+                break;
 
-                ret = dict_set_bin (xattr, priv->pending_key[i],
-                                    arr, 3 * sizeof (int32_t));
-                /* 3 = data+metadata+entry */
+        case AFR_METADATA_TRANSACTION:
+                ret = priv->child_count;
+                break;
 
-                if (ret < 0)
-                        goto out;
+        case AFR_ENTRY_TRANSACTION:
+        case AFR_ENTRY_RENAME_TRANSACTION:
+                ret = priv->child_count;
+                break;
         }
 
-out:
         return ret;
 }
 
+/* {{{ pending */
+
 
 int
-afr_lock_server_count (afr_private_t *priv, afr_transaction_type type)
+afr_changelog_post_op_done (call_frame_t *frame, xlator_t *this)
 {
-	int ret = 0;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+        afr_internal_lock_t *int_lock = NULL;
 
-	switch (type) {
-	case AFR_DATA_TRANSACTION:
-		ret = priv->child_count;
-		break;
+	local = frame->local;
+	priv = this->private;
+        int_lock = &local->internal_lock;
 
-	case AFR_METADATA_TRANSACTION:
-		ret = priv->child_count;
-		break;
+	if (local->transaction.resume_stub) {
+		call_resume (local->transaction.resume_stub);
+		local->transaction.resume_stub = NULL;
+	}
 
-	case AFR_ENTRY_TRANSACTION:
-	case AFR_ENTRY_RENAME_TRANSACTION:
-		ret = priv->child_count;
-		break;
+	if (afr_lock_server_count (priv, local->transaction.type) == 0) {
+		local->transaction.done (frame, this);
+	} else {
+		int_lock->lock_cbk = local->transaction.done;
+		afr_unlock (frame, this);
 	}
 
-	return ret;
+	return 0;
 }
 
-/* {{{ pending */
 
-int32_t
-afr_changelog_post_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int32_t op_ret, int32_t op_errno, dict_t *xattr)
+afr_inodelk_t*
+afr_get_inodelk (afr_internal_lock_t *int_lock, char *dom)
 {
-        afr_internal_lock_t *int_lock = NULL;
-	afr_private_t       *priv     = NULL;
-	afr_local_t         *local    = NULL;
-        int                  child_index = 0;
-
-	int call_count = -1;
-
-	priv     = this->private;
-	local    = frame->local;
-        int_lock = &local->internal_lock;
-
-        child_index = (long) cookie;
+        afr_inodelk_t *inodelk = NULL;
+        int           i = 0;
 
-        if (op_ret == 1) {
+        for (i = 0; int_lock->inodelk[i].domain; i++) {
+                inodelk = &int_lock->inodelk[i];
+                if (strcmp (dom, inodelk->domain) == 0)
+                        return inodelk;
         }
+        return NULL;
+}
 
-        if (op_ret == 0) {
-                __mark_pre_op_undone_on_fd (frame, this, child_index);
+unsigned char*
+afr_locked_nodes_get (afr_transaction_type type, afr_internal_lock_t *int_lock)
+{
+        unsigned char *locked_nodes = NULL;
+        afr_inodelk_t *inodelk = NULL;
+        switch (type) {
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
+                inodelk = afr_get_inodelk (int_lock, int_lock->domain);
+                locked_nodes = inodelk->locked_nodes;
+        break;
+
+        case AFR_ENTRY_TRANSACTION:
+        case AFR_ENTRY_RENAME_TRANSACTION:
+                /*Because same set of subvols participate in all lockee
+                 * entities*/
+                locked_nodes = int_lock->lockee[0].locked_nodes;
+        break;
         }
+        return locked_nodes;
+}
 
-	LOCK (&frame->lock);
-	{
-		call_count = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
 
-	if (call_count == 0) {
-                if (afr_lock_server_count (priv, local->transaction.type) == 0) {
-                        local->transaction.done (frame, this);
-                } else {
-                        int_lock->lock_cbk = local->transaction.done;
-                        afr_unlock (frame, this);
-                }
-	}
+int
+afr_changelog_call_count (afr_transaction_type type,
+			  unsigned char *pre_op_subvols,
+			  unsigned int child_count)
+{
+        int call_count = 0;
 
-	return 0;
+	call_count = AFR_COUNT(pre_op_subvols, child_count);
+
+        if (type == AFR_ENTRY_RENAME_TRANSACTION)
+                call_count *= 2;
+
+        return call_count;
 }
 
 
-void
-afr_update_read_child (call_frame_t *frame, xlator_t *this, inode_t *inode,
-                       afr_transaction_type type)
+gf_boolean_t
+afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this)
 {
-        int           curr_read_child = -1;
-        int           new_read_child = -1;
-        afr_private_t   *priv = NULL;
-        afr_local_t  *local = NULL;
-        int         **pending = NULL;
-        int           idx = 0;
-
-        idx = afr_index_for_transaction_type (type);
+        afr_private_t *priv = NULL;
+        afr_local_t *local = NULL;
+        int i = 0;
 
-        priv = this->private;
         local = frame->local;
-        curr_read_child = afr_read_child (this, inode);
-        pending = local->pending;
+	priv = this->private;
 
-        if (pending[curr_read_child][idx] != 0)
-                return;
+        for (i = 0; i < priv->child_count; i++) {
+                if (local->transaction.failed_subvols[i])
+                        return _gf_false;
+        }
 
-        /* need to set new read_child */
-        for (new_read_child = 0; new_read_child < priv->child_count;
-             new_read_child++) {
+        return _gf_true;
+}
 
-                if (!priv->child_up[new_read_child])
-                        /* child is down */
-                        continue;
 
-                if (pending[new_read_child][idx] == 0)
-                        /* op just failed */
-                        continue;
+void
+afr_handle_symmetric_errors (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int op_errno = 0;
+	int i_errno = 0;
+	gf_boolean_t matching_errors = _gf_true;
+	int i = 0;
 
-                break;
-        }
+	priv = this->private;
+	local = frame->local;
 
-        if (new_read_child == priv->child_count)
-                /* all children uneligible. leave as-is */
-                return;
+	for (i = 0; i < priv->child_count; i++) {
+		if (!local->replies[i].valid)
+			continue;
+		if (local->replies[i].op_ret != -1) {
+			/* Operation succeeded on at least on subvol,
+			   so it is not a failed-everywhere situation.
+			*/
+			matching_errors = _gf_false;
+			break;
+		}
+		i_errno = local->replies[i].op_errno;
+
+		if (i_errno == ENOTCONN) {
+			/* ENOTCONN is not a symmetric error. We do not
+			   know if the operation was performed on the
+			   backend or not.
+			*/
+			matching_errors = _gf_false;
+			break;
+		}
+
+		if (!op_errno) {
+			op_errno = i_errno;
+		} else if (op_errno != i_errno) {
+			/* Mismatching op_errno's */
+			matching_errors = _gf_false;
+			break;
+		}
+	}
 
-        afr_set_read_child (this, inode, new_read_child);
+	if (matching_errors)
+		__mark_all_success (frame, this);
 }
 
 
-int 
-afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
+int
+afr_changelog_post_op_now (call_frame_t *frame, xlator_t *this)
 {
-	afr_private_t * priv = this->private;
-        afr_internal_lock_t *int_lock = NULL;
-	int ret        = 0;
-	int i          = 0;
-	int call_count = 0;
-
-	afr_local_t *  local = NULL;
-        afr_fd_ctx_t  *fdctx = NULL;
-	dict_t        **xattr = NULL;
-        int            piggyback = 0;
-        int            index = 0;
+        afr_private_t * priv = this->private;
+        int i          = 0;
+	int ret = 0;
+	int idx = 0;
+        afr_local_t *  local = NULL;
+        dict_t        *xattr = NULL;
         int            nothing_failed = 1;
+	gf_boolean_t   need_undirty = _gf_false;
 
-	local    = frame->local;
-        int_lock = &local->internal_lock;
-
-	__mark_down_children (local->pending, priv->child_count,
-                              local->child_up, local->transaction.type);
+        local = frame->local;
+	idx = afr_index_for_transaction_type (local->transaction.type);
 
-        if (local->fd)
-                afr_update_read_child (frame, this, local->fd->inode,
-                                       local->transaction.type);
+        nothing_failed = afr_txn_nothing_failed (frame, this);
 
-        xattr = alloca (priv->child_count * sizeof (*xattr));
-        memset (xattr, 0, (priv->child_count * sizeof (*xattr)));
-	for (i = 0; i < priv->child_count; i++) {
-                xattr[i] = get_new_dict ();
-                dict_ref (xattr[i]);
-        }
+	if (afr_changelog_pre_op_uninherit (frame, this))
+		need_undirty = _gf_false;
+	else
+		need_undirty = _gf_true;
 
-        call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-        if (local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
-                call_count *= 2;
-        }
+	if (nothing_failed && !need_undirty) {
+		afr_changelog_post_op_done (frame, this);
+                goto out;
+	}
 
-	local->call_count = call_count;
+	xattr = dict_new ();
+	if (!xattr) {
+		local->op_ret = -1;
+		local->op_errno = ENOMEM;
+		afr_changelog_post_op_done (frame, this);
+		goto out;
+	}
 
-        if (local->fd)
-                fdctx = afr_fd_ctx_get (local->fd, this);
+	if (need_undirty) {
+		local->dirty[idx] = hton32(-1);
 
-	if (call_count == 0) {
-		/* no child is up */
-                for (i = 0; i < priv->child_count; i++) {
-                        dict_unref (xattr[i]);
-                }
+		ret = dict_set_static_bin (xattr, AFR_DIRTY, local->dirty,
+					   sizeof(int) * AFR_NUM_CHANGE_LOGS);
+		if (ret) {
+			local->op_ret = -1;
+			local->op_errno = ENOMEM;
+			afr_changelog_post_op_done (frame, this);
+			goto out;
+		}
 
-                int_lock->lock_cbk = local->transaction.done;
-		afr_unlock (frame, this);
-		return 0;
 	}
 
-        /* check if something has failed, to handle piggybacking */
-        nothing_failed = 1;
-        index = afr_index_for_transaction_type (local->transaction.type);
-        for (i = 0; i < priv->child_count; i++) {
-                if (local->pending[i][index] == 0) {
-                        nothing_failed = 0;
-                        break;
-                }
-        }
+	if (!nothing_failed) {
+		for (i = 0; i < priv->child_count; i++) {
+			if (local->transaction.failed_subvols[i])
+				local->pending[i][idx] = hton32(1);
+		}
+		ret = afr_set_pending_dict (priv, xattr, local->pending);
+		if (ret < 0) {
+			local->op_ret = -1;
+			local->op_errno = ENOMEM;
+			afr_changelog_post_op_done (frame, this);
+			goto out;
+		}
 
-	for (i = 0; i < priv->child_count; i++) {
-		if (!local->child_up[i])
-                        continue;
+	}
 
-                ret = afr_set_pending_dict (priv, xattr[i],
-                                            local->pending);
+	afr_changelog_do (frame, this, xattr, afr_changelog_post_op_done);
+out:
+	if (xattr)
+                dict_unref (xattr);
 
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to set pending entry");
+        return 0;
+}
 
 
-                switch (local->transaction.type) {
-                case AFR_DATA_TRANSACTION:
-                {
-                        if (!fdctx) {
-                                STACK_WIND (frame, afr_changelog_post_op_cbk,
-                                            priv->children[i],
-                                            priv->children[i]->fops->xattrop,
-                                            &local->loc,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i]);
-                                break;
-                        }
+gf_boolean_t
+afr_changelog_pre_op_uninherit (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	fd_t *fd = NULL;
+	int i = 0;
+	gf_boolean_t ret = _gf_false;
+	afr_fd_ctx_t *fd_ctx = NULL;
+	int type = 0;
 
-                        LOCK (&local->fd->lock);
-                        {
-                                piggyback = 0;
-                                if (fdctx->pre_op_piggyback[i]) {
-                                        fdctx->pre_op_piggyback[i]--;
-                                        piggyback = 1;
-                                }
-                        }
-                        UNLOCK (&local->fd->lock);
+	local = frame->local;
+	priv = this->private;
+	fd = local->fd;
 
-                        if (piggyback && !nothing_failed)
-                                ret = afr_set_piggyback_dict (priv, xattr[i],
-                                                              local->pending,
-                                                              local->transaction.type);
+	type = afr_index_for_transaction_type (local->transaction.type);
+	if (type != AFR_DATA_TRANSACTION)
+		return !local->transaction.dirtied;
 
-                        if (nothing_failed && piggyback) {
-                                afr_changelog_post_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i]);
-                        } else {
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_post_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fxattrop,
-                                                   local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
-                        }
-                }
-                break;
-                case AFR_METADATA_TRANSACTION:
-                {
-                        if (local->fd)
-                                STACK_WIND (frame, afr_changelog_post_op_cbk,
-                                            priv->children[i],
-                                            priv->children[i]->fops->fxattrop,
-                                            local->fd,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i]);
-                        else
-                                STACK_WIND (frame, afr_changelog_post_op_cbk,
-                                            priv->children[i],
-                                            priv->children[i]->fops->xattrop,
-                                            &local->loc,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i]);
-                }
-                break;
+	if (!fd)
+		return !local->transaction.dirtied;
 
-                case AFR_ENTRY_RENAME_TRANSACTION:
-                {
-                        STACK_WIND_COOKIE (frame, afr_changelog_post_op_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->xattrop,
-                                           &local->transaction.new_parent_loc,
-                                           GF_XATTROP_ADD_ARRAY, xattr[i]);
-                        call_count--;
-                }
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+		return _gf_false;
 
-                /*
-                  set it again because previous stack_wind
-                  might have already returned (think of case
-                  where subvolume is posix) and would have
-                  used the dict as placeholder for return
-                  value
-                */
+	if (local->transaction.no_uninherit)
+		return _gf_false;
 
-                ret = afr_set_pending_dict (priv, xattr[i],
-                                            local->pending);
+	/* This function must be idempotent. So check if we
+	   were called before and return the same answer again.
 
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to set pending entry");
+	   It is important to keep this function idempotent for
+	   the call in afr_changelog_post_op_safe() to not have
+	   side effects on the call from afr_changelog_post_op_now()
+	*/
+	if (local->transaction.uninherit_done)
+		return local->transaction.uninherit_value;
 
-                /* fall through */
+	LOCK(&fd->lock);
+	{
+		for (i = 0; i < priv->child_count; i++) {
+			if (local->transaction.pre_op[i] !=
+			    fd_ctx->pre_op_done[type][i]) {
+				ret = !local->transaction.dirtied;
+				goto unlock;
+			}
+		}
 
-                case AFR_ENTRY_TRANSACTION:
-                {
-                        if (local->fd)
-                                STACK_WIND (frame, afr_changelog_post_op_cbk,
-                                            priv->children[i],
-                                            priv->children[i]->fops->fxattrop,
-                                            local->fd,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i]);
-                        else
-                                STACK_WIND (frame, afr_changelog_post_op_cbk,
-                                            priv->children[i],
-                                            priv->children[i]->fops->xattrop,
-                                            &local->transaction.parent_loc,
-                                            GF_XATTROP_ADD_ARRAY, xattr[i]);
-                }
-                break;
-                }
+		if (fd_ctx->inherited[type]) {
+			ret = _gf_true;
+			fd_ctx->inherited[type]--;
+		} else if (fd_ctx->on_disk[type]) {
+			ret = _gf_false;
+			fd_ctx->on_disk[type]--;
+		} else {
+			/* ASSERT */
+			ret = _gf_false;
+		}
 
-                if (!--call_count)
-                        break;
+		if (!fd_ctx->inherited[type] && !fd_ctx->on_disk[type]) {
+			for (i = 0; i < priv->child_count; i++)
+				fd_ctx->pre_op_done[type][i] = 0;
+		}
 	}
+unlock:
+	UNLOCK(&fd->lock);
 
-        for (i = 0; i < priv->child_count; i++) {
-                dict_unref (xattr[i]);
-        }
+	local->transaction.uninherit_done = _gf_true;
+	local->transaction.uninherit_value = ret;
 
-	return 0;
+	return ret;
 }
 
 
-int32_t
-afr_changelog_pre_op_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *xattr)
+gf_boolean_t
+afr_changelog_pre_op_inherit (call_frame_t *frame, xlator_t *this)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = this->private;
-	loc_t       *   loc   = NULL;
-
-	int call_count  = -1;
-	int child_index = (long) cookie;
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	fd_t *fd = NULL;
+	int i = 0;
+	gf_boolean_t ret = _gf_false;
+	afr_fd_ctx_t *fd_ctx = NULL;
+	int type = 0;
 
 	local = frame->local;
-	loc   = &local->loc;
+	priv = this->private;
+	fd = local->fd;
 
-	LOCK (&frame->lock);
-	{
-                if (op_ret == 1) {
-                        /* special op_ret for piggyback */
-                }
+	if (local->transaction.type != AFR_DATA_TRANSACTION)
+		return _gf_false;
 
-                if (op_ret == 0) {
-                        __mark_pre_op_done_on_fd (frame, this, child_index);
-                }
+	type = afr_index_for_transaction_type (local->transaction.type);
+
+	if (!fd)
+		return _gf_false;
 
-		if (op_ret == -1) {
-			local->child_up[child_index] = 0;
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+		return _gf_false;
 
-			if (op_errno == ENOTSUP) {
-				gf_log (this->name, GF_LOG_ERROR,
-					"xattrop not supported by %s",
-					priv->children[child_index]->name);
-				local->op_ret = -1;
+	LOCK(&fd->lock);
+	{
+		if (!fd_ctx->on_disk[type]) {
+			/* nothing to inherit yet */
+			ret = _gf_false;
+			goto unlock;
+		}
 
-			} else if (!child_went_down (op_ret, op_errno)) {
-				gf_log (this->name, GF_LOG_ERROR,
-					"xattrop failed on child %s: %s",
-					priv->children[child_index]->name,
-					strerror (op_errno));
+		for (i = 0; i < priv->child_count; i++) {
+			if (local->transaction.pre_op[i] !=
+			    fd_ctx->pre_op_done[type][i]) {
+				/* either inherit exactly, or don't */
+				ret = _gf_false;
+				goto unlock;
 			}
-			local->op_errno = op_errno;
 		}
 
-		call_count = --local->call_count;
+		fd_ctx->inherited[type]++;
+
+		ret = _gf_true;
+
+		local->transaction.inherited = _gf_true;
 	}
-	UNLOCK (&frame->lock);
+unlock:
+	UNLOCK(&fd->lock);
 
-	if (call_count == 0) {
-		if ((local->op_ret == -1) &&
-		    (local->op_errno == ENOTSUP)) {
-			local->transaction.resume (frame, this);
-		} else {
-                        __mark_all_success (local->pending, priv->child_count,
-                                            local->transaction.type);
+	return ret;
+}
 
-                        afr_pid_restore (frame);
 
-			local->transaction.fop (frame, this);
+gf_boolean_t
+afr_changelog_pre_op_update (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	fd_t *fd = NULL;
+	afr_fd_ctx_t *fd_ctx = NULL;
+	int i = 0;
+	gf_boolean_t ret = _gf_false;
+	int type = 0;
+
+	local = frame->local;
+	priv = this->private;
+	fd = local->fd;
+
+	if (!fd)
+		return _gf_false;
+
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+		return _gf_false;
+
+	if (local->transaction.inherited)
+		/* was already inherited in afr_changelog_pre_op */
+		return _gf_false;
+
+	if (!local->transaction.dirtied)
+		return _gf_false;
+
+        if (!afr_txn_nothing_failed (frame, this))
+		return _gf_false;
+
+	type = afr_index_for_transaction_type (local->transaction.type);
+
+	ret = _gf_false;
+
+	LOCK(&fd->lock);
+	{
+		if (!fd_ctx->on_disk[type]) {
+			for (i = 0; i < priv->child_count; i++)
+				fd_ctx->pre_op_done[type][i] =
+					local->transaction.pre_op[i];
+		} else {
+			for (i = 0; i < priv->child_count; i++)
+				if (fd_ctx->pre_op_done[type][i] !=
+				    local->transaction.pre_op[i]) {
+					local->transaction.no_uninherit = 1;
+					goto unlock;
+				}
 		}
+		fd_ctx->on_disk[type]++;
+
+		ret = _gf_true;
 	}
+unlock:
+	UNLOCK(&fd->lock);
 
-	return 0;
+	return ret;
 }
 
 
 int
-afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
+afr_changelog_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+		   int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-	afr_private_t * priv = this->private;
-	int i = 0;
-	int ret = 0;
-	int call_count = 0;
-	dict_t **xattr = NULL;
-        afr_fd_ctx_t *fdctx = NULL;
+        afr_local_t *local = NULL;
+        int call_count = -1;
 
-	afr_local_t *local = NULL;
-        int          piggyback = 0;
+        local = frame->local;
 
-	local = frame->local;
+	if (op_ret == -1)
+		afr_transaction_fop_failed (frame, this, (long) cookie);
 
-        xattr = alloca (priv->child_count * sizeof (*xattr));
-        memset (xattr, 0, (priv->child_count * sizeof (*xattr)));
+	call_count = afr_frame_return (frame);
 
-	for (i = 0; i < priv->child_count; i++) {
-                xattr[i] = get_new_dict ();
-                dict_ref (xattr[i]);
-        }
+        if (call_count == 0)
+		local->transaction.changelog_resume (frame, this);
 
-	call_count = afr_up_children_count (priv->child_count,
-					    local->child_up);
+        return 0;
+}
 
-	if (local->transaction.type == AFR_ENTRY_RENAME_TRANSACTION) {
-		call_count *= 2;
-	}
 
-	if (call_count == 0) {
-		/* no child is up */
-                for (i = 0; i < priv->child_count; i++) {
-                        dict_unref (xattr[i]);
-                }
+int
+afr_changelog_do (call_frame_t *frame, xlator_t *this, dict_t *xattr,
+		  afr_changelog_resume_t changelog_resume)
+{
+	afr_local_t *local = NULL;
+	afr_private_t *priv = NULL;
+	int i = 0;
+	int call_count = 0;
 
-                local->internal_lock.lock_cbk =
-                        local->transaction.done;
-		afr_unlock (frame, this);
+	local = frame->local;
+	priv = this->private;
+
+        call_count = afr_changelog_call_count (local->transaction.type,
+					       local->transaction.pre_op,
+					       priv->child_count);
+
+	if (call_count == 0) {
+		changelog_resume (frame, this);
 		return 0;
 	}
 
 	local->call_count = call_count;
 
-	__mark_all_pending (local->pending, priv->child_count,
-                            local->transaction.type);
-
-        if (local->fd)
-                fdctx = afr_fd_ctx_get (local->fd, this);
+	local->transaction.changelog_resume = changelog_resume;
 
-	for (i = 0; i < priv->child_count; i++) {
-                if (!local->child_up[i])
+        for (i = 0; i < priv->child_count; i++) {
+                if (!local->transaction.pre_op[i])
                         continue;
-                ret = afr_set_pending_dict (priv, xattr[i],
-                                            local->pending);
-
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to set pending entry");
-
 
                 switch (local->transaction.type) {
                 case AFR_DATA_TRANSACTION:
-                {
-                        if (!fdctx) {
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->xattrop,
-                                                   &(local->loc),
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
-                                break;
+                case AFR_METADATA_TRANSACTION:
+                        if (!local->fd) {
+                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
+						   (void *) (long) i,
+						   priv->children[i],
+						   priv->children[i]->fops->xattrop,
+						   &local->loc,
+						   GF_XATTROP_ADD_ARRAY, xattr,
+						   NULL);
+                        } else {
+                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
+						   (void *) (long) i,
+						   priv->children[i],
+						   priv->children[i]->fops->fxattrop,
+						   local->fd,
+						   GF_XATTROP_ADD_ARRAY, xattr,
+						   NULL);
                         }
+			break;
+                case AFR_ENTRY_RENAME_TRANSACTION:
 
-                        LOCK (&local->fd->lock);
-                        {
-                                piggyback = 0;
-                                if (fdctx->pre_op_done[i]) {
-                                        fdctx->pre_op_piggyback[i]++;
-                                        piggyback = 1;
-                                        fdctx->hit++;
-                                } else {
-                                        fdctx->miss++;
-                                }
-                        }
-                        UNLOCK (&local->fd->lock);
+			STACK_WIND_COOKIE (frame, afr_changelog_cbk,
+					   (void *) (long) i,
+					   priv->children[i],
+					   priv->children[i]->fops->xattrop,
+					   &local->transaction.new_parent_loc,
+					   GF_XATTROP_ADD_ARRAY, xattr,
+					   NULL);
+                        call_count--;
 
-                        if (piggyback)
-                                afr_changelog_pre_op_cbk (frame, (void *)(long)i,
-                                                          this, 1, 0, xattr[i]);
-                        else
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fxattrop,
-                                                   local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
-                }
-                break;
-                case AFR_METADATA_TRANSACTION:
-                {
+                /* fall through */
+
+                case AFR_ENTRY_TRANSACTION:
                         if (local->fd)
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fxattrop,
-                                                   local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
+                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
+						   (void *) (long) i,
+						   priv->children[i],
+						   priv->children[i]->fops->fxattrop,
+						   local->fd,
+						   GF_XATTROP_ADD_ARRAY, xattr,
+						   NULL);
                         else
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->xattrop,
-                                                   &(local->loc),
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
-                }
-                break;
+                                STACK_WIND_COOKIE (frame, afr_changelog_cbk,
+						   (void *) (long) i,
+						   priv->children[i],
+						   priv->children[i]->fops->xattrop,
+						   &local->transaction.parent_loc,
+						   GF_XATTROP_ADD_ARRAY, xattr,
+						   NULL);
+			break;
+		}
 
-                case AFR_ENTRY_RENAME_TRANSACTION:
-                {
-                        STACK_WIND_COOKIE (frame,
-                                           afr_changelog_pre_op_cbk,
-                                           (void *) (long) i,
-                                           priv->children[i],
-                                           priv->children[i]->fops->xattrop,
-                                           &local->transaction.new_parent_loc,
-                                           GF_XATTROP_ADD_ARRAY, xattr[i]);
+                if (!--call_count)
+                        break;
+        }
 
-                        call_count--;
-                }
+	return 0;
+}
 
 
-                /*
-                  set it again because previous stack_wind
-                  might have already returned (think of case
-                  where subvolume is posix) and would have
-                  used the dict as placeholder for return
-                  value
-                */
+int
+afr_changelog_pre_op (call_frame_t *frame, xlator_t *this)
+{
+        afr_private_t * priv = this->private;
+        int i = 0;
+        int ret = 0;
+        int call_count = 0;
+	int op_errno = 0;
+        afr_local_t *local = NULL;
+        afr_internal_lock_t *int_lock = NULL;
+        unsigned char       *locked_nodes = NULL;
+	unsigned char       *pending_subvols = NULL;
+	int idx = -1;
+	gf_boolean_t pre_nop = _gf_true;
+	dict_t *xdata_req = NULL;
 
-                ret = afr_set_pending_dict (priv, xattr[i],
-                                            local->pending);
+        local = frame->local;
+        int_lock = &local->internal_lock;
+	idx = afr_index_for_transaction_type (local->transaction.type);
 
-                if (ret < 0)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "failed to set pending entry");
+        locked_nodes = afr_locked_nodes_get (local->transaction.type, int_lock);
 
-                /* fall through */
+	pending_subvols = alloca0 (priv->child_count);
 
-                case AFR_ENTRY_TRANSACTION:
-                {
-                        if (local->fd)
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->fxattrop,
-                                                   local->fd,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
-                        else
-                                STACK_WIND_COOKIE (frame,
-                                                   afr_changelog_pre_op_cbk,
-                                                   (void *) (long) i,
-                                                   priv->children[i],
-                                                   priv->children[i]->fops->xattrop,
-                                                   &local->transaction.parent_loc,
-                                                   GF_XATTROP_ADD_ARRAY, xattr[i]);
-                }
-                break;
-                }
+	for (i = 0; i < priv->child_count; i++) {
+		if (locked_nodes[i]) {
+			local->transaction.pre_op[i] = 1;
+			call_count++;
+		} else {
+			pending_subvols[i] = 1;
+		}
+	}
 
-                if (!--call_count)
-                        break;
+	/* TBD: quorum check w/ call_count */
+
+        if (call_count == 0) {
+		op_errno = ENOTCONN;
+		goto err;
 	}
 
-        for (i = 0; i < priv->child_count; i++) {
-                dict_unref (xattr[i]);
-        }
+	xdata_req = dict_new();
+	if (!xdata_req) {
+		op_errno = ENOMEM;
+		goto err;
+	}
+
+	pre_nop = _gf_true;
+
+	if (afr_changelog_pre_op_inherit (frame, this))
+		goto next;
+
+	if (call_count < priv->child_count) {
+		/* For subvols we are not performing operation on,
+		   mark them as pending up-front along with the FOP
+		   so that we can safely defer unmarking dirty until
+		   later.
+		*/
+		for (i = 0; i < priv->child_count; i++) {
+			if (pending_subvols[i])
+				local->pending[i][idx] = hton32(1);
+		}
+		ret = afr_set_pending_dict (priv, xdata_req,
+					    local->pending);
+		if (ret < 0) {
+			op_errno = ENOMEM;
+			goto err;
+		}
+		pre_nop = _gf_false;
+	}
+
+	if (call_count > 1 &&
+	    (local->transaction.type == AFR_DATA_TRANSACTION ||
+	     !local->optimistic_change_log)) {
+
+		/* If we are performing change on only one subvol, no
+		   need to mark dirty, because we are setting the pending
+		   counts already anyways
+		*/
+		local->dirty[idx] = hton32(1);
+
+		ret = dict_set_static_bin (xdata_req, AFR_DIRTY, local->dirty,
+					   sizeof(int) * AFR_NUM_CHANGE_LOGS);
+		if (ret) {
+			op_errno = ENOMEM;
+			goto err;
+		}
+
+		pre_nop = _gf_false;
+		local->transaction.dirtied = 1;
+	}
+
+	if (pre_nop)
+		goto next;
+
+	if (!local->pre_op_compat) {
+		dict_copy (xdata_req, local->xdata_req);
+		goto next;
+	}
+
+	afr_changelog_do (frame, this, xdata_req, afr_transaction_perform_fop);
+
+	if (xdata_req)
+		dict_unref (xdata_req);
+
+	return 0;
+next:
+	afr_transaction_perform_fop (frame, this);
+
+	if (xdata_req)
+		dict_unref (xdata_req);
+
+        return 0;
+err:
+	local->internal_lock.lock_cbk = local->transaction.done;
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+
+	afr_unlock (frame, this);
+
+	if (xdata_req)
+		dict_unref (xdata_req);
 
 	return 0;
 }
@@ -902,9 +945,9 @@ afr_post_blocking_inodelk_cbk (call_frame_t *frame, xlator_t *this)
         int_lock = &local->internal_lock;
 
         if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_INFO,
                         "Blocking inodelks failed.");
-		local->transaction.done (frame, this);
+                local->transaction.done (frame, this);
         } else {
 
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -952,9 +995,9 @@ afr_post_blocking_entrylk_cbk (call_frame_t *frame, xlator_t *this)
         int_lock = &local->internal_lock;
 
         if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_INFO,
                         "Blocking entrylks failed.");
-		local->transaction.done (frame, this);
+                local->transaction.done (frame, this);
         } else {
 
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -1002,9 +1045,9 @@ afr_post_blocking_rename_cbk (call_frame_t *frame, xlator_t *this)
         int_lock = &local->internal_lock;
 
         if (int_lock->lock_op_ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_INFO,
                         "Blocking entrylks failed.");
-		local->transaction.done (frame, this);
+                local->transaction.done (frame, this);
         } else {
 
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -1037,12 +1080,14 @@ int
 afr_set_transaction_flock (afr_local_t *local)
 {
         afr_internal_lock_t *int_lock = NULL;
+        afr_inodelk_t       *inodelk  = NULL;
 
         int_lock = &local->internal_lock;
+        inodelk = afr_get_inodelk (int_lock, int_lock->domain);
 
-        int_lock->lk_flock.l_len   = local->transaction.len;
-        int_lock->lk_flock.l_start = local->transaction.start;
-        int_lock->lk_flock.l_type  = F_WRLCK;
+        inodelk->flock.l_len   = local->transaction.len;
+        inodelk->flock.l_start = local->transaction.start;
+        inodelk->flock.l_type  = F_WRLCK;
 
         return 0;
 }
@@ -1057,24 +1102,25 @@ afr_lock_rec (call_frame_t *frame, xlator_t *this)
         int_lock = &local->internal_lock;
 
         int_lock->transaction_lk_type = AFR_TRANSACTION_LK;
+        int_lock->domain = this->name;
 
-	switch (local->transaction.type) {
-	case AFR_DATA_TRANSACTION:
-	case AFR_METADATA_TRANSACTION:
+        switch (local->transaction.type) {
+        case AFR_DATA_TRANSACTION:
+        case AFR_METADATA_TRANSACTION:
                 afr_set_transaction_flock (local);
 
                 int_lock->lock_cbk = afr_post_nonblocking_inodelk_cbk;
 
                 afr_nonblocking_inodelk (frame, this);
-		break;
+                break;
 
         case AFR_ENTRY_RENAME_TRANSACTION:
 
-                int_lock->lock_cbk = afr_post_blocking_rename_cbk;
-                afr_blocking_lock (frame, this);
+                int_lock->lock_cbk = afr_post_nonblocking_entrylk_cbk;
+                afr_nonblocking_entrylk (frame, this);
                 break;
 
-	case AFR_ENTRY_TRANSACTION:
+        case AFR_ENTRY_TRANSACTION:
                 int_lock->lk_basename = local->transaction.basename;
                 if (&local->transaction.parent_loc)
                         int_lock->lk_loc = &local->transaction.parent_loc;
@@ -1093,15 +1139,9 @@ afr_lock_rec (call_frame_t *frame, xlator_t *this)
 int
 afr_lock (call_frame_t *frame, xlator_t *this)
 {
-        afr_pid_save (frame);
-
-        frame->root->pid = (long) frame->root;
-
-        afr_set_lk_owner (frame, this);
-
         afr_set_lock_number (frame, this);
 
-	return afr_lock_rec (frame, this);
+        return afr_lock_rec (frame, this);
 }
 
 
@@ -1110,90 +1150,573 @@ afr_lock (call_frame_t *frame, xlator_t *this)
 int
 afr_internal_lock_finish (call_frame_t *frame, xlator_t *this)
 {
-        afr_local_t   *local = NULL;
-        afr_private_t *priv  = NULL;
+        if (__fop_changelog_needed (frame, this)) {
+                afr_changelog_pre_op (frame, this);
+        } else {
+                afr_transaction_perform_fop (frame, this);
+        }
+
+        return 0;
+}
+
+
+void
+afr_set_delayed_post_op (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t    *local = NULL;
+        afr_private_t  *priv = NULL;
+
+        /* call this function from any of the related optimizations
+           which benefit from delaying post op are enabled, namely:
+
+           - changelog piggybacking
+           - eager locking
+        */
+
+        priv = this->private;
+        if (!priv)
+                return;
+
+        if (!priv->post_op_delay_secs)
+                return;
 
-        priv  = this->private;
         local = frame->local;
+        if (!local->transaction.eager_lock_on)
+                return;
 
-        if (__changelog_needed_pre_op (frame, this)) {
-                afr_changelog_pre_op (frame, this);
-        } else {
-                __mark_all_success (local->pending, priv->child_count,
-                                    local->transaction.type);
+        if (!local)
+                return;
 
-                afr_pid_restore (frame);
+        if (!local->fd)
+                return;
+
+        if (local->op == GF_FOP_WRITE)
+                local->delayed_post_op = _gf_true;
+}
 
-                local->transaction.fop (frame, this);
+gf_boolean_t
+afr_are_multiple_fds_opened (fd_t *fd, xlator_t *this)
+{
+        afr_fd_ctx_t *fd_ctx = NULL;
+
+        if (!fd) {
+                /* If false is returned, it may keep on taking eager-lock
+                 * which may lead to starvation, so return true to avoid that.
+                 */
+                gf_log_callingfn (this->name, GF_LOG_ERROR, "Invalid fd");
+                return _gf_true;
         }
+        /* Lets say mount1 has eager-lock(full-lock) and after the eager-lock
+         * is taken mount2 opened the same file, it won't be able to
+         * perform any data operations until mount1 releases eager-lock.
+         * To avoid such scenario do not enable eager-lock for this transaction
+         * if open-fd-count is > 1
+         */
+
+        fd_ctx = afr_fd_ctx_get (fd, this);
+        if (!fd_ctx)
+                return _gf_true;
+
+        if (fd_ctx->open_fd_count > 1)
+                return _gf_true;
+
+        return _gf_false;
+}
+
+
+gf_boolean_t
+is_afr_delayed_changelog_post_op_needed (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t      *local = NULL;
+        gf_boolean_t      res = _gf_false;
+
+        local = frame->local;
+        if (!local)
+                goto out;
+
+        if (!local->delayed_post_op)
+                goto out;
+
+        //Mark pending changelog ASAP
+        if (!afr_txn_nothing_failed (frame, this))
+                goto out;
+
+        if (local->fd && afr_are_multiple_fds_opened (local->fd, this))
+                goto out;
+
+        res = _gf_true;
+out:
+        return res;
+}
+
+
+void
+afr_delayed_changelog_post_op (xlator_t *this, call_frame_t *frame, fd_t *fd,
+                               call_stub_t *stub);
+
+void
+afr_delayed_changelog_wake_up_cbk (void *data)
+{
+        fd_t           *fd = NULL;
+
+        fd = data;
+
+        afr_delayed_changelog_wake_up (THIS, fd);
+}
+
+
+/* SET operation */
+int
+afr_fd_report_unstable_write (xlator_t *this, fd_t *fd)
+{
+        afr_fd_ctx_t *fdctx = NULL;
+
+        fdctx = afr_fd_ctx_get (fd, this);
+
+        LOCK(&fd->lock);
+        {
+                fdctx->witnessed_unstable_write = _gf_true;
+        }
+        UNLOCK(&fd->lock);
 
         return 0;
 }
 
+/* TEST and CLEAR operation */
+gf_boolean_t
+afr_fd_has_witnessed_unstable_write (xlator_t *this, fd_t *fd)
+{
+        afr_fd_ctx_t *fdctx = NULL;
+        gf_boolean_t witness = _gf_false;
+
+        fdctx = afr_fd_ctx_get (fd, this);
+        if (!fdctx)
+                return _gf_true;
+
+        LOCK(&fd->lock);
+        {
+                if (fdctx->witnessed_unstable_write) {
+                        witness = _gf_true;
+                        fdctx->witnessed_unstable_write = _gf_false;
+                }
+        }
+        UNLOCK (&fd->lock);
+
+        return witness;
+}
+
 
 int
-afr_transaction_resume (call_frame_t *frame, xlator_t *this)
+afr_changelog_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno, struct iatt *pre,
+                         struct iatt *post, dict_t *xdata)
 {
-        afr_internal_lock_t *int_lock = NULL;
-	afr_local_t         *local    = NULL;
-	afr_private_t       *priv     = NULL;
+        afr_private_t *priv = NULL;
+        int child_index = (long) cookie;
+        int call_count = -1;
+        afr_local_t *local = NULL;
 
-	local    = frame->local;
-        int_lock = &local->internal_lock;
-	priv     = this->private;
+        priv = this->private;
+        local = frame->local;
 
-	if (__changelog_needed_post_op (frame, this)) {
-		afr_changelog_post_op (frame, this);
-	} else {
-		if (afr_lock_server_count (priv, local->transaction.type) == 0) {
-			local->transaction.done (frame, this);
-		} else {
-                        int_lock->lock_cbk = local->transaction.done;
-			afr_unlock (frame, this);
-		}
+        if (op_ret != 0) {
+                /* Failure of fsync() is as good as failure of previous
+                   write(). So treat it like one.
+		*/
+                gf_log (this->name, GF_LOG_WARNING,
+                        "fsync(%s) failed on subvolume %s. Transaction was %s",
+                        uuid_utoa (local->fd->inode->gfid),
+                        priv->children[child_index]->name,
+                        gf_fop_list[local->op]);
+
+                afr_transaction_fop_failed (frame, this, child_index);
 	}
 
+	call_count = afr_frame_return (frame);
+
+	if (call_count == 0)
+		afr_changelog_post_op_now (frame, this);
+
 	return 0;
 }
 
 
+int
+afr_changelog_fsync (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t *local = NULL;
+        int i = 0;
+        int call_count = 0;
+        afr_private_t *priv = NULL;
+        dict_t *xdata = NULL;
+        GF_UNUSED int ret = -1;
+
+        local = frame->local;
+        priv = this->private;
+
+        call_count = AFR_COUNT (local->transaction.pre_op, priv->child_count);
+
+        if (!call_count) {
+                /* will go straight to unlock */
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        local->call_count = call_count;
+
+        xdata = dict_new();
+        if (xdata)
+                ret = dict_set_int32 (xdata, "batch-fsync", 1);
+
+        for (i = 0; i < priv->child_count; i++) {
+                if (!local->transaction.pre_op[i])
+                        continue;
+
+                STACK_WIND_COOKIE (frame, afr_changelog_fsync_cbk,
+				   (void *) (long) i, priv->children[i],
+				   priv->children[i]->fops->fsync, local->fd,
+				   1, xdata);
+                if (!--call_count)
+                        break;
+        }
+
+        if (xdata)
+                dict_unref (xdata);
+
+        return 0;
+}
+
+
+int
+afr_changelog_post_op_safe (call_frame_t *frame, xlator_t *this)
+{
+	afr_local_t    *local = NULL;
+        afr_private_t  *priv = NULL;
+
+	local = frame->local;
+        priv = this->private;
+
+        if (!local->fd || local->transaction.type != AFR_DATA_TRANSACTION) {
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        if (afr_changelog_pre_op_uninherit (frame, this) &&
+	    afr_txn_nothing_failed (frame, this)) {
+                /* just detected that this post-op is about to
+                   be optimized away as a new write() has
+                   already piggybacked on this frame's changelog.
+                   */
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        /* Calling afr_changelog_post_op_now() now will result in
+           issuing ->[f]xattrop().
+
+           Performing a hard POST-OP (->[f]xattrop() FOP) is a more
+           responsible operation that what it might appear on the surface.
+
+           The changelog of a file (in the xattr of the file on the server)
+           stores information (pending count) about the state of the file
+           on the OTHER server. This changelog is blindly trusted, and must
+           therefore be updated in such a way it remains trustworthy. This
+           implies that decrementing the pending count (essentially "clearing
+           the dirty flag") must be done STRICTLY after we are sure that the
+           operation on the other server has reached stable storage.
+
+           While the backend filesystem on that server will eventually flush
+           it to stable storage, we (being in userspace) have no mechanism
+           to get notified when the write became "stable".
+
+           This means we need take matter into our own hands and issue an
+           fsync() EVEN IF THE APPLICATION WAS PERFORMING UNSTABLE WRITES,
+           and get an acknowledgement for it. And we need to wait for the
+           fsync() acknowledgement before initiating the hard POST-OP.
+
+           However if the FD itself was opened in O_SYNC or O_DSYNC then
+           we are already guaranteed that the writes were made stable as
+           part of the FOP itself. The same holds true for NFS stable
+           writes which happen on an anonymous FD with O_DSYNC or O_SYNC
+           flag set in the writev() @flags param. For all other write types,
+           mark a flag in the fdctx whenever an unstable write is witnessed.
+           */
+
+        if (!afr_fd_has_witnessed_unstable_write (this, local->fd)) {
+                afr_changelog_post_op_now (frame, this);
+                return 0;
+        }
+
+        /* Check whether users want durability and perform fsync/post-op
+         * accordingly.
+         */
+        if (priv->ensure_durability) {
+                /* Time to fsync() */
+                afr_changelog_fsync (frame, this);
+        } else {
+                afr_changelog_post_op_now (frame, this);
+        }
+
+        return 0;
+}
+
+
+void
+afr_delayed_changelog_post_op (xlator_t *this, call_frame_t *frame, fd_t *fd,
+                               call_stub_t *stub)
+{
+	afr_fd_ctx_t      *fd_ctx = NULL;
+	call_frame_t      *prev_frame = NULL;
+	struct timespec    delta = {0, };
+	afr_private_t     *priv = NULL;
+	afr_local_t       *local = NULL;
+
+	priv = this->private;
+
+	fd_ctx = afr_fd_ctx_get (fd, this);
+	if (!fd_ctx)
+                goto out;
+
+	delta.tv_sec = priv->post_op_delay_secs;
+	delta.tv_nsec = 0;
+
+	pthread_mutex_lock (&fd_ctx->delay_lock);
+	{
+		prev_frame = fd_ctx->delay_frame;
+		fd_ctx->delay_frame = NULL;
+		if (fd_ctx->delay_timer)
+			gf_timer_call_cancel (this->ctx, fd_ctx->delay_timer);
+		fd_ctx->delay_timer = NULL;
+		if (!frame)
+			goto unlock;
+		fd_ctx->delay_timer = gf_timer_call_after (this->ctx, delta,
+							   afr_delayed_changelog_wake_up_cbk,
+							   fd);
+		fd_ctx->delay_frame = frame;
+	}
+unlock:
+	pthread_mutex_unlock (&fd_ctx->delay_lock);
+
+out:
+	if (prev_frame) {
+		local = prev_frame->local;
+		local->transaction.resume_stub = stub;
+		afr_changelog_post_op_now (prev_frame, this);
+	} else if (stub) {
+		call_resume (stub);
+	}
+}
+
+
+void
+afr_changelog_post_op (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t  *local = NULL;
+
+        local = frame->local;
+
+        if (is_afr_delayed_changelog_post_op_needed (frame, this))
+                afr_delayed_changelog_post_op (this, frame, local->fd, NULL);
+        else
+                afr_changelog_post_op_safe (frame, this);
+}
+
+
+
+/* Wake up the sleeping/delayed post-op, and also register
+   a stub to have it resumed after this transaction
+   completely finishes.
+
+   The @stub gets saved in @local and gets resumed in
+   afr_local_cleanup()
+   */
+void
+afr_delayed_changelog_wake_resume (xlator_t *this, fd_t *fd, call_stub_t *stub)
+{
+        afr_delayed_changelog_post_op (this, NULL, fd, stub);
+}
+
+
+void
+afr_delayed_changelog_wake_up (xlator_t *this, fd_t *fd)
+{
+        afr_delayed_changelog_post_op (this, NULL, fd, NULL);
+}
+
+
+int
+afr_transaction_resume (call_frame_t *frame, xlator_t *this)
+{
+        afr_local_t         *local    = NULL;
+
+        local    = frame->local;
+
+        if (local->transaction.eager_lock_on) {
+                /* We don't need to retain "local" in the
+                   fd list anymore, writes to all subvols
+                   are finished by now */
+                afr_remove_eager_lock_stub (local);
+        }
+
+        afr_restore_lk_owner (frame);
+
+        afr_handle_symmetric_errors (frame, this);
+
+	if (!local->pre_op_compat)
+		/* new mode, pre-op was done along
+		   with OP */
+		afr_changelog_pre_op_update (frame, this);
+
+        if (__fop_changelog_needed (frame, this)) {
+                afr_changelog_post_op (frame, this);
+        } else {
+		afr_changelog_post_op_done (frame, this);
+        }
+
+        return 0;
+}
+
+
 /**
  * afr_transaction_fop_failed - inform that an fop failed
  */
 
 void
-afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this, int child_index)
+afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
+                            int child_index)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
+        afr_local_t *   local = NULL;
+
+        local = frame->local;
+
+	local->transaction.failed_subvols[child_index] = 1;
+}
+
 
-	local = frame->local;
-	priv  = this->private;
 
-        __mark_child_dead (local->pending, priv->child_count,
-                           child_index, local->transaction.type);
+        static gf_boolean_t
+afr_locals_overlap (afr_local_t *local1, afr_local_t *local2)
+{
+        uint64_t start1 = local1->transaction.start;
+        uint64_t start2 = local2->transaction.start;
+        uint64_t end1 = 0;
+        uint64_t end2 = 0;
+
+        if (local1->transaction.len)
+                end1 = start1 + local1->transaction.len - 1;
+        else
+                end1 = ULLONG_MAX;
+
+        if (local2->transaction.len)
+                end2 = start2 + local2->transaction.len - 1;
+        else
+                end2 = ULLONG_MAX;
+
+        return ((end1 >= start2) && (end2 >= start1));
+}
+
+void
+afr_transaction_eager_lock_init (afr_local_t *local, xlator_t *this)
+{
+        afr_private_t *priv = NULL;
+        afr_fd_ctx_t  *fdctx = NULL;
+        afr_local_t   *each = NULL;
+
+        priv = this->private;
+
+        if (!local->fd)
+                return;
+
+        if (local->transaction.type != AFR_DATA_TRANSACTION)
+                return;
+
+        if (!priv->eager_lock)
+                return;
+
+        fdctx = afr_fd_ctx_get (local->fd, this);
+        if (!fdctx)
+                return;
+
+        if (afr_are_multiple_fds_opened (local->fd, this))
+                return;
+        /*
+         * Once full file lock is acquired in eager-lock phase, overlapping
+         * writes do not compete for inode-locks, instead are transferred to the
+         * next writes. Because of this overlapping writes are not ordered.
+         * This can cause inconsistencies in replication.
+         * Example:
+         * Two overlapping writes w1, w2 are sent in parallel on same fd
+         * in two threads t1, t2.
+         * Both threads can execute afr_writev_wind in the following manner.
+         * t1 winds w1 on brick-0
+         * t2 winds w2 on brick-0
+         * t2 winds w2 on brick-1
+         * t1 winds w1 on brick-1
+         *
+         * This check makes sure the locks are not transferred for
+         * overlapping writes.
+         */
+        LOCK (&local->fd->lock);
+        {
+                list_for_each_entry (each, &fdctx->eager_locked,
+                                     transaction.eager_locked) {
+                        if (afr_locals_overlap (each, local)) {
+                                local->transaction.eager_lock_on = _gf_false;
+                                goto unlock;
+                        }
+                }
+
+                local->transaction.eager_lock_on = _gf_true;
+                list_add_tail (&local->transaction.eager_locked,
+                               &fdctx->eager_locked);
+        }
+unlock:
+        UNLOCK (&local->fd->lock);
 }
 
 
 int
 afr_transaction (call_frame_t *frame, xlator_t *this, afr_transaction_type type)
 {
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
+        afr_local_t *   local = NULL;
+        afr_private_t * priv  = NULL;
+        fd_t            *fd   = NULL;
+        int             ret   = -1;
 
-	local = frame->local;
-	priv  = this->private;
+        local = frame->local;
+        priv  = this->private;
 
-	afr_transaction_local_init (local, priv);
+        local->transaction.resume = afr_transaction_resume;
+        local->transaction.type   = type;
 
-	local->transaction.resume = afr_transaction_resume;
-	local->transaction.type   = type;
+        ret = afr_transaction_local_init (local, this);
+        if (ret < 0)
+            goto out;
 
-	if (afr_lock_server_count (priv, local->transaction.type) == 0) {
-                afr_internal_lock_finish (frame, this);
-	} else {
-		afr_lock (frame, this);
-	}
+        afr_transaction_eager_lock_init (local, this);
 
-	return 0;
+        if (local->fd && local->transaction.eager_lock_on)
+                afr_set_lk_owner (frame, this, local->fd);
+        else
+                afr_set_lk_owner (frame, this, frame->root);
+
+        if (!local->transaction.eager_lock_on && local->loc.inode) {
+                fd = fd_lookup (local->loc.inode, frame->root->pid);
+                if (fd == NULL)
+                        fd = fd_lookup_anonymous (local->loc.inode);
+
+                if (fd) {
+                        afr_delayed_changelog_wake_up (this, fd);
+                        fd_unref (fd);
+                }
+        }
+
+        if (afr_lock_server_count (priv, local->transaction.type) == 0) {
+                afr_internal_lock_finish (frame, this);
+        } else {
+                afr_lock (frame, this);
+        }
+        ret = 0;
+out:
+        return ret;
 }
diff --git a/xlators/cluster/afr/src/afr-transaction.h b/xlators/cluster/afr/src/afr-transaction.h
index 5b45d5a5d..77cc8eed0 100644
--- a/xlators/cluster/afr/src/afr-transaction.h
+++ b/xlators/cluster/afr/src/afr-transaction.h
@@ -1,25 +1,18 @@
 /*
-   Copyright (c) 2007-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __TRANSACTION_H__
 #define __TRANSACTION_H__
 
+#include "afr.h"
+
 void
 afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
 			    int child_index);
@@ -27,7 +20,34 @@ afr_transaction_fop_failed (call_frame_t *frame, xlator_t *this,
 int
 afr_lock_server_count (afr_private_t *priv, afr_transaction_type type);
 
+afr_inodelk_t*
+afr_get_inodelk (afr_internal_lock_t *int_lock, char *dom);
+
 int32_t
 afr_transaction (call_frame_t *frame, xlator_t *this, afr_transaction_type type);
 
+int
+afr_set_pending_dict (afr_private_t *priv, dict_t *xattr, int32_t **pending);
+
+void
+afr_set_delayed_post_op (call_frame_t *frame, xlator_t *this);
+
+void
+afr_delayed_changelog_wake_up (xlator_t *this, fd_t *fd);
+
+void
+__mark_all_success (call_frame_t *frame, xlator_t *this);
+
+gf_boolean_t
+afr_txn_nothing_failed (call_frame_t *frame, xlator_t *this);
+
+int afr_read_txn (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		  afr_read_txn_wind_t readfn, afr_transaction_type type);
+
+int afr_read_txn_continue (call_frame_t *frame, xlator_t *this, int subvol);
+
+int __afr_txn_write_fop (call_frame_t *frame, xlator_t *this);
+int __afr_txn_write_done (call_frame_t *frame, xlator_t *this);
+call_frame_t *afr_transaction_detach_fop_frame (call_frame_t *frame);
+
 #endif /* __TRANSACTION_H__ */
diff --git a/xlators/cluster/afr/src/afr.c b/xlators/cluster/afr/src/afr.c
index b026db594..ead08425f 100644
--- a/xlators/cluster/afr/src/afr.c
+++ b/xlators/cluster/afr/src/afr.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <libgen.h>
@@ -30,13 +21,20 @@
 #endif
 #include "afr-common.c"
 
+struct volume_options options[];
+
 int32_t
 notify (xlator_t *this, int32_t event,
-	void *data, ...)
+        void *data, ...)
 {
         int ret = -1;
+        va_list         ap;
+        void *data2 = NULL;
 
-        ret = afr_notify (this, event, data);
+        va_start (ap, data);
+        data2 = va_arg (ap, dict_t*);
+        va_end (ap);
+        ret = afr_notify (this, event, data, data2);
 
         return ret;
 }
@@ -53,896 +51,449 @@ mem_acct_init (xlator_t *this)
 
         if (ret != 0) {
                 gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
+                       "failed");
                 return ret;
         }
 
         return ret;
 }
+
+
 int
-validate_options (xlator_t *this, dict_t *options, char **op_errstr)
+xlator_subvolume_index (xlator_t *this, xlator_t *subvol)
 {
+        int index = -1;
+        int i = 0;
+        xlator_list_t *list = NULL;
 
- 
-        gf_boolean_t metadata_self_heal;   
-        gf_boolean_t entry_self_heal;
-        gf_boolean_t data_self_heal;
-        gf_boolean_t data_change_log;       
-        gf_boolean_t metadata_change_log; 
-        gf_boolean_t entry_change_log;      
-        gf_boolean_t strict_readdir;
-
-        xlator_list_t * trav        = NULL;
-        
-        char * read_subvol     = NULL;
-        char * self_heal       = NULL;
-        char * change_log      = NULL;
-        char * str_readdir     = NULL;
-
-        int32_t background_count  = 0;
-        int32_t window_size       = 0;
-
-        int    read_ret      = -1;
-        int    dict_ret      = -1;
-        int    flag          = 1;
-        int    ret           = 0;
-        int    temp_ret      = -1;
-        
-
-        
-        dict_ret = dict_get_int32 (options, "background-self-heal-count",
-                                   &background_count);
-        if (dict_ret == 0) {
-                if (background_count < 0) {
-                        *op_errstr = gf_strdup ("Error, option should be >= 0");
-                        ret = -1;
-                        goto out;
-                }
+        list = this->children;
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "validated background self-heal count to %d",
-                        background_count);
+        while (list) {
+                if (subvol == list->xlator ||
+                    strcmp (subvol->name, list->xlator->name) == 0) {
+                        index = i;
+                        break;
+                }
+                list = list->next;
+                i++;
         }
 
-        dict_ret = dict_get_str (options, "metadata-self-heal",
-                                 &self_heal);
-        if (dict_ret == 0) {
-                temp_ret = gf_string2boolean (self_heal, &metadata_self_heal);
-                if (temp_ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "validation failed 'option metadata"
-                                                "-self-heal %s'.not correct.", 
-                                                self_heal);
-                        *op_errstr = gf_strdup ("Error, option should be boolean");
-                        ret = -1;
-                        goto out;
-                } 
-        
+        return index;
+}
 
-                
+void
+fix_quorum_options (xlator_t *this, afr_private_t *priv, char *qtype)
+{
+        if (priv->quorum_count && strcmp(qtype,"fixed")) {
+                gf_log(this->name,GF_LOG_WARNING,
+                       "quorum-type %s overriding quorum-count %u",
+                       qtype, priv->quorum_count);
         }
-
-        dict_ret = dict_get_str (options, "data-self-heal",
-                                 &self_heal);
-        if (dict_ret == 0) {
-                temp_ret = gf_string2boolean (self_heal, &data_self_heal);
-                if (temp_ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Validation failed for data self heal", 
-                                self_heal);
-                        *op_errstr = gf_strdup ("Error, option should be boolean");
-                        ret = -1;
-                        goto out;
-                } 
-        
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Reconfiguring 'option data"
-                                        "-self-heal %s'.", self_heal);
+        if (!strcmp(qtype,"none")) {
+                priv->quorum_count = 0;
         }
-
-        dict_ret = dict_get_str (options, "entry-self-heal",
-                                 &self_heal);
-        if (dict_ret == 0) {
-                temp_ret = gf_string2boolean (self_heal, &entry_self_heal);
-                if (temp_ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Validation faled for entry-self-heal", 
-                                self_heal);
-                        *op_errstr = gf_strdup ("Error, option should be boolean");
-                        ret = -1;
-                        goto out;
-                } 
-        
-
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Validated 'option entry"
-                                        "-self-heal %s'.", self_heal);
+        else if (!strcmp(qtype,"auto")) {
+                priv->quorum_count = AFR_QUORUM_AUTO;
         }
+}
 
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        afr_private_t *priv        = NULL;
+        xlator_t      *read_subvol = NULL;
+        int            read_subvol_index = -1;
+        int            ret         = -1;
+        int            index       = -1;
+        char          *qtype       = NULL;
 
-        dict_ret = dict_get_str (options, "strict-readdir",
-                                 &str_readdir);
-        if (dict_ret == 0) {
-                temp_ret = gf_string2boolean (str_readdir, &strict_readdir);
-                if (temp_ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Validation faled for strict_readdir",
-                                str_readdir);
-                        *op_errstr = gf_strdup ("Error, option should be boolean");
-                        ret = -1;
-                        goto out;
-                }
-                
+        priv = this->private;
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Validated 'option strict"
-                                        "-readdir %s'.", str_readdir);
-        }
+	GF_OPTION_RECONF ("afr-dirty-xattr",
+			  priv->afr_dirty, options, str,
+			  out);
 
-        dict_ret = dict_get_int32 (options, "data-self-heal-window-size",
-                                   &window_size);
-        if (dict_ret == 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "validated data self-heal window size to %d",
-                        window_size);
+	GF_OPTION_RECONF ("metadata-splitbrain-forced-heal",
+			  priv->metadata_splitbrain_forced_heal, options, bool,
+			  out);
 
-                if (window_size < 0) {
-                        *op_errstr = gf_strdup ("Error, option should be >= 0");
-                        ret = -1;
-                        goto out;
-                }
-
-                if (window_size > 1024) {
-                        *op_errstr = gf_strdup ("Error, option should be <= 1024");
-                        ret = -1;
-                        goto out;
-                }
+        GF_OPTION_RECONF ("background-self-heal-count",
+                          priv->background_self_heal_count, options, uint32,
+                          out);
 
+        GF_OPTION_RECONF ("metadata-self-heal",
+                          priv->metadata_self_heal, options, bool, out);
 
-        }
+        GF_OPTION_RECONF ("data-self-heal", priv->data_self_heal, options, str,
+                          out);
 
-        dict_ret = dict_get_str (options, "data-change-log",
-                                 &change_log);
-        if (dict_ret == 0) {
-                temp_ret = gf_string2boolean (change_log, &data_change_log);
-                if (temp_ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Validation faled for data-change-log");
-                        *op_errstr = gf_strdup ("Error, option should be boolean");
-                        ret = -1;
-                        goto out;
-                }
- 
+        GF_OPTION_RECONF ("entry-self-heal", priv->entry_self_heal, options,
+                          bool, out);
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Validated 'option data-"
-                                        "change-log %s'.", change_log);
-        }
+        GF_OPTION_RECONF ("data-self-heal-window-size",
+                          priv->data_self_heal_window_size, options,
+                          uint32, out);
 
-        dict_ret = dict_get_str (options, "metadata-change-log",
-                                 &change_log);
-        if (dict_ret == 0) {
-                temp_ret = gf_string2boolean (change_log,
-                                              &metadata_change_log);
-                if (temp_ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Validation faild for metadata-change-log");
-                        *op_errstr = gf_strdup ("Error, option should be boolean");
-                        ret = -1;
-                        goto out;
-                } 
-                
+        GF_OPTION_RECONF ("data-change-log", priv->data_change_log, options,
+                          bool, out);
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Validated 'option metadata-"
-                                        "change-log %s'.", change_log);
-        }
+        GF_OPTION_RECONF ("metadata-change-log",
+                          priv->metadata_change_log, options, bool, out);
 
-        dict_ret = dict_get_str (options, "entry-change-log",
-                                 &change_log);
-        if (dict_ret == 0) {
-                temp_ret = gf_string2boolean (change_log, &entry_change_log);
-                if (temp_ret < 0) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "Validation faild for entr-change-log");
-                        *op_errstr = gf_strdup ("Error, option should be boolean");
-                        ret = -1;
-                        goto out;
-                } 
+        GF_OPTION_RECONF ("entry-change-log", priv->entry_change_log, options,
+                          bool, out);
 
+        GF_OPTION_RECONF ("data-self-heal-algorithm",
+                          priv->data_self_heal_algorithm, options, str, out);
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Validated 'option entry-"
-                                        "change-log %s'.", change_log);
-        }
+        GF_OPTION_RECONF ("read-subvolume", read_subvol, options, xlator, out);
 
-        read_ret = dict_get_str (options, "read-subvolume", &read_subvol);
+        GF_OPTION_RECONF ("read-hash-mode", priv->hash_mode,
+                          options, uint32, out);
 
-        if (read_ret) 
-                goto next;// No need to traverse, hence set the next option
-        
-        trav = this->children;
-        flag = 0;
-        while (trav) {
-                if (!read_ret && !strcmp (read_subvol, trav->xlator->name)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Validated Subvolume '%s'  as read child.",
-                                trav->xlator->name);
-
-                        flag = 1;
-                        ret = 0; 
+        if (read_subvol) {
+                index = xlator_subvolume_index (this, read_subvol);
+                if (index == -1) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s not a subvolume",
+                                read_subvol->name);
                         goto out;
                 }
-
-                
-                trav = trav->next;
+                priv->read_child = index;
         }
 
-        if (flag == 0 ) {
+        GF_OPTION_RECONF ("read-subvolume-index",read_subvol_index, options,int32,out);
 
-                gf_log (this->name, GF_LOG_WARNING,
-                        "Invalid 'option read-subvolume %s', no such subvolume"
-                                        , read_subvol);
-                *op_errstr = gf_strdup ("Error, the sub-volume is not right");
-                ret = -1;
-                goto out;
+        if (read_subvol_index >-1) {
+                index=read_subvol_index;
+                if (index >= priv->child_count) {
+                        gf_log (this->name, GF_LOG_ERROR, "%d not a subvolume-index",
+                                index);
+                        goto out;
+                }
+                priv->read_child = index;
         }
 
-next:
-out:
-                return ret;
-
-
-}
-
+        GF_OPTION_RECONF ("pre-op-compat", priv->pre_op_compat, options, bool, out);
 
-int
-reconfigure (xlator_t *this, dict_t *options)
-{
+        GF_OPTION_RECONF ("eager-lock", priv->eager_lock, options, bool, out);
+        GF_OPTION_RECONF ("quorum-type", qtype, options, str, out);
+        GF_OPTION_RECONF ("quorum-count", priv->quorum_count, options,
+                          uint32, out);
+        fix_quorum_options(this,priv,qtype);
 
-	gf_boolean_t metadata_self_heal;   /* on/off */
-	gf_boolean_t entry_self_heal;
-	gf_boolean_t data_self_heal;
-	gf_boolean_t data_change_log;       /* on/off */
-	gf_boolean_t metadata_change_log;   /* on/off */
-	gf_boolean_t entry_change_log;      /* on/off */
-	gf_boolean_t strict_readdir;
-
-	afr_private_t * priv        = NULL;
-	xlator_list_t * trav        = NULL;
-	
-	char * read_subvol     = NULL;
-	char * self_heal       = NULL;
-	char * change_log      = NULL;
-	char * str_readdir     = NULL;
-
-        int32_t background_count  = 0;
-        int32_t window_size       = 0;
-
-	int    read_ret      = -1;
-	int    dict_ret      = -1;
-	int    flag	     = 1;
-	int    ret           = 0;
-	int    temp_ret	     = -1;
-	
-	priv = this->private;
-	
-	dict_ret = dict_get_int32 (options, "background-self-heal-count",
-				   &background_count);
-	if (dict_ret == 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"Reconfiguring background self-heal count to %d",
-			background_count);
-
-		priv->background_self_heal_count = background_count;
-	}
+	GF_OPTION_RECONF ("post-op-delay-secs", priv->post_op_delay_secs, options,
+			  uint32, out);
 
-	dict_ret = dict_get_str (options, "metadata-self-heal",
-				 &self_heal);
-	if (dict_ret == 0) {
-		temp_ret = gf_string2boolean (self_heal, &metadata_self_heal);
-		if (temp_ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Reconfiguration Invalid 'option metadata"
-				"-self-heal %s'. Defaulting to old value.", 
-				self_heal);
-			ret = -1;
-			goto out;
-		} 
-	
-		priv->metadata_self_heal = metadata_self_heal;
-		gf_log (this->name, GF_LOG_DEBUG,
-			"Reconfiguring 'option metadata"
-			"-self-heal %s'.", 
-			self_heal);
-	}
+        GF_OPTION_RECONF (AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size,
+                          options, size_uint64, out);
+        /* Reset this so we re-discover in case the topology changed.  */
+        GF_OPTION_RECONF ("ensure-durability", priv->ensure_durability, options,
+                          bool, out);
 
-	dict_ret = dict_get_str (options, "data-self-heal",
-				 &self_heal);
-	if (dict_ret == 0) {
-		temp_ret = gf_string2boolean (self_heal, &data_self_heal);
-		if (temp_ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Reconfiguration Invalid 'option data"
-				"-self-heal %s'. Defaulting to old value.", 
-				self_heal);
-			ret = -1;
-			goto out;
-		} 
-	
-		priv->data_self_heal = data_self_heal;
-		gf_log (this->name, GF_LOG_DEBUG,
-			"Reconfiguring 'option data"
-			"-self-heal %s'.", self_heal);
-	}
+	GF_OPTION_RECONF ("self-heal-daemon", priv->shd.enabled, options,
+			  bool, out);
 
-	dict_ret = dict_get_str (options, "entry-self-heal",
-				 &self_heal);
-	if (dict_ret == 0) {
-		temp_ret = gf_string2boolean (self_heal, &entry_self_heal);
-		if (temp_ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Reconfiguration Invalid 'option data"
-				"-self-heal %s'. Defaulting to old value.", 
-				self_heal);
-			ret = -1;
-			goto out;
-		} 
-	
-		priv->entry_self_heal = entry_self_heal;
-		gf_log (this->name, GF_LOG_DEBUG,
-			"Reconfiguring 'option entry"
-			"-self-heal %s'.", self_heal);
-	}
-
-
-	dict_ret = dict_get_str (options, "strict-readdir",
-				 &str_readdir);
-	if (dict_ret == 0) {
-		temp_ret = gf_string2boolean (str_readdir, &strict_readdir);
-		if (temp_ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Invalid 'option strict-readdir %s'. "
-				"Defaulting to old value.",
-				str_readdir);
-			ret = -1;
-			goto out;
-		}
-		
-		priv->strict_readdir = strict_readdir;
-		gf_log (this->name, GF_LOG_DEBUG,
-			"Reconfiguring 'option strict"
-			"-readdir %s'.", str_readdir);
-	}
-
-	dict_ret = dict_get_int32 (options, "data-self-heal-window-size",
-				   &window_size);
-	if (dict_ret == 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"Reconfiguring, Setting data self-heal window size to %d",
-			window_size);
+	GF_OPTION_RECONF ("iam-self-heal-daemon", priv->shd.iamshd, options,
+			  bool, out);
 
-		priv->data_self_heal_window_size = window_size;
-	}
-        else {
-                priv->data_self_heal_window_size = 16;
-        }
-        
-
-	dict_ret = dict_get_str (options, "data-change-log",
-				 &change_log);
-	if (dict_ret == 0) {
-		temp_ret = gf_string2boolean (change_log, &data_change_log);
-		if (temp_ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Reconfiguration Invalid 'option data-"
-				"change-log %s'. Defaulting to old value.", 
-				change_log);
-			ret = -1;
-			goto out;
-		}
- 
-		priv->data_change_log = data_change_log;
-		gf_log (this->name, GF_LOG_DEBUG,
-			"Reconfiguring 'option data-"
-			"change-log %s'.", change_log);
-	}
-
-	dict_ret = dict_get_str (options, "metadata-change-log",
-				 &change_log);
-	if (dict_ret == 0) {
-		temp_ret = gf_string2boolean (change_log,
-					 &metadata_change_log);
-		if (temp_ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Invalid 'option metadata-change-log %s'. "
-				"Defaulting to metadata-change-log as 'off'.",
-				change_log);
-			ret = -1;
-			goto out;
-		} 
-		
-		priv->metadata_change_log = metadata_change_log;
-		gf_log (this->name, GF_LOG_DEBUG,
-			"Reconfiguring 'option metadata-"
-			"change-log %s'.", change_log);
-	}
-
-	dict_ret = dict_get_str (options, "entry-change-log",
-				 &change_log);
-	if (dict_ret == 0) {
-		temp_ret = gf_string2boolean (change_log, &entry_change_log);
-		if (temp_ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Invalid 'option entry-change-log %s'. "
-				"Defaulting to entry-change-log as 'on'.", 
-				change_log);
-			ret = -1;
-			goto out;
-		} 
-
-		priv->entry_change_log = entry_change_log;
-		gf_log (this->name, GF_LOG_DEBUG,
-			"Reconfiguring 'option entry-"
-			"change-log %s'.", change_log);
-	}
-
-	read_ret = dict_get_str (options, "read-subvolume", &read_subvol);
-
-	if (read_ret < 0) 
-		goto next;// No need to traverse, hence set the next option
-	
-	trav = this->children;
-	flag = 0;
-	while (trav) {
-		if (!read_ret && !strcmp (read_subvol, trav->xlator->name)) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"Subvolume '%s' specified as read child.",
-				trav->xlator->name);
-
-			flag = 1;
-			ret = -1; 
-			goto out;
-		}
-
-		
-		trav = trav->next;
-	}
-
-	if (flag == 0 ) {
-
-		gf_log (this->name, GF_LOG_ERROR,
-			"Invalid 'option read-subvolume %s', no such subvolume"
-			, read_subvol);
-		ret = -1;
-		goto out;
-	}
+        priv->did_discovery = _gf_false;
 
-next:
+        ret = 0;
 out:
-	return ret;
+        return ret;
 
 }
 
 
 static const char *favorite_child_warning_str = "You have specified subvolume '%s' "
-	"as the 'favorite child'. This means that if a discrepancy in the content "
-	"or attributes (ownership, permission, etc.) of a file is detected among "
-	"the subvolumes, the file on '%s' will be considered the definitive "
-	"version and its contents will OVERWRITE the contents of the file on other "
-	"subvolumes. All versions of the file except that on '%s' "
-	"WILL BE LOST.";
-
-static const char *no_lock_servers_warning_str = "You have set lock-server-count = 0. "
-	"This means correctness is NO LONGER GUARANTEED in all cases. If two or more "
-	"applications write to the same region of a file, there is a possibility that "
-	"its copies will be INCONSISTENT. Set it to a value greater than 0 unless you "
-	"are ABSOLUTELY SURE of what you are doing and WILL NOT HOLD GlusterFS "
-	"RESPONSIBLE for inconsistent data. If you are in doubt, set it to a value "
-	"greater than 0.";
-
-int32_t 
+        "as the 'favorite child'. This means that if a discrepancy in the content "
+        "or attributes (ownership, permission, etc.) of a file is detected among "
+        "the subvolumes, the file on '%s' will be considered the definitive "
+        "version and its contents will OVERWRITE the contents of the file on other "
+        "subvolumes. All versions of the file except that on '%s' "
+        "WILL BE LOST.";
+
+
+int32_t
 init (xlator_t *this)
 {
-	afr_private_t * priv        = NULL;
-	int             child_count = 0;
-	xlator_list_t * trav        = NULL;
-	int             i           = 0;
-	int             ret         = -1;
-	int             op_errno    = 0;
-
-	char * read_subvol     = NULL;
-	char * fav_child       = NULL;
-	char * self_heal       = NULL;
-        char * algo            = NULL;
-	char * change_log      = NULL;
-	char * strict_readdir  = NULL;
-        char * inodelk_trace   = NULL;
-        char * entrylk_trace   = NULL;
-
-        int32_t background_count  = 0;
-	int32_t lock_server_count = 1;
-        int32_t window_size       = 0;
-
-	int    fav_ret       = -1;
-	int    read_ret      = -1;
-	int    dict_ret      = -1;
-
-
-	if (!this->children) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"replicate translator needs more than one "
+        afr_private_t *priv        = NULL;
+        int            child_count = 0;
+        xlator_list_t *trav        = NULL;
+        int            i           = 0;
+        int            ret         = -1;
+        GF_UNUSED int  op_errno    = 0;
+        xlator_t      *read_subvol = NULL;
+        int            read_subvol_index = -1;
+        xlator_t      *fav_child   = NULL;
+        char          *qtype       = NULL;
+
+        if (!this->children) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "replicate translator needs more than one "
                         "subvolume defined.");
-		return -1;
-	}
-  
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"Volume is dangling.");
-	}
-
-
-	ALLOC_OR_GOTO (this->private, afr_private_t, out);
-
-	priv = this->private;
-
-	read_ret = dict_get_str (this->options, "read-subvolume", &read_subvol);
-	priv->read_child = -1;
-
-	fav_ret = dict_get_str (this->options, "favorite-child", &fav_child);
-	priv->favorite_child = -1;
-
-        priv->background_self_heal_count = 16;
-
-	dict_ret = dict_get_int32 (this->options, "background-self-heal-count",
-				   &background_count);
-	if (dict_ret == 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"Setting background self-heal count to %d",
-			background_count);
-
-		priv->background_self_heal_count = background_count;
-	}
-
-	/* Default values */
-
-	priv->data_self_heal     = 1;
-	priv->metadata_self_heal = 1;
-	priv->entry_self_heal    = 1;
-
-	dict_ret = dict_get_str (this->options, "data-self-heal", &self_heal);
-	if (dict_ret == 0) {
-		ret = gf_string2boolean (self_heal, &priv->data_self_heal);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Invalid 'option data-self-heal %s'. "
-				"Defaulting to data-self-heal as 'on'",
-				self_heal);
-			priv->data_self_heal = 1;
-		}
-	}
-
-        priv->data_self_heal_algorithm = "";
-
-        dict_ret = dict_get_str (this->options, "data-self-heal-algorithm",
-                                 &algo);
-        if (dict_ret == 0) {
-                priv->data_self_heal_algorithm = gf_strdup (algo);
+                return -1;
         }
 
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Volume is dangling.");
+        }
 
-        priv->data_self_heal_window_size = 16;
-
-	dict_ret = dict_get_int32 (this->options, "data-self-heal-window-size",
-				   &window_size);
-	if (dict_ret == 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"Setting data self-heal window size to %d",
-			window_size);
+	this->private = GF_CALLOC (1, sizeof (afr_private_t),
+                                   gf_afr_mt_afr_private_t);
+        if (!this->private)
+                goto out;
 
-		priv->data_self_heal_window_size = window_size;
-	}
+        priv = this->private;
+        LOCK_INIT (&priv->lock);
 
-	dict_ret = dict_get_str (this->options, "metadata-self-heal",
-				 &self_heal);
-	if (dict_ret == 0) {
-		ret = gf_string2boolean (self_heal, &priv->metadata_self_heal);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Invalid 'option metadata-self-heal %s'. "
-				"Defaulting to metadata-self-heal as 'on'.", 
-				self_heal);
-			priv->metadata_self_heal = 1;
-		} 
-	}
+        child_count = xlator_subvolume_count (this);
 
-	dict_ret = dict_get_str (this->options, "entry-self-heal", &self_heal);
-	if (dict_ret == 0) {
-		ret = gf_string2boolean (self_heal, &priv->entry_self_heal);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Invalid 'option entry-self-heal %s'. "
-				"Defaulting to entry-self-heal as 'on'.", 
-				self_heal);
-			priv->entry_self_heal = 1;
-		} 
-	}
+        priv->child_count = child_count;
 
-	/* Change log options */
-
-	priv->data_change_log     = 1;
-	priv->metadata_change_log = 1;
-	priv->entry_change_log    = 1;
-
-	dict_ret = dict_get_str (this->options, "data-change-log",
-				 &change_log);
-	if (dict_ret == 0) {
-		ret = gf_string2boolean (change_log, &priv->data_change_log);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Invalid 'option data-change-log %s'. "
-				"Defaulting to data-change-log as 'on'.", 
-				change_log);
-			priv->data_change_log = 1;
-		} 
-	}
+        priv->read_child = -1;
 
-	dict_ret = dict_get_str (this->options, "metadata-change-log",
-				 &change_log);
-	if (dict_ret == 0) {
-		ret = gf_string2boolean (change_log,
-					 &priv->metadata_change_log);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Invalid 'option metadata-change-log %s'. "
-				"Defaulting to metadata-change-log as 'off'.",
-				change_log);
-			priv->metadata_change_log = 0;
-		} 
-	}
+	GF_OPTION_INIT ("afr-dirty-xattr", priv->afr_dirty, str, out);
 
-	dict_ret = dict_get_str (this->options, "entry-change-log",
-				 &change_log);
-	if (dict_ret == 0) {
-		ret = gf_string2boolean (change_log, &priv->entry_change_log);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Invalid 'option entry-change-log %s'. "
-				"Defaulting to entry-change-log as 'on'.", 
-				change_log);
-			priv->entry_change_log = 1;
-		} 
-	}
+	GF_OPTION_INIT ("metadata-splitbrain-forced-heal",
+			priv->metadata_splitbrain_forced_heal, bool, out);
 
-	/* Locking options */
+        GF_OPTION_INIT ("read-subvolume", read_subvol, xlator, out);
+        if (read_subvol) {
+                priv->read_child = xlator_subvolume_index (this, read_subvol);
+                if (priv->read_child == -1) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s not a subvolume",
+                                read_subvol->name);
+                        goto out;
+                }
+        }
+        GF_OPTION_INIT ("read-subvolume-index",read_subvol_index,int32,out);
+        if (read_subvol_index > -1) {
+                if (read_subvol_index >= priv->child_count) {
+                        gf_log (this->name, GF_LOG_ERROR, "%d not a subvolume-index",
+                                read_subvol_index);
+                        goto out;
+                }
+                priv->read_child = read_subvol_index;
+        }
+        GF_OPTION_INIT ("choose-local", priv->choose_local, bool, out);
 
-        priv->inodelk_trace = 0;
-        priv->entrylk_trace = 0;
+        GF_OPTION_INIT ("read-hash-mode", priv->hash_mode, uint32, out);
 
-	dict_ret = dict_get_str (this->options, "inodelk-trace",
-				 &inodelk_trace);
-	if (dict_ret == 0) {
-		ret = gf_string2boolean (inodelk_trace, &priv->inodelk_trace);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Invalid 'option inodelk-trace %s' ",
-                                inodelk_trace);
+        priv->favorite_child = -1;
+        GF_OPTION_INIT ("favorite-child", fav_child, xlator, out);
+        if (fav_child) {
+                priv->favorite_child = xlator_subvolume_index (this, fav_child);
+                if (priv->favorite_child == -1) {
+                        gf_log (this->name, GF_LOG_ERROR, "%s not a subvolume",
+                                fav_child->name);
+                        goto out;
+                }
+                gf_log (this->name, GF_LOG_WARNING,
+                        favorite_child_warning_str, fav_child->name,
+                        fav_child->name, fav_child->name);
+        }
 
-			priv->inodelk_trace = 0;
-		}
-	}
 
+        GF_OPTION_INIT ("background-self-heal-count",
+                        priv->background_self_heal_count, uint32, out);
 
-	dict_ret = dict_get_str (this->options, "entrylk-trace",
-				 &entrylk_trace);
-	if (dict_ret == 0) {
-		ret = gf_string2boolean (entrylk_trace, &priv->entrylk_trace);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Invalid 'option entrylk-trace %s' ",
-                                inodelk_trace);
+        GF_OPTION_INIT ("data-self-heal", priv->data_self_heal, str, out);
 
-			priv->entrylk_trace = 0;
-		}
-	}
+        GF_OPTION_INIT ("data-self-heal-algorithm",
+                        priv->data_self_heal_algorithm, str, out);
 
+        GF_OPTION_INIT ("data-self-heal-window-size",
+                        priv->data_self_heal_window_size, uint32, out);
 
-	priv->data_lock_server_count = 1;
-	priv->metadata_lock_server_count = 0;
-	priv->entry_lock_server_count = 1;
+        GF_OPTION_INIT ("metadata-self-heal", priv->metadata_self_heal, bool,
+                        out);
 
-	dict_ret = dict_get_int32 (this->options, "data-lock-server-count", 
-				   &lock_server_count);
-	if (dict_ret == 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"Setting data lock server count to %d.",
-			lock_server_count);
+        GF_OPTION_INIT ("entry-self-heal", priv->entry_self_heal, bool, out);
 
-		if (lock_server_count == 0) 
-			gf_log (this->name, GF_LOG_WARNING, "%s",
-                                no_lock_servers_warning_str);
+        GF_OPTION_INIT ("data-change-log", priv->data_change_log, bool, out);
 
-		priv->data_lock_server_count = lock_server_count;
-	}
+        GF_OPTION_INIT ("metadata-change-log", priv->metadata_change_log, bool,
+                        out);
 
+        GF_OPTION_INIT ("entry-change-log", priv->entry_change_log, bool, out);
 
-	dict_ret = dict_get_int32 (this->options,
-				   "metadata-lock-server-count", 
-				   &lock_server_count);
-	if (dict_ret == 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"Setting metadata lock server count to %d.",
-			lock_server_count);
-		priv->metadata_lock_server_count = lock_server_count;
-	}
+        GF_OPTION_INIT ("optimistic-change-log", priv->optimistic_change_log,
+                        bool, out);
 
+        GF_OPTION_INIT ("inodelk-trace", priv->inodelk_trace, bool, out);
 
-	dict_ret = dict_get_int32 (this->options, "entry-lock-server-count", 
-				   &lock_server_count);
-	if (dict_ret == 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"Setting entry lock server count to %d.",
-			lock_server_count);
+        GF_OPTION_INIT ("entrylk-trace", priv->entrylk_trace, bool, out);
 
-		priv->entry_lock_server_count = lock_server_count;
-	}
+        GF_OPTION_INIT ("pre-op-compat", priv->pre_op_compat, bool, out);
 
-	priv->strict_readdir = _gf_false;
-
-	dict_ret = dict_get_str (this->options, "strict-readdir",
-				 &strict_readdir);
-	if (dict_ret == 0) {
-		ret = gf_string2boolean (strict_readdir, &priv->strict_readdir);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Invalid 'option strict-readdir %s'. "
-				"Defaulting to strict-readdir as 'off'.",
-				strict_readdir);
-		}
-	}
+        GF_OPTION_INIT ("eager-lock", priv->eager_lock, bool, out);
+        GF_OPTION_INIT ("quorum-type", qtype, str, out);
+        GF_OPTION_INIT ("quorum-count", priv->quorum_count, uint32, out);
+        GF_OPTION_INIT (AFR_SH_READDIR_SIZE_KEY, priv->sh_readdir_size, size_uint64,
+                        out);
+        fix_quorum_options(this,priv,qtype);
 
-	trav = this->children;
-	while (trav) {
-		if (!read_ret && !strcmp (read_subvol, trav->xlator->name)) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"Subvolume '%s' specified as read child.",
-				trav->xlator->name);
-
-			priv->read_child = child_count;
-		}
-
-		if (fav_ret == 0 && !strcmp (fav_child, trav->xlator->name)) {
-			gf_log (this->name, GF_LOG_WARNING,
-				favorite_child_warning_str, trav->xlator->name,
-				trav->xlator->name, trav->xlator->name);
-			priv->favorite_child = child_count;
-		}
-
-		child_count++;
-		trav = trav->next;
-	}
+	GF_OPTION_INIT ("post-op-delay-secs", priv->post_op_delay_secs, uint32, out);
+        GF_OPTION_INIT ("ensure-durability", priv->ensure_durability, bool,
+                        out);
 
-	priv->wait_count = 1;
+	GF_OPTION_INIT ("self-heal-daemon", priv->shd.enabled, bool, out);
 
-	priv->child_count = child_count;
+	GF_OPTION_INIT ("iam-self-heal-daemon", priv->shd.iamshd, bool, out);
 
-	LOCK_INIT (&priv->lock);
-        LOCK_INIT (&priv->read_child_lock);
+        priv->wait_count = 1;
 
-	priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,
+        priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,
                                     gf_afr_mt_char);
-	if (!priv->child_up) {
-		gf_log (this->name, GF_LOG_ERROR,	
-			"Out of memory.");		
-		ret = -ENOMEM;
-		goto out;
-	}
+        if (!priv->child_up) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        for (i = 0; i < child_count; i++)
+                priv->child_up[i] = -1; /* start with unknown state.
+                                           this initialization needed
+                                           for afr_notify() to work
+                                           reliably
+                                        */
 
-	priv->children = GF_CALLOC (sizeof (xlator_t *), child_count,
+        priv->children = GF_CALLOC (sizeof (xlator_t *), child_count,
                                     gf_afr_mt_xlator_t);
-	if (!priv->children) {
-		gf_log (this->name, GF_LOG_ERROR,	
-			"Out of memory.");		
-		ret = -ENOMEM;
-		goto out;
-	}
+        if (!priv->children) {
+                ret = -ENOMEM;
+                goto out;
+        }
 
-        priv->pending_key = GF_CALLOC (sizeof (*priv->pending_key), 
-                                        child_count,
-                                        gf_afr_mt_char);
+        priv->pending_key = GF_CALLOC (sizeof (*priv->pending_key),
+                                       child_count,
+                                       gf_afr_mt_char);
         if (!priv->pending_key) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
                 ret = -ENOMEM;
                 goto out;
         }
 
-	trav = this->children;
-	i = 0;
-	while (i < child_count) {
-		priv->children[i] = trav->xlator;
+        trav = this->children;
+        i = 0;
+        while (i < child_count) {
+                priv->children[i] = trav->xlator;
 
-                ret = gf_asprintf (&priv->pending_key[i], "%s.%s", 
+                ret = gf_asprintf (&priv->pending_key[i], "%s.%s",
                                    AFR_XATTR_PREFIX,
                                    trav->xlator->name);
                 if (-1 == ret) {
-                        gf_log (this->name, GF_LOG_ERROR, 
-                                "asprintf failed to set pending key");
                         ret = -ENOMEM;
                         goto out;
                 }
 
-		trav = trav->next;
-		i++;
+                trav = trav->next;
+                i++;
+        }
+
+        ret = gf_asprintf (&priv->sh_domain, AFR_SH_DATA_DOMAIN_FMT,
+                           this->name);
+        if (-1 == ret) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        priv->last_event = GF_CALLOC (child_count, sizeof (*priv->last_event),
+                                      gf_afr_mt_int32_t);
+        if (!priv->last_event) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+	ret = afr_selfheal_daemon_init (this);
+	if (ret) {
+		ret = -ENOMEM;
+		goto out;
 	}
 
-        LOCK_INIT (&priv->root_inode_lk);
-        priv->first_lookup = 1;
-        priv->root_inode = NULL;
+        /* keep more local here as we may need them for self-heal etc */
+        this->local_pool = mem_pool_new (afr_local_t, 512);
+        if (!this->local_pool) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto out;
+        }
 
-        pthread_mutex_init (&priv->mutex, NULL);
-        INIT_LIST_HEAD (&priv->saved_fds);
+        priv->root_inode = NULL;
 
-	ret = 0;
+        ret = 0;
 out:
-	return ret;
+        return ret;
 }
 
 
 int
 fini (xlator_t *this)
 {
-	return 0;
+        afr_private_t *priv = NULL;
+
+        priv = this->private;
+        this->private = NULL;
+        afr_priv_destroy (priv);
+        //if (this->itable);//I dont see any destroy func
+
+        return 0;
 }
 
 
 struct xlator_fops fops = {
-	.lookup      = afr_lookup,
-	.open        = afr_open,
-	.lk          = afr_lk,
-	.flush       = afr_flush,
-	.statfs      = afr_statfs,
-	.fsync       = afr_fsync,
-	.fsyncdir    = afr_fsyncdir,
-	.xattrop     = afr_xattrop,
-	.fxattrop    = afr_fxattrop,
-	.inodelk     = afr_inodelk,
-	.finodelk    = afr_finodelk,
-	.entrylk     = afr_entrylk,
-	.fentrylk    = afr_fentrylk,
-
-	/* inode read */
-	.access      = afr_access,
-	.stat        = afr_stat,
-	.fstat       = afr_fstat,
-	.readlink    = afr_readlink,
-	.getxattr    = afr_getxattr,
-	.readv       = afr_readv,
-
-	/* inode write */
-	.writev      = afr_writev,
-	.truncate    = afr_truncate,
-	.ftruncate   = afr_ftruncate,
-	.setxattr    = afr_setxattr,
+        .lookup      = afr_lookup,
+        .open        = afr_open,
+        .lk          = afr_lk,
+        .flush       = afr_flush,
+        .statfs      = afr_statfs,
+        .fsync       = afr_fsync,
+        .fsyncdir    = afr_fsyncdir,
+        .xattrop     = afr_xattrop,
+        .fxattrop    = afr_fxattrop,
+        .inodelk     = afr_inodelk,
+        .finodelk    = afr_finodelk,
+        .entrylk     = afr_entrylk,
+        .fentrylk    = afr_fentrylk,
+	.fallocate   = afr_fallocate,
+	.discard     = afr_discard,
+        .zerofill    = afr_zerofill,
+
+        /* inode read */
+        .access      = afr_access,
+        .stat        = afr_stat,
+        .fstat       = afr_fstat,
+        .readlink    = afr_readlink,
+        .getxattr    = afr_getxattr,
+        .fgetxattr   = afr_fgetxattr,
+        .readv       = afr_readv,
+
+        /* inode write */
+        .writev      = afr_writev,
+        .truncate    = afr_truncate,
+        .ftruncate   = afr_ftruncate,
+        .setxattr    = afr_setxattr,
+        .fsetxattr   = afr_fsetxattr,
         .setattr     = afr_setattr,
-	.fsetattr    = afr_fsetattr,
-	.removexattr = afr_removexattr,
-
-	/* dir read */
-	.opendir     = afr_opendir,
-	.readdir     = afr_readdir,
-	.readdirp    = afr_readdirp,
-
-	/* dir write */
-	.create      = afr_create,
-	.mknod       = afr_mknod,
-	.mkdir       = afr_mkdir,
-	.unlink      = afr_unlink,
-	.rmdir       = afr_rmdir,
-	.link        = afr_link,
-	.symlink     = afr_symlink,
-	.rename      = afr_rename,
+        .fsetattr    = afr_fsetattr,
+        .removexattr = afr_removexattr,
+        .fremovexattr = afr_fremovexattr,
+
+        /* dir read */
+        .opendir     = afr_opendir,
+        .readdir     = afr_readdir,
+        .readdirp    = afr_readdirp,
+
+        /* dir write */
+        .create      = afr_create,
+        .mknod       = afr_mknod,
+        .mkdir       = afr_mkdir,
+        .unlink      = afr_unlink,
+        .rmdir       = afr_rmdir,
+        .link        = afr_link,
+        .symlink     = afr_symlink,
+        .rename      = afr_rename,
 };
 
 
@@ -952,62 +503,247 @@ struct xlator_dumpops dumpops = {
 
 
 struct xlator_cbks cbks = {
-	.release     = afr_release,
-	.releasedir  = afr_releasedir,
+        .release     = afr_release,
+        .releasedir  = afr_releasedir,
+        .forget      = afr_forget,
 };
 
 
 struct volume_options options[] = {
-	{ .key  = {"read-subvolume" }, 
-	  .type = GF_OPTION_TYPE_XLATOR
-	},
-	{ .key  = {"favorite-child"}, 
-	  .type = GF_OPTION_TYPE_XLATOR
-	},
+        { .key  = {"read-subvolume" },
+          .type = GF_OPTION_TYPE_XLATOR,
+          .description = "inode-read fops happen only on one of the bricks in "
+                         "replicate. Afr will prefer the one specified using "
+                         "this option if it is not stale. Option value must be "
+                         "one of the xlator names of the children. "
+                         "Ex: <volname>-client-0 till "
+                         "<volname>-client-<number-of-bricks - 1>"
+        },
+        { .key  = {"read-subvolume-index" },
+          .type = GF_OPTION_TYPE_INT,
+          .default_value = "-1",
+          .description = "inode-read fops happen only on one of the bricks in "
+                         "replicate. AFR will prefer the one specified using "
+                         "this option if it is not stale. allowed options"
+                         " include -1 till replica-count - 1"
+        },
+        { .key = {"read-hash-mode" },
+          .type = GF_OPTION_TYPE_INT,
+          .min = 0,
+          .max = 2,
+          .default_value = "1",
+          .description = "inode-read fops happen only on one of the bricks in "
+                         "replicate. AFR will prefer the one computed using "
+                         "the method specified using this option"
+                         "0 = first up server, "
+                         "1 = hash by GFID of file (all clients use "
+                                                    "same subvolume), "
+                         "2 = hash by GFID of file and client PID",
+        },
+        { .key  = {"choose-local" },
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "true",
+          .description = "Choose a local subvolume (i.e. Brick) to read from"
+	                 " if read-subvolume is not explicitly set.",
+        },
+        { .key  = {"favorite-child"},
+          .type = GF_OPTION_TYPE_XLATOR,
+          .description = "If a split-brain happens choose subvol/brick set by "
+                         "this option as source."
+        },
         { .key  = {"background-self-heal-count"},
           .type = GF_OPTION_TYPE_INT,
-          .min  = 0
+          .min  = 0,
+          .default_value = "16",
+          .validate = GF_OPT_VALIDATE_MIN,
+          .description = "This specifies the number of self-heals that can be "
+                         " performed in background without blocking the fop"
+        },
+        { .key  = {"data-self-heal"},
+          .type = GF_OPTION_TYPE_STR,
+          .value = {"1", "on", "yes", "true", "enable",
+                    "0", "off", "no", "false", "disable",
+                    "open"},
+          .default_value = "on",
+          .description   = "Using this option we can enable/disable data "
+                           "self-heal on the file. \"open\" means data "
+                           "self-heal action will only be triggered by file "
+                           "open operations."
         },
-	{ .key  = {"data-self-heal"},  
-	  .type = GF_OPTION_TYPE_BOOL 
-	},
         { .key  = {"data-self-heal-algorithm"},
-          .type = GF_OPTION_TYPE_STR
+          .type = GF_OPTION_TYPE_STR,
+          .description   = "Select between \"full\", \"diff\". The "
+                           "\"full\" algorithm copies the entire file from "
+                           "source to sink. The \"diff\" algorithm copies to "
+                           "sink only those blocks whose checksums don't match "
+                           "with those of source. If no option is configured "
+                           "the option is chosen dynamically as follows: "
+                           "If the file does not exist on one of the sinks "
+                           "or empty file exists or if the source file size is "
+                           "about the same as page size the entire file will "
+                           "be read and written i.e \"full\" algo, "
+                           "otherwise \"diff\" algo is chosen.",
+          .value = { "diff", "full"}
         },
         { .key  = {"data-self-heal-window-size"},
           .type = GF_OPTION_TYPE_INT,
           .min  = 1,
-          .max  = 1024
+          .max  = 1024,
+          .default_value = "1",
+          .description = "Maximum number blocks per file for which self-heal "
+                         "process would be applied simultaneously."
         },
-	{ .key  = {"metadata-self-heal"},  
-	  .type = GF_OPTION_TYPE_BOOL
-	},
-	{ .key  = {"entry-self-heal"},  
-	  .type = GF_OPTION_TYPE_BOOL 
-	},
-	{ .key  = {"data-change-log"},  
-	  .type = GF_OPTION_TYPE_BOOL 
-	},
-	{ .key  = {"metadata-change-log"},  
-	  .type = GF_OPTION_TYPE_BOOL
-	},
-	{ .key  = {"entry-change-log"},  
-	  .type = GF_OPTION_TYPE_BOOL
-	},
-	{ .key  = {"data-lock-server-count"},  
-	  .type = GF_OPTION_TYPE_INT, 
-	  .min  = 0
-	},
-	{ .key  = {"metadata-lock-server-count"},  
-	  .type = GF_OPTION_TYPE_INT, 
-	  .min  = 0
+        { .key  = {"metadata-self-heal"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "Using this option we can enable/disable metadata "
+                         "i.e. Permissions, ownerships, xattrs self-heal on "
+                         "the file/directory."
+        },
+        { .key  = {"entry-self-heal"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "Using this option we can enable/disable entry "
+                         "self-heal on the directory."
+        },
+        { .key  = {"data-change-log"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "Data fops like write/truncate will not perform "
+                         "pre/post fop changelog operations in afr transaction "
+                         "if this option is disabled"
+        },
+        { .key  = {"metadata-change-log"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "Metadata fops like setattr/setxattr will not perform "
+                         "pre/post fop changelog operations in afr transaction "
+                         "if this option is disabled"
+        },
+        { .key  = {"entry-change-log"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "Entry fops like create/unlink will not perform "
+                         "pre/post fop changelog operations in afr transaction "
+                         "if this option is disabled"
+        },
+        { .key  = {"optimistic-change-log"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "Entry/Metadata fops will not perform "
+                         "pre fop changelog operations in afr transaction "
+                         "if this option is enabled."
+        },
+        { .key = {"inodelk-trace"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "Enabling this option logs inode lock/unlocks"
+        },
+        { .key = {"entrylk-trace"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "Enabling this option logs entry lock/unlocks"
+        },
+	{ .key = {"pre-op-compat"},
+	  .type = GF_OPTION_TYPE_BOOL,
+	  .default_value = "on",
+	  .description = "Use separate pre-op xattrop() FOP rather than "
+	                 "overloading xdata of the OP"
 	},
-	{ .key  = {"entry-lock-server-count"},  
-	  .type = GF_OPTION_TYPE_INT,
-	  .min  = 0
+        { .key = {"eager-lock"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "Lock phase of a transaction has two sub-phases. "
+                         "First is an attempt to acquire locks in parallel by "
+                         "broadcasting non-blocking lock requests. If lock "
+                         "acquisition fails on any server, then the held locks "
+                         "are unlocked and revert to a blocking locked mode "
+                         "sequentially on one server after another.  If this "
+                         "option is enabled the initial broadcasting lock "
+                         "request attempt to acquire lock on the entire file. "
+                         "If this fails, we revert back to the sequential "
+                         "\"regional\" blocking lock as before. In the case "
+                         "where such an \"eager\" lock is granted in the "
+                         "non-blocking phase, it gives rise to an opportunity "
+                         "for optimization. i.e, if the next write transaction "
+                         "on the same FD arrives before the unlock phase of "
+                         "the first transaction, it \"takes over\" the full "
+                         "file lock. Similarly if yet another data transaction "
+                         "arrives before the unlock phase of the \"optimized\" "
+                         "transaction, that in turn \"takes over\" the lock as "
+                         "well. The actual unlock now happens at the end of "
+                         "the last \"optimized\" transaction."
+
+        },
+        { .key = {"self-heal-daemon"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "This option applies to only self-heal-daemon. "
+                         "Index directory crawl and automatic healing of files "
+                         "will not be performed if this option is turned off."
+        },
+        { .key = {"iam-self-heal-daemon"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option differentiates if the replicate "
+                         "translator is running as part of self-heal-daemon "
+                         "or not."
+        },
+        { .key = {"quorum-type"},
+          .type = GF_OPTION_TYPE_STR,
+          .value = { "none", "auto", "fixed"},
+          .default_value = "none",
+          .description = "If value is \"fixed\" only allow writes if "
+                         "quorum-count bricks are present.  If value is "
+                         "\"auto\" only allow writes if more than half of "
+                         "bricks, or exactly half including the first, are "
+                         "present.",
+        },
+        { .key = {"quorum-count"},
+          .type = GF_OPTION_TYPE_INT,
+          .min = 1,
+          .max = INT_MAX,
+          .default_value = 0,
+          .description = "If quorum-type is \"fixed\" only allow writes if "
+                         "this many bricks or present.  Other quorum types "
+                         "will OVERWRITE this value.",
+        },
+        { .key  = {"node-uuid"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "Local glusterd uuid string, used in starting "
+                         "self-heal-daemon so that it can crawl only on "
+                         "local index directories.",
+        },
+        { .key  = {"post-op-delay-secs"},
+          .type = GF_OPTION_TYPE_INT,
+          .min  = 0,
+          .max  = INT_MAX,
+          .default_value = "1",
+          .description = "Time interval induced artificially before "
+	                 "post-operation phase of the transaction to "
+                         "enhance overlap of adjacent write operations.",
+        },
+        { .key = {AFR_SH_READDIR_SIZE_KEY},
+          .type = GF_OPTION_TYPE_SIZET,
+          .description = "readdirp size for performing entry self-heal",
+          .min = 1024,
+          .max = 131072,
+          .default_value = "1KB",
+        },
+        { .key = {"ensure-durability"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .description = "Afr performs fsyncs for transactions if this "
+                         "option is on to make sure the changelogs/data is "
+                         "written to the disk",
+          .default_value = "on",
+        },
+	{ .key = {"afr-dirty-xattr"},
+	  .type = GF_OPTION_TYPE_STR,
+	  .default_value = AFR_DIRTY_DEFAULT,
 	},
-	{ .key  = {"strict-readdir"},
+	{ .key = {"metadata-splitbrain-forced-heal"},
 	  .type = GF_OPTION_TYPE_BOOL,
+	  .default_value = "off",
 	},
-	{ .key  = {NULL} },
+        { .key  = {NULL} },
 };
diff --git a/xlators/cluster/afr/src/afr.h b/xlators/cluster/afr/src/afr.h
index 9593fec3c..36042f7b2 100644
--- a/xlators/cluster/afr/src/afr.h
+++ b/xlators/cluster/afr/src/afr.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -30,153 +21,115 @@
 #include "compat-errno.h"
 #include "afr-mem-types.h"
 
+#include "libxlator.h"
+#include "timer.h"
+#include "syncop.h"
+
+#include "afr-self-heald.h"
+
 #define AFR_XATTR_PREFIX "trusted.afr"
+#define AFR_PATHINFO_HEADER "REPLICATE:"
+#define AFR_SH_READDIR_SIZE_KEY "self-heal-readdir-size"
+#define AFR_SH_DATA_DOMAIN_FMT "%s:self-heal"
+#define AFR_DIRTY_DEFAULT AFR_XATTR_PREFIX ".dirty"
+#define AFR_DIRTY (((afr_private_t *) (THIS->private))->afr_dirty)
 
-struct _pump_private;
+#define AFR_LOCKEE_COUNT_MAX    3
+#define AFR_DOM_COUNT_MAX    3
+#define AFR_NUM_CHANGE_LOGS            3 /*data + metadata + entry*/
 
-typedef struct _afr_private {
-	gf_lock_t lock;               /* to guard access to child_count, etc */
-	unsigned int child_count;     /* total number of children   */
+typedef int (*afr_lock_cbk_t) (call_frame_t *frame, xlator_t *this);
+
+typedef int (*afr_read_txn_wind_t) (call_frame_t *frame, xlator_t *this, int subvol);
 
-        unsigned int read_child_rr;   /* round-robin index of the read_child */
-        gf_lock_t read_child_lock;    /* lock to protect above */
+typedef int (*afr_inode_refresh_cbk_t) (call_frame_t *frame, xlator_t *this, int err);
+
+typedef int (*afr_changelog_resume_t) (call_frame_t *frame, xlator_t *this);
+
+#define alloca0(size) ({void *__ptr; __ptr = alloca(size); memset(__ptr, 0, size); __ptr;})
+#define AFR_COUNT(array,max) ({int __i; int __res = 0; for (__i = 0; __i < max; __i++) if (array[__i]) __res++; __res;})
+#define AFR_INTERSECT(dst,src1,src2,max) ({int __i; for (__i = 0; __i < max; __i++) dst[__i] = src1[__i] && src2[__i];})
+
+typedef struct _afr_private {
+        gf_lock_t lock;               /* to guard access to child_count, etc */
+        unsigned int child_count;     /* total number of children   */
 
-	xlator_t **children;
+        xlator_t **children;
 
-        gf_lock_t root_inode_lk;
-        int first_lookup;
         inode_t *root_inode;
 
-	unsigned char *child_up;
+        unsigned char *child_up;
 
         char **pending_key;
 
-	gf_boolean_t data_self_heal;              /* on/off */
+        char         *data_self_heal;              /* on/off/open */
         char *       data_self_heal_algorithm;    /* name of algorithm */
         unsigned int data_self_heal_window_size;  /* max number of pipelined
                                                      read/writes */
 
         unsigned int background_self_heal_count;
         unsigned int background_self_heals_started;
-	gf_boolean_t metadata_self_heal;   /* on/off */
-	gf_boolean_t entry_self_heal;      /* on/off */
+        gf_boolean_t metadata_self_heal;   /* on/off */
+        gf_boolean_t entry_self_heal;      /* on/off */
 
-	gf_boolean_t data_change_log;       /* on/off */
-	gf_boolean_t metadata_change_log;   /* on/off */
-	gf_boolean_t entry_change_log;      /* on/off */
+        gf_boolean_t data_change_log;       /* on/off */
+        gf_boolean_t metadata_change_log;   /* on/off */
+        gf_boolean_t entry_change_log;      /* on/off */
 
-	int read_child;               /* read-subvolume */
-	unsigned int favorite_child;  /* subvolume to be preferred in resolving
-					 split-brain cases */
+	gf_boolean_t metadata_splitbrain_forced_heal; /* on/off */
+        int read_child;               /* read-subvolume */
+        unsigned int hash_mode;       /* for when read_child is not set */
+        int favorite_child;  /* subvolume to be preferred in resolving
+                                         split-brain cases */
 
-	unsigned int data_lock_server_count;
-	unsigned int metadata_lock_server_count;
-	unsigned int entry_lock_server_count;
+        gf_boolean_t inodelk_trace;
+        gf_boolean_t entrylk_trace;
 
-	gf_boolean_t inodelk_trace;
-	gf_boolean_t entrylk_trace;
-
-	gf_boolean_t strict_readdir;
-
-	unsigned int wait_count;      /* # of servers to wait for success */
+        unsigned int wait_count;      /* # of servers to wait for success */
 
         uint64_t up_count;      /* number of CHILD_UPs we have seen */
         uint64_t down_count;    /* number of CHILD_DOWNs we have seen */
 
-	struct _pump_private *pump_private; /* Set if we are loaded as pump */
-        int                   use_afr_in_pump;
-
-        pthread_mutex_t  mutex;
-        struct list_head saved_fds;   /* list of fds on which locks have succeeded */
-} afr_private_t;
-
-typedef struct {
-        /* External interface: These are variables (some optional) that
-           are set by whoever has triggered self-heal */
-
-        gf_boolean_t need_data_self_heal;
-        gf_boolean_t need_metadata_self_heal;
-        gf_boolean_t need_entry_self_heal;
-
-        gf_boolean_t forced_merge;        /* Is this a self-heal triggered to
-                                             forcibly merge the directories? */
-
-        gf_boolean_t healing_fd_opened;   /* true if caller has already
-                                             opened fd */
-
-        gf_boolean_t data_lock_held;      /* true if caller has already
-                                             acquired 0-0 lock */
-
-	fd_t *healing_fd;                 /* set if callers has opened fd */
-
-        gf_boolean_t background;          /* do self-heal in background
-                                             if possible */
-
-        ia_type_t type;                   /* st_mode of the entry we're doing
-                                             self-heal on */
-
-        /* Function to call to unwind. If self-heal is being done in the
-           background, this function will be called as soon as possible. */
-
-        int (*unwind) (call_frame_t *frame, xlator_t *this);
-
-        /* End of external interface members */
-
-
-	/* array of stat's, one for each child */
-	struct iatt *buf;
-        struct iatt parentbuf;
-
-	/* array of xattr's, one for each child */
-	dict_t **xattr;
-
-	/* array of errno's, one for each child */
-	int *child_errno;
-
-	int32_t **pending_matrix;
-	int32_t **delta_matrix;
-
-	int *sources;
-	int source;
-	int active_source;
-	int active_sinks;
-	int *success;
-	unsigned char *locked_nodes;
-        int lock_count;
-
-        mode_t impunging_entry_mode;
-        const char *linkname;
-
-	int   op_failed;
-
-	int   file_has_holes;
-	blksize_t block_size;
-	off_t file_size;
-	off_t offset;
-
-	loc_t parent_loc;
-
-        call_frame_t *orig_frame;
-        gf_boolean_t unwound;
-
-        /* private data for the particular self-heal algorithm */
-        void *private;
+        gf_boolean_t      optimistic_change_log;
+        gf_boolean_t      eager_lock;
+        gf_boolean_t      pre_op_compat;      /* on/off */
+	uint32_t          post_op_delay_secs;
+        unsigned int      quorum_count;
+
+        char                   vol_uuid[UUID_SIZE + 1];
+        int32_t                *last_event;
+
+	/* @event_generation: Keeps count of number of events received which can
+	   potentially impact consistency decisions. The events are CHILD_UP
+	   and CHILD_DOWN, when we have to recalculate the freshness/staleness
+	   of copies to detect if changes had happened while the other server
+	   was down. CHILD_DOWN and CHILD_UP can also be received on network
+	   disconnect/reconnects and not necessarily server going down/up.
+	   Recalculating freshness/staleness on network events is equally
+	   important as we might have had a network split brain.
+	*/
+	uint32_t               event_generation;
 
-        int (*flush_self_heal_cbk) (call_frame_t *frame, xlator_t *this);
+        gf_boolean_t           choose_local;
+        gf_boolean_t           did_discovery;
+        uint64_t               sh_readdir_size;
+        gf_boolean_t           ensure_durability;
+        char                   *sh_domain;
+	char                   *afr_dirty;
 
-	int (*completion_cbk) (call_frame_t *frame, xlator_t *this);
-        int (*algo_completion_cbk) (call_frame_t *frame, xlator_t *this);
-        int (*algo_abort_cbk) (call_frame_t *frame, xlator_t *this);
+	afr_self_heald_t       shd;
 
-	call_frame_t *sh_frame;
-} afr_self_heal_t;
+	/* pump dependencies */
+	void                   *pump_private;
+	gf_boolean_t           use_afr_in_pump;
+} afr_private_t;
 
 
 typedef enum {
-	AFR_DATA_TRANSACTION,          /* truncate, write, ... */
-	AFR_METADATA_TRANSACTION,      /* chmod, chown, ... */
-	AFR_ENTRY_TRANSACTION,         /* create, rmdir, ... */
-	AFR_ENTRY_RENAME_TRANSACTION,  /* rename */
+        AFR_DATA_TRANSACTION,          /* truncate, write, ... */
+        AFR_METADATA_TRANSACTION,      /* chmod, chown, ... */
+        AFR_ENTRY_TRANSACTION,         /* create, rmdir, ... */
+        AFR_ENTRY_RENAME_TRANSACTION,  /* rename */
 } afr_transaction_type;
 
 typedef enum {
@@ -232,11 +185,31 @@ afr_index_for_transaction_type (afr_transaction_type type)
         return -1;  /* make gcc happy */
 }
 
+typedef struct {
+        loc_t                   loc;
+        char                    *basename;
+        unsigned char           *locked_nodes;
+        int                     locked_count;
+
+} afr_entry_lockee_t;
+
+int
+afr_entry_lockee_cmp (const void *l1, const void *l2);
+
+typedef struct {
+        char    *domain; /* Domain on which inodelk is taken */
+        struct gf_flock flock;
+        unsigned char *locked_nodes;
+        int32_t lock_count;
+} afr_inodelk_t;
 
 typedef struct {
         loc_t *lk_loc;
-        struct gf_flock lk_flock;
 
+        int                     lockee_count;
+        afr_entry_lockee_t      lockee[AFR_LOCKEE_COUNT_MAX];
+
+        afr_inodelk_t       inodelk[AFR_DOM_COUNT_MAX];
         const char *lk_basename;
         const char *lower_basename;
         const char *higher_basename;
@@ -245,451 +218,564 @@ typedef struct {
 
         unsigned char *locked_nodes;
         unsigned char *lower_locked_nodes;
-        unsigned char *inode_locked_nodes;
-        unsigned char *entry_locked_nodes;
 
         selfheal_lk_type_t selfheal_lk_type;
         transaction_lk_type_t transaction_lk_type;
 
         int32_t lock_count;
-        int32_t inodelk_lock_count;
         int32_t entrylk_lock_count;
 
         uint64_t lock_number;
         int32_t lk_call_count;
+        int32_t lk_expected_count;
+        int32_t lk_attempted_count;
 
         int32_t lock_op_ret;
         int32_t lock_op_errno;
+        afr_lock_cbk_t lock_cbk;
+        char *domain; /* Domain on which inode/entry lock/unlock in progress.*/
+} afr_internal_lock_t;
 
-        int (*lock_cbk) (call_frame_t *, xlator_t *);
+struct afr_reply {
+	int	valid;
+	int32_t	op_ret;
+	int32_t	op_errno;
+	dict_t *xdata;
+	struct iatt poststat;
+	struct iatt postparent;
+	struct iatt prestat;
+	struct iatt preparent;
+	struct iatt preparent2;
+	struct iatt postparent2;
+	uint8_t checksum[MD5_DIGEST_LENGTH];
+};
 
-} afr_internal_lock_t;
+typedef enum {
+        AFR_FD_NOT_OPENED,
+        AFR_FD_OPENED,
+        AFR_FD_OPENING
+} afr_fd_open_status_t;
+
+typedef struct {
+        unsigned int *pre_op_done[AFR_NUM_CHANGE_LOGS];
+	int inherited[AFR_NUM_CHANGE_LOGS];
+	int on_disk[AFR_NUM_CHANGE_LOGS];
+        afr_fd_open_status_t *opened_on; /* which subvolumes the fd is open on */
+
+        unsigned int *lock_piggyback;
+        unsigned int *lock_acquired;
+
+        int flags;
+
+	/* used for delayed-post-op optimization */
+	pthread_mutex_t    delay_lock;
+	gf_timer_t        *delay_timer;
+	call_frame_t      *delay_frame;
+
+	/* set if any write on this fd was a non stable write
+	   (i.e, without O_SYNC or O_DSYNC)
+	*/
+	gf_boolean_t      witnessed_unstable_write;
+
+	/* @open_fd_count:
+	   Number of open FDs queried from the server, as queried through
+	   xdata in FOPs. Currently, used to decide if eager-locking must be
+	   temporarily disabled.
+	*/
+        uint32_t        open_fd_count;
+
+
+	/* list of frames currently in progress */
+	struct list_head  eager_locked;
+} afr_fd_ctx_t;
 
-typedef struct _afr_locked_fd {
-        fd_t  *fd;
-        struct list_head list;
-} afr_locked_fd_t;
 
 typedef struct _afr_local {
-	unsigned int call_count;
-	unsigned int success_count;
-	unsigned int enoent_count;
+	glusterfs_fop_t  op;
+        unsigned int call_count;
 
+	/* @event_generation: copy of priv->event_generation taken at the
+	   time of starting the transaction. The copy is made so that we
+	   have a stable value through the various phases of the transaction.
+	*/
+	unsigned int event_generation;
 
-	unsigned int govinda_gOvinda;
+        uint32_t     open_fd_count;
+        gf_boolean_t update_open_fd_count;
 
-	unsigned int read_child_index;
-        unsigned char read_child_returned;
-        unsigned int first_up_child;
+	gf_lkowner_t  saved_lk_owner;
 
-        pid_t saved_pid;
+        int32_t op_ret;
+        int32_t op_errno;
 
-	int32_t op_ret;
-	int32_t op_errno;
+        int32_t **pending;
 
-	int32_t **pending;
+	int dirty[AFR_NUM_CHANGE_LOGS];
 
-	loc_t loc;
-	loc_t newloc;
+        loc_t loc;
+        loc_t newloc;
 
-	fd_t *fd;
+        fd_t *fd;
+	afr_fd_ctx_t *fd_ctx;
 
-	glusterfs_fop_t fop;
+	/* @child_up: copy of priv->child_up taken at the time of transaction
+	   start. The copy is taken so that we have a stable child_up array
+	   through the phases of the transaction as priv->child_up[i] can keep
+	   changing through time.
+	*/
+        unsigned char *child_up;
+
+	/* @read_attempted:
+	   array of flags representing subvolumes where read operations of
+	   the read transaction have already been attempted. The array is
+	   first pre-filled with down subvolumes, and as reads are performed
+	   on other subvolumes, those are set as well. This way if the read
+	   operation fails we do not retry on that subvolume again.
+	*/
+	unsigned char *read_attempted;
 
-	unsigned char *child_up;
+	/* @readfn:
 
-	int32_t *child_errno;
+	   pointer to function which will perform the read operation on a given
+	   subvolume. Used in read transactions.
+	*/
 
-	dict_t  *xattr_req;
+	afr_read_txn_wind_t readfn;
 
-	int32_t  inodelk_count;
-	int32_t  entrylk_count;
+	/* @refreshed:
 
-        afr_internal_lock_t internal_lock;
+	   the inode was "refreshed" (i.e, pending xattrs from all subvols
+	   freshly inspected and inode ctx updated accordingly) as part of
+	   this transaction already.
+	*/
+	gf_boolean_t refreshed;
 
-        afr_locked_fd_t *locked_fd;
-        int32_t          source_child;
-        int32_t          lock_recovery_child;
+	/* @inode:
 
-        dict_t  *dict;
+	   the inode on which the read txn is performed on. ref'ed and copied
+	   from either fd->inode or loc.inode
+	*/
 
-        int (*openfd_flush_cbk) (call_frame_t *frame, xlator_t *this);
+	inode_t *inode;
 
-	/*
-	   This struct contains the arguments for the "continuation"
-	   (scheme-like) of fops
+	/* @parent[2]:
+
+	   parent inode[s] on which directory transactions are performed.
 	*/
 
-	int   op;
-	struct {
-		struct {
-			unsigned char buf_set;
-			struct statvfs buf;
-		} statfs;
+	inode_t *parent;
+	inode_t *parent2;
 
-		struct {
-			inode_t *inode;
-			struct iatt buf;
-                        struct iatt read_child_buf;
-                        struct iatt postparent;
-                        ino_t ino;
-                        uint64_t gen;
-                        ino_t parent_ino;
-			dict_t *xattr;
-			dict_t **xattrs;
-                        gf_boolean_t is_revalidate;
-		} lookup;
+	/* @readable:
 
-		struct {
-			int32_t flags;
-                        int32_t wbflags;
-		} open;
+	   array of flags representing servers from which a read can be
+	   performed. This is the output of afr_inode_refresh()
+	*/
+	unsigned char *readable;
 
-		struct {
-			int32_t cmd;
-                        struct gf_flock user_flock;
-                        struct gf_flock ret_flock;
-			unsigned char *locked_nodes;
-		} lk;
+	afr_inode_refresh_cbk_t refreshfn;
 
-		/* inode read */
+	/* @refreshinode:
 
-		struct {
-			int32_t mask;
-			int last_tried;  /* index of the child we tried previously */
-		} access;
+	   Inode currently getting refreshed.
+	*/
+	inode_t *refreshinode;
 
-		struct {
-			int last_tried;
-			ino_t ino;
-		} stat;
+	/*
+	  @pre_op_compat:
 
-		struct {
-			int last_tried;
-			ino_t ino;
-		} fstat;
+	  compatibility mode of pre-op. send a separate pre-op and
+	  op operations as part of transaction, rather than combining
+	*/
 
-		struct {
-			size_t size;
-			int last_tried;
-                        ino_t ino;
-		} readlink;
+	gf_boolean_t pre_op_compat;
 
-		struct {
-			char *name;
-			int last_tried;
-		} getxattr;
+        dict_t  *xattr_req;
 
-		struct {
-                        ino_t ino;
-			size_t size;
-			off_t offset;
-			int last_tried;
-		} readv;
+        afr_internal_lock_t internal_lock;
 
-		/* dir read */
+        dict_t  *dict;
 
-		struct {
-			int success_count;
-			int32_t op_ret;
-			int32_t op_errno;
+        int      optimistic_change_log;
+	gf_boolean_t      delayed_post_op;
 
-                        uint32_t *checksum;
-		} opendir;
+	/* Is the current writev() going to perform a stable write?
+	   i.e, is fd->flags or @flags writev param have O_SYNC or
+	   O_DSYNC?
+	*/
+	gf_boolean_t      stable_write;
 
-		struct {
-			int32_t op_ret;
-			int32_t op_errno;
-			size_t size;
-			off_t offset;
+	/* This write appended to the file. Nnot necessarily O_APPEND,
+	   just means the offset of write was at the end of file.
+	*/
+	gf_boolean_t      append_write;
 
-                        gf_boolean_t failed;
-			int last_tried;
-		} readdir;
+        /*
+          This struct contains the arguments for the "continuation"
+          (scheme-like) of fops
+        */
 
-		struct {
-			int32_t op_ret;
-			int32_t op_errno;
+        struct {
+                struct {
+                        unsigned char buf_set;
+                        struct statvfs buf;
+                } statfs;
 
-			size_t size;
-			off_t offset;
-			int32_t flag;
+                struct {
+                        int32_t flags;
+                } open;
 
-			int last_tried;
-		} getdents;
+                struct {
+                        int32_t cmd;
+                        struct gf_flock user_flock;
+                        struct gf_flock ret_flock;
+                        unsigned char *locked_nodes;
+                } lk;
 
-		/* inode write */
+                /* inode read */
 
-		struct {
-			ino_t ino;
-			struct iatt prebuf;
-			struct iatt postbuf;
+                struct {
+                        int32_t mask;
+                        int last_index;  /* index of the child we tried previously */
+                } access;
 
-			int32_t op_ret;
+                struct {
+                        int last_index;
+                } stat;
 
-			struct iovec *vector;
-			struct iobref *iobref;
-			int32_t count;
-			off_t offset;
-		} writev;
+                struct {
+                        int last_index;
+                } fstat;
+
+                struct {
+                        size_t size;
+                        int last_index;
+                } readlink;
+
+                struct {
+                        char *name;
+                        int last_index;
+                        long xattr_len;
+                } getxattr;
+
+                struct {
+                        size_t size;
+                        off_t offset;
+                        int last_index;
+                        uint32_t flags;
+                } readv;
+
+                /* dir read */
+
+                struct {
+                        int success_count;
+                        int32_t op_ret;
+                        int32_t op_errno;
+
+                        uint32_t *checksum;
+                } opendir;
+
+                struct {
+                        int32_t op_ret;
+                        int32_t op_errno;
+                        size_t size;
+                        off_t offset;
+                        dict_t *dict;
+                        gf_boolean_t failed;
+                        int last_index;
+                } readdir;
+                /* inode write */
 
                 struct {
-                        ino_t ino;
                         struct iatt prebuf;
                         struct iatt postbuf;
-                } fsync;
+                } inode_wfop; //common structure for all inode-write-fops
 
-		struct {
-			ino_t ino;
-			off_t offset;
-			struct iatt prebuf;
-                        struct iatt postbuf;
-		} truncate;
+                struct {
+                        int32_t op_ret;
 
-		struct {
-			ino_t ino;
-			off_t offset;
-			struct iatt prebuf;
-                        struct iatt postbuf;
-		} ftruncate;
+                        struct iovec *vector;
+                        struct iobref *iobref;
+                        int32_t count;
+                        off_t offset;
+                        uint32_t flags;
+                } writev;
 
-		struct {
-			ino_t ino;
-			struct iatt in_buf;
+                struct {
+                        off_t offset;
+                } truncate;
+
+                struct {
+                        off_t offset;
+                } ftruncate;
+
+                struct {
+                        struct iatt in_buf;
                         int32_t valid;
-                        struct iatt preop_buf;
-                        struct iatt postop_buf;
-		} setattr;
+                } setattr;
 
-		struct {
-			ino_t ino;
-			struct iatt in_buf;
+                struct {
+                        struct iatt in_buf;
                         int32_t valid;
-                        struct iatt preop_buf;
-                        struct iatt postop_buf;
-		} fsetattr;
+                } fsetattr;
 
-		struct {
-			dict_t *dict;
-			int32_t flags;
-		} setxattr;
+                struct {
+                        dict_t *dict;
+                        int32_t flags;
+                } setxattr;
 
-		struct {
-			char *name;
-		} removexattr;
+                struct {
+                        dict_t *dict;
+                        int32_t flags;
+                } fsetxattr;
 
-		/* dir write */
+                struct {
+                        char *name;
+                } removexattr;
 
-		struct {
-			ino_t ino;
-                        uint64_t gen;
-                        ino_t parent_ino;
-			fd_t *fd;
-                        dict_t *params;
-			int32_t flags;
-			mode_t mode;
-			inode_t *inode;
-			struct iatt buf;
+                struct {
+                        dict_t *xattr;
+                } xattrop;
+
+                struct {
+                        dict_t *xattr;
+                } fxattrop;
+
+                /* dir write */
+
+                struct {
+                        inode_t *inode;
+                        struct iatt buf;
                         struct iatt preparent;
                         struct iatt postparent;
-                        struct iatt read_child_buf;
-		} create;
+                        struct iatt prenewparent;
+                        struct iatt postnewparent;
+                } dir_fop; //common structure for all dir fops
 
-		struct {
-			ino_t ino;
-                        uint64_t gen;
-                        ino_t parent_ino;
-			dev_t dev;
-			mode_t mode;
+                struct {
+                        fd_t *fd;
                         dict_t *params;
-			inode_t *inode;
-			struct iatt buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-                        struct iatt read_child_buf;
-		} mknod;
+                        int32_t flags;
+                        mode_t mode;
+                } create;
 
-		struct {
-			ino_t ino;
-                        uint64_t gen;
-                        ino_t parent_ino;
-			int32_t mode;
+                struct {
+                        dev_t dev;
+                        mode_t mode;
                         dict_t *params;
-			inode_t *inode;
-			struct iatt buf;
-                        struct iatt read_child_buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-		} mkdir;
+                } mknod;
 
-		struct {
-                        ino_t parent_ino;
-			int32_t op_ret;
-			int32_t op_errno;
-                        struct iatt preparent;
-                        struct iatt postparent;
-		} unlink;
+                struct {
+                        int32_t mode;
+                        dict_t *params;
+                } mkdir;
 
-		struct {
-                        int   flags;
-                        ino_t parent_ino;
-			int32_t op_ret;
-			int32_t op_errno;
-                        struct iatt preparent;
-                        struct iatt postparent;
-		} rmdir;
+                struct {
+                        int flags;
+                } rmdir;
 
-		struct {
-                        ino_t oldparent_ino;
-                        ino_t newparent_ino;
-			ino_t ino;
-			struct iatt buf;
-                        struct iatt read_child_buf;
-                        struct iatt preoldparent;
-                        struct iatt prenewparent;
-                        struct iatt postoldparent;
-                        struct iatt postnewparent;
-		} rename;
+                struct {
+                        dict_t *params;
+                        char *linkpath;
+                } symlink;
 
 		struct {
-			ino_t ino;
-                        uint64_t gen;
-                        ino_t parent_ino;
-			inode_t *inode;
-			struct iatt buf;
-                        struct iatt read_child_buf;
-                        struct iatt preparent;
-                        struct iatt postparent;
-		} link;
+			int32_t mode;
+			off_t offset;
+			size_t len;
+		} fallocate;
 
 		struct {
-			ino_t ino;
-                        uint64_t gen;
-                        ino_t parent_ino;
-			inode_t *inode;
-                        dict_t *params;
-			struct iatt buf;
-                        struct iatt read_child_buf;
-			char *linkpath;
-                        struct iatt preparent;
-                        struct iatt postparent;
-		} symlink;
+			off_t offset;
+			size_t len;
+		} discard;
 
-		struct {
-			int32_t flags;
-			dir_entry_t *entries;
-			int32_t count;
-		} setdents;
-	} cont;
+                struct {
+                        off_t offset;
+                        off_t len;
+                        struct iatt prebuf;
+                        struct iatt postbuf;
+                } zerofill;
 
-	struct {
-		off_t start, len;
 
-		char *basename;
-		char *new_basename;
+        } cont;
 
-		loc_t parent_loc;
-		loc_t new_parent_loc;
+        struct {
+                off_t start, len;
 
-		afr_transaction_type type;
+                gf_boolean_t    eager_lock_on;
+                int *eager_lock;
 
-		int success_count;
-		int erase_pending;
-		int failure_count;
+                char *basename;
+                char *new_basename;
 
-		int last_tried;
-		int32_t *child_errno;
+                loc_t parent_loc;
+                loc_t new_parent_loc;
 
-		call_frame_t *main_frame;
+                afr_transaction_type type;
 
-		int (*fop) (call_frame_t *frame, xlator_t *this);
+		/* stub to resume on destruction
+		   of the transaction frame */
+		call_stub_t      *resume_stub;
 
-		int (*done) (call_frame_t *frame, xlator_t *this);
+		struct list_head  eager_locked;
 
-		int (*resume) (call_frame_t *frame, xlator_t *this);
+                unsigned char   *pre_op;
 
-		int (*unwind) (call_frame_t *frame, xlator_t *this);
+		/* @fop_subvols: subvolumes on which FOP will be attempted */
+                unsigned char   *fop_subvols;
 
-                /* post-op hook */
-	} transaction;
+		/* @failed_subvols: subvolumes on which FOP failed. Always
+		   a subset of @fop_subvols */
+                unsigned char   *failed_subvols;
 
-	afr_self_heal_t self_heal;
-} afr_local_t;
+		/* @dirtied: flag which indicates whether we set dirty flag
+		   in the OP. Typically true when we are performing operation
+		   on more than one subvol and optimistic changelog is disabled
 
+		   A 'true' value set in @dirtied flag means an 'undirtying'
+		   has to be done in POST-OP phase.
+		*/
+		gf_boolean_t  dirtied;
 
-typedef struct {
-        unsigned int *pre_op_done;
-        unsigned int *opened_on;     /* which subvolumes the fd is open on */
-        unsigned int *pre_op_piggyback;
+		/* @inherited: flag which indicates that the dirty flags
+		   of the previous transaction were inherited
+		*/
+		gf_boolean_t  inherited;
 
-        int flags;
-        int32_t wbflags;
-        uint64_t up_count;   /* number of CHILD_UPs this fd has seen */
-        uint64_t down_count; /* number of CHILD_DOWNs this fd has seen */
+		/*
+		  @no_uninherit: flag which indicates that a pre_op_uninherit()
+		  must _not_ be attempted (and returned as failure) always. This
+		  flag is set when a hard pre-op is performed, but not accounted
+		  for it in fd_ctx->on_disk[]. Such transactions are "isolated"
+		  from the pre-op piggybacking entirely and therefore uninherit
+		  must not be attempted.
+		*/
+		gf_boolean_t no_uninherit;
 
-        int32_t last_tried;
+		/* @uninherit_done:
+		   @uninherit_value:
 
-        int  hit, miss;
-        gf_boolean_t failed_over;
-        struct list_head entries; /* needed for readdir failover */
+		   The above pair variables make pre_op_uninherit() idempotent.
+		   Both are FALSE initially. The first call to pre_op_uninherit
+		   sets @uninherit_done to TRUE and the return value to
+		   @uninherit_value. Further calls will check for @uninherit_done
+		   to be TRUE and if so will simply return @uninherit_value.
+		*/
+		gf_boolean_t uninherit_done;
+		gf_boolean_t uninherit_value;
 
-        unsigned char *locked_on; /* which subvolumes locks have been successful */
-} afr_fd_ctx_t;
+		/* @changelog_resume: function to be called after changlogging
+		   (either pre-op or post-op) is done
+		*/
+
+		afr_changelog_resume_t changelog_resume;
+
+                call_frame_t *main_frame;
+
+                int (*wind) (call_frame_t *frame, xlator_t *this, int subvol);
+
+                int (*fop) (call_frame_t *frame, xlator_t *this);
 
+                int (*done) (call_frame_t *frame, xlator_t *this);
+
+                int (*resume) (call_frame_t *frame, xlator_t *this);
+
+                int (*unwind) (call_frame_t *frame, xlator_t *this);
+
+                /* post-op hook */
+        } transaction;
 
-/* try alloc and if it fails, goto label */
-#define ALLOC_OR_GOTO(var, type, label) do {                     \
-		var = GF_CALLOC (sizeof (type), 1,               \
-                                 gf_afr_mt_##type);              \
-		if (!var) {                                      \
-			gf_log (this->name, GF_LOG_ERROR,        \
-				"out of memory :(");             \
-			op_errno = ENOMEM;                       \
-			goto label;                              \
-		}                                                \
-	} while (0);
+	syncbarrier_t barrier;
+
+        struct marker_str     marker;
+
+        /* extra data for fops */
+        dict_t         *xdata_req;
+        dict_t         *xdata_rsp;
+
+        mode_t          umask;
+        int             xflag;
+        gf_boolean_t    do_discovery;
+	struct afr_reply *replies;
+} afr_local_t;
 
 
 /* did a call fail due to a child failing? */
-#define child_went_down(op_ret, op_errno) (((op_ret) < 0) &&	      \
-					   ((op_errno == ENOTCONN) || \
-					    (op_errno == EBADFD)))
+#define child_went_down(op_ret, op_errno) (((op_ret) < 0) &&            \
+                                           ((op_errno == ENOTCONN) ||   \
+                                            (op_errno == EBADFD)))
+
+int
+afr_inode_read_subvol_get (inode_t *inode, xlator_t *this,
+			   unsigned char *data_subvols,
+			   unsigned char *metadata_subvols,
+			   int *event_generation);
+int
+__afr_inode_read_subvol_get (inode_t *inode, xlator_t *this,
+			     unsigned char *data_subvols,
+			     unsigned char *metadata_subvols,
+			     int *event_generation);
+
+int
+__afr_inode_read_subvol_set (inode_t *inode, xlator_t *this,
+			     unsigned char *data_subvols,
+			     unsigned char *metadata_subvol,
+			     int event_generation);
+int
+afr_inode_read_subvol_set (inode_t *inode, xlator_t *this,
+			   unsigned char *data_subvols,
+			   unsigned char *metadata_subvols,
+			   int event_generation);
 
-#define afr_fop_failed(op_ret, op_errno) ((op_ret) == -1)
+int
+afr_inode_read_subvol_reset (inode_t *inode, xlator_t *this);
 
-/* have we tried all children? */
-#define all_tried(i, count)  ((i) == (count) - 1)
+int
+afr_read_subvol_select_by_policy (inode_t *inode, xlator_t *this,
+				  unsigned char *readable);
 
-int32_t
-afr_set_dict_gfid (dict_t *dict, uuid_t gfid);
+int
+afr_inode_read_subvol_type_get (inode_t *inode, xlator_t *this,
+				unsigned char *readable, int *event_p,
+				int type);
+int
+afr_read_subvol_get (inode_t *inode, xlator_t *this, int *subvol_p,
+		     int *event_p, afr_transaction_type type);
+
+#define afr_data_subvol_get(i, t, s, e) \
+	afr_read_subvol_get(i, t, s, e, AFR_DATA_TRANSACTION)
+
+#define afr_metadata_subvol_get(i, t, s, e) \
+	afr_read_subvol_get(i, t, s, e, AFR_METADATA_TRANSACTION)
 
 int
-pump_command_reply (call_frame_t *frame, xlator_t *this);
+afr_inode_refresh (call_frame_t *frame, xlator_t *this, inode_t *inode,
+		   afr_inode_refresh_cbk_t cbk);
 
 int32_t
-afr_notify (xlator_t *this, int32_t event,
-            void *data, ...);
+afr_notify (xlator_t *this, int32_t event, void *data, void *data2);
 
 int
-afr_attempt_lock_recovery (xlator_t *this, int32_t child_index);
+afr_init_entry_lockee (afr_entry_lockee_t *lockee, afr_local_t *local,
+                       loc_t *loc, char *basename, int child_count);
+
+void
+afr_entry_lockee_cleanup (afr_internal_lock_t *int_lock);
 
 int
-afr_save_locked_fd (xlator_t *this, fd_t *fd);
+afr_attempt_lock_recovery (xlator_t *this, int32_t child_index);
 
 int
 afr_mark_locked_nodes (xlator_t *this, fd_t *fd,
                        unsigned char *locked_nodes);
 
 void
-afr_set_lk_owner (call_frame_t *frame, xlator_t *this);
+afr_set_lk_owner (call_frame_t *frame, xlator_t *this, void *lk_owner);
 
 int
 afr_set_lock_number (call_frame_t *frame, xlator_t *this);
 
-
-loc_t *
-lower_path (loc_t *l1, const char *b1, loc_t *l2, const char *b2);
-
 int32_t
 afr_unlock (call_frame_t *frame, xlator_t *this);
 
@@ -705,32 +791,30 @@ afr_blocking_lock (call_frame_t *frame, xlator_t *this);
 int
 afr_internal_lock_finish (call_frame_t *frame, xlator_t *this);
 
+int
+afr_lk_transfer_datalock (call_frame_t *dst, call_frame_t *src, char *dom,
+                          unsigned int child_count);
 
-int pump_start (call_frame_t *frame, xlator_t *this);
+int
+__afr_fd_ctx_set (xlator_t *this, fd_t *fd);
 
 int
 afr_fd_ctx_set (xlator_t *this, fd_t *fd);
 
-uint64_t
-afr_read_child (xlator_t *this, inode_t *inode);
-
-void
-afr_set_read_child (xlator_t *this, inode_t *inode, int32_t read_child);
-
-void
-afr_build_parent_loc (loc_t *parent, loc_t *child);
+afr_fd_ctx_t *
+afr_fd_ctx_get (fd_t *fd, xlator_t *this);
 
 int
-afr_up_children_count (int child_count, unsigned char *child_up);
+afr_build_parent_loc (loc_t *parent, loc_t *child, int32_t *op_errno);
 
 int
 afr_locked_nodes_count (unsigned char *locked_nodes, int child_count);
 
-ino64_t
-afr_itransform (ino64_t ino, int child_count, int child_index);
-
 int
-afr_deitransform (ino64_t ino, int child_count);
+afr_replies_interpret (call_frame_t *frame, xlator_t *this, inode_t *inode);
+
+void
+afr_replies_wipe (afr_local_t *local, afr_private_t *priv);
 
 void
 afr_local_cleanup (afr_local_t *local, xlator_t *this);
@@ -738,21 +822,9 @@ afr_local_cleanup (afr_local_t *local, xlator_t *this);
 int
 afr_frame_return (call_frame_t *frame);
 
-uint64_t
-afr_is_split_brain (xlator_t *this, inode_t *inode);
-
-void
-afr_set_split_brain (xlator_t *this, inode_t *inode, gf_boolean_t set);
-
 int
 afr_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-          fd_t *fd, int32_t wbflags);
-
-void
-afr_set_opendir_done (xlator_t *this, inode_t *inode);
-
-uint64_t
-afr_is_opendir_done (xlator_t *this, inode_t *inode);
+          fd_t *fd, dict_t *xdata);
 
 void
 afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this);
@@ -760,163 +832,145 @@ afr_local_transaction_cleanup (afr_local_t *local, xlator_t *this);
 int
 afr_cleanup_fd_ctx (xlator_t *this, fd_t *fd);
 
-int
-afr_openfd_flush (call_frame_t *frame, xlator_t *this, fd_t *fd);
-
-#define AFR_STACK_UNWIND(fop, frame, params ...)        \
-	do {						\
-		afr_local_t *__local = NULL;		\
-		xlator_t    *__this = NULL;		\
-                if (frame) {                            \
-                        __local = frame->local;		\
-                        __this = frame->this;           \
-                        frame->local = NULL;            \
+#define AFR_STACK_UNWIND(fop, frame, params ...)                \
+        do {                                                    \
+                afr_local_t *__local = NULL;                    \
+                xlator_t    *__this = NULL;                     \
+                if (frame) {                                    \
+                        __local = frame->local;                 \
+                        __this = frame->this;                   \
+                        frame->local = NULL;                    \
+                }                                               \
+                STACK_UNWIND_STRICT (fop, frame, params);       \
+                if (__local) {                                  \
+                        afr_local_cleanup (__local, __this);    \
+                        mem_put (__local);                      \
+                }                                               \
+        } while (0)
+
+#define AFR_STACK_DESTROY(frame)                                \
+        do {                                                    \
+                afr_local_t *__local = NULL;                    \
+                xlator_t    *__this = NULL;                     \
+                __local = frame->local;                         \
+                __this = frame->this;                           \
+                frame->local = NULL;                            \
+                STACK_DESTROY (frame->root);                    \
+                if (__local) {                                  \
+                        afr_local_cleanup (__local, __this);    \
+                        mem_put (__local);                      \
                 }                                               \
-		STACK_UNWIND_STRICT (fop, frame, params);       \
-		afr_local_cleanup (__local, __this);            \
-		GF_FREE (__local);				\
         } while (0);
 
-#define AFR_STACK_DESTROY(frame)			\
-	do {						\
-		afr_local_t *__local = NULL;		\
-		xlator_t    *__this = NULL;		\
-		__local = frame->local;			\
-		__this = frame->this;			\
-		frame->local = NULL;                    \
-		STACK_DESTROY (frame->root);		\
-		afr_local_cleanup (__local, __this);	\
-		GF_FREE (__local);			\
-        } while (0);
+#define AFR_FRAME_INIT(frame, op_errno)				       \
+	({frame->local = mem_get0 (THIS->local_pool);		       \
+	if (afr_local_init (frame->local, THIS->private, &op_errno)) { \
+		afr_local_cleanup (frame->local, THIS);		       \
+		mem_put (frame->local);				       \
+		frame->local = NULL; };				       \
+	frame->local;})
+
+#define AFR_STACK_RESET(frame) do { int opr; STACK_RESET (frame->root); AFR_FRAME_INIT(frame, opr);} while (0)
 
 /* allocate and return a string that is the basename of argument */
 static inline char *
 AFR_BASENAME (const char *str)
 {
-	char *__tmp_str = NULL;
-	char *__basename_str = NULL;
-	__tmp_str = gf_strdup (str);
-	__basename_str = gf_strdup (basename (__tmp_str));
-	GF_FREE (__tmp_str);
-	return __basename_str;
+        char *__tmp_str = NULL;
+        char *__basename_str = NULL;
+        __tmp_str = gf_strdup (str);
+        __basename_str = gf_strdup (basename (__tmp_str));
+        GF_FREE (__tmp_str);
+        return __basename_str;
 }
 
-/* initialize local_t */
-static inline int
-AFR_LOCAL_INIT (afr_local_t *local, afr_private_t *priv)
-{
-	local->child_up = GF_CALLOC (sizeof (*local->child_up),
-                                     priv->child_count,
-                                     gf_afr_mt_char);
-	if (!local->child_up) {
-		return -ENOMEM;
-	}
+call_frame_t *
+afr_copy_frame (call_frame_t *base);
 
-	memcpy (local->child_up, priv->child_up,
-		sizeof (*local->child_up) * priv->child_count);
+int
+afr_transaction_local_init (afr_local_t *local, xlator_t *this);
 
+int32_t
+afr_marker_getxattr (call_frame_t *frame, xlator_t *this,
+                     loc_t *loc, const char *name,afr_local_t *local, afr_private_t *priv );
 
-	local->call_count = afr_up_children_count (priv->child_count, local->child_up);
-	if (local->call_count == 0)
-		return -ENOTCONN;
+int
+afr_local_init (afr_local_t *local, afr_private_t *priv, int32_t *op_errno);
 
-	local->transaction.erase_pending = 1;
+int
+afr_internal_lock_init (afr_internal_lock_t *lk, size_t child_count,
+                        transaction_lk_type_t lk_type);
 
-	local->op_ret = -1;
-	local->op_errno = EUCLEAN;
+int
+afr_higher_errno (int32_t old_errno, int32_t new_errno);
 
-        local->internal_lock.lock_op_ret   = -1;
-        local->internal_lock.lock_op_errno = EUCLEAN;
+int
+afr_final_errno (afr_local_t *local, afr_private_t *priv);
 
+int
+afr_xattr_req_prepare (xlator_t *this, dict_t *xattr_req);
 
-	return 0;
-}
+void
+afr_fix_open (fd_t *fd, xlator_t *this);
 
+afr_fd_ctx_t *
+afr_fd_ctx_get (fd_t *fd, xlator_t *this);
 
-/**
- * first_up_child - return the index of the first child that is up
- */
+void
+afr_set_low_priority (call_frame_t *frame);
+int
+afr_child_fd_ctx_set (xlator_t *this, fd_t *fd, int32_t child,
+                      int flags);
 
-static inline int
-afr_first_up_child (afr_private_t *priv)
-{
-	xlator_t ** children = NULL;
-	int         ret      = -1;
-	int         i        = 0;
-
-	LOCK (&priv->lock);
-	{
-		children = priv->children;
-		for (i = 0; i < priv->child_count; i++) {
-			if (priv->child_up[i]) {
-				ret = i;
-				break;
-			}
-		}
-	}
-	UNLOCK (&priv->lock);
-
-	return ret;
-}
+gf_boolean_t
+afr_have_quorum (char *logname, afr_private_t *priv);
 
+void
+afr_matrix_cleanup (int32_t **pending, unsigned int m);
 
-static inline int
-afr_transaction_local_init (afr_local_t *local, afr_private_t *priv)
-{
-        int i;
-
-        local->first_up_child = afr_first_up_child (priv);
-
-	local->child_errno = GF_CALLOC (sizeof (*local->child_errno),
-				        priv->child_count,
-                                        gf_afr_mt_int32_t);
-	if (!local->child_errno) {
-		return -ENOMEM;
-	}
-
-	local->pending = GF_CALLOC (sizeof (*local->pending),
-                                    priv->child_count,
-                                    gf_afr_mt_int32_t);
-
-	if (!local->pending) {
-		return -ENOMEM;
-	}
-
-        for (i = 0; i < priv->child_count; i++) {
-                local->pending[i] = GF_CALLOC (sizeof (*local->pending[i]),
-                                               3, /* data + metadata + entry */
-                                               gf_afr_mt_int32_t);
-                if (!local->pending[i])
-                        return -ENOMEM;
-        }
+int32_t**
+afr_matrix_create (unsigned int m, unsigned int n);
 
-	local->internal_lock.inode_locked_nodes =
-                GF_CALLOC (sizeof (*local->internal_lock.inode_locked_nodes),
-                           priv->child_count,
-                           gf_afr_mt_char);
+void
+afr_filter_xattrs (dict_t *xattr);
 
-	local->internal_lock.entry_locked_nodes =
-                GF_CALLOC (sizeof (*local->internal_lock.entry_locked_nodes),
-                           priv->child_count,
-                           gf_afr_mt_char);
+/*
+ * Special value indicating we should use the "auto" quorum method instead of
+ * a fixed value (including zero to turn off quorum enforcement).
+ */
+#define AFR_QUORUM_AUTO INT_MAX
 
-	local->internal_lock.locked_nodes =
-                GF_CALLOC (sizeof (*local->internal_lock.locked_nodes),
-                           priv->child_count,
-                           gf_afr_mt_char);
+/*
+ * Having this as a macro will make debugging a bit weirder, but does reduce
+ * the probability of functions handling this check inconsistently.
+ */
+#define QUORUM_CHECK(_func,_label) do {                                  \
+        if (priv->quorum_count && !afr_have_quorum(this->name,priv)) { \
+                gf_log(this->name,GF_LOG_WARNING,                        \
+                       "failing "#_func" due to lack of quorum");        \
+                op_errno = EROFS;                                        \
+                goto _label;                                             \
+        }                                                                \
+} while (0);
 
-	local->internal_lock.lower_locked_nodes
-                = GF_CALLOC (sizeof (*local->internal_lock.lower_locked_nodes),
-                             priv->child_count,
-                             gf_afr_mt_char);
+int
+afr_fd_report_unstable_write (xlator_t *this, fd_t *fd);
 
-	local->transaction.child_errno = GF_CALLOC (sizeof (*local->transaction.child_errno),
-					            priv->child_count,
-                                                    gf_afr_mt_int32_t);
+gf_boolean_t
+afr_fd_has_witnessed_unstable_write (xlator_t *this, fd_t *fd);
 
-        local->internal_lock.transaction_lk_type = AFR_TRANSACTION_LK;
+void
+afr_delayed_changelog_wake_resume (xlator_t *this, fd_t *fd, call_stub_t *stub);
 
-	return 0;
-}
+int
+afr_inodelk_init (afr_inodelk_t *lk, char *dom, size_t child_count);
+
+void
+afr_handle_open_fd_count (call_frame_t *frame, xlator_t *this);
+
+int
+afr_local_pathinfo (char *pathinfo, gf_boolean_t *is_local);
 
+void
+afr_remove_eager_lock_stub (afr_local_t *local);
 #endif /* __AFR_H__ */
diff --git a/xlators/cluster/afr/src/pump.c b/xlators/cluster/afr/src/pump.c
index a499cbb33..eed509956 100644
--- a/xlators/cluster/afr/src/pump.c
+++ b/xlators/cluster/afr/src/pump.c
@@ -1,25 +1,17 @@
 /*
-   Copyright (c) 2007-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #include <unistd.h>
 #include <sys/time.h>
 #include <stdlib.h>
+#include <fnmatch.h>
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -28,6 +20,129 @@
 
 #include "afr-common.c"
 #include "defaults.c"
+#include "glusterfs.h"
+#include "pump.h"
+
+
+static int
+afr_set_dict_gfid (dict_t *dict, uuid_t gfid)
+{
+        int ret       = 0;
+        uuid_t *pgfid = NULL;
+
+        GF_ASSERT (gfid);
+
+        pgfid = GF_CALLOC (1, sizeof (uuid_t), gf_common_mt_char);
+        if (!pgfid) {
+                ret = -1;
+                goto out;
+        }
+
+        uuid_copy (*pgfid, gfid);
+
+        ret = dict_set_dynptr (dict, "gfid-req", pgfid, sizeof (uuid_t));
+        if (ret)
+                gf_log (THIS->name, GF_LOG_ERROR, "gfid set failed");
+
+out:
+        if (ret && pgfid)
+                GF_FREE (pgfid);
+	return ret;
+}
+
+static int
+afr_set_root_gfid (dict_t *dict)
+{
+        uuid_t gfid;
+        int ret = 0;
+
+        memset (gfid, 0, 16);
+        gfid[15] = 1;
+
+        ret = afr_set_dict_gfid (dict, gfid);
+
+        return ret;
+}
+
+static int
+afr_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name)
+{
+        int   ret = -1;
+        uuid_t pargfid = {0};
+
+        if (!child)
+                goto out;
+
+        if (!uuid_is_null (parent->inode->gfid))
+                uuid_copy (pargfid, parent->inode->gfid);
+        else if (!uuid_is_null (parent->gfid))
+                uuid_copy (pargfid, parent->gfid);
+
+        if (uuid_is_null (pargfid))
+                goto out;
+
+        if (strcmp (parent->path, "/") == 0)
+                ret = gf_asprintf ((char **)&child->path, "/%s", name);
+        else
+                ret = gf_asprintf ((char **)&child->path, "%s/%s", parent->path,
+                                   name);
+
+        if (-1 == ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "asprintf failed while setting child path");
+        }
+
+        child->name = strrchr (child->path, '/');
+        if (child->name)
+                child->name++;
+
+        child->parent = inode_ref (parent->inode);
+        child->inode = inode_new (parent->inode->table);
+        uuid_copy (child->pargfid, pargfid);
+
+        if (!child->inode) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if ((ret == -1) && child)
+                loc_wipe (child);
+
+        return ret;
+}
+
+static void
+afr_build_root_loc (xlator_t *this, loc_t *loc)
+{
+        afr_private_t   *priv = NULL;
+
+        priv = this->private;
+        loc->path = gf_strdup ("/");
+        loc->name = "";
+        loc->inode = inode_ref (priv->root_inode);
+        uuid_copy (loc->gfid, loc->inode->gfid);
+}
+
+static void
+afr_update_loc_gfids (loc_t *loc, struct iatt *buf, struct iatt *postparent)
+{
+        GF_ASSERT (loc);
+        GF_ASSERT (buf);
+
+        uuid_copy (loc->gfid, buf->ia_gfid);
+        if (postparent)
+                uuid_copy (loc->pargfid, postparent->ia_gfid);
+}
+
+static uint64_t pump_pid = 0;
+static inline void
+pump_fill_loc_info (loc_t *loc, struct iatt *iatt, struct iatt *parent)
+{
+        afr_update_loc_gfids (loc, iatt, parent);
+        uuid_copy (loc->inode->gfid, iatt->ia_gfid);
+}
 
 static int
 pump_mark_start_pending (xlator_t *this)
@@ -135,92 +250,24 @@ pump_set_resume_path (xlator_t *this, const char *path)
         priv = this->private;
         pump_priv = priv->pump_private;
 
-        assert (pump_priv);
+        GF_ASSERT (pump_priv);
 
         LOCK (&pump_priv->resume_path_lock);
         {
-                pump_priv->resume_path = strdup (path);
-                if (!pump_priv->resume_path)
-                        ret = -1;
+                strncpy (pump_priv->resume_path, path, strlen (path) + 1);
         }
         UNLOCK (&pump_priv->resume_path_lock);
 
         return ret;
 }
 
-static void
-build_child_loc (loc_t *parent, loc_t *child, char *path, char *name)
-{
-        child->path = path;
-        child->name = name;
-
-        child->parent = inode_ref (parent->inode);
-        child->inode = inode_new (parent->inode->table);
-}
-
-static char *
-build_file_path (loc_t *loc, gf_dirent_t *entry)
-{
-        xlator_t *this = NULL;
-        char *file_path = NULL;
-        int pathlen = 0;
-        int total_size = 0;
-
-        this = THIS;
-
-        pathlen = STRLEN_0 (loc->path);
-
-        if (IS_ROOT_PATH (loc->path)) {
-                total_size = pathlen + entry->d_len;
-                file_path = GF_CALLOC (1, total_size, gf_afr_mt_char);
-                if (!file_path) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
-                        return NULL;
-                }
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "constructing file path of size=%d"
-                        "pathlen=%d, d_len=%d",
-                        total_size, pathlen,
-                        entry->d_len);
-
-                snprintf(file_path, total_size, "%s%s", loc->path, entry->d_name);
-
-        } else {
-                total_size = pathlen + entry->d_len + 1; /* for the extra '/' in the path */
-                file_path = GF_CALLOC (1, total_size + 1, gf_afr_mt_char);
-                if (!file_path) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
-                        return NULL;
-                }
-
-                gf_log (this->name, GF_LOG_TRACE,
-                        "constructing file path of size=%d"
-                        "pathlen=%d, d_len=%d",
-                        total_size, pathlen,
-                        entry->d_len);
-
-                snprintf(file_path, total_size, "%s/%s", loc->path, entry->d_name);
-        }
-
-        gf_log (this->name, GF_LOG_TRACE,
-                "path=%s and d_name=%s", loc->path, entry->d_name);
-        gf_log (this->name, GF_LOG_TRACE,
-                "constructed file_path=%s of size=%d", file_path, total_size);
-
-        return file_path;
-}
-
 static int
 pump_save_path (xlator_t *this, const char *path)
 {
         afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
         pump_state_t state;
         dict_t *dict = NULL;
-        loc_t  loc;
+        loc_t  loc = {0};
         int dict_ret = 0;
         int ret = -1;
 
@@ -229,29 +276,30 @@ pump_save_path (xlator_t *this, const char *path)
                 return 0;
 
         priv = this->private;
-        pump_priv = priv->pump_private;
 
-        assert (priv->root_inode);
+        GF_ASSERT (priv->root_inode);
 
-        build_root_loc (priv->root_inode, &loc);
+        afr_build_root_loc (this, &loc);
 
         dict = dict_new ();
         dict_ret = dict_set_str (dict, PUMP_PATH, (char *)path);
+        if (dict_ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to set the key %s", path, PUMP_PATH);
 
         ret = syncop_setxattr (PUMP_SOURCE_CHILD (this), &loc, dict, 0);
 
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_INFO,
                         "setxattr failed - could not save path=%s", path);
         } else {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "setxattr succeeded - saved path=%s", path);
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Saving path for status info");
         }
 
         dict_unref (dict);
 
+        loc_wipe (&loc);
         return 0;
 }
 
@@ -314,15 +362,9 @@ pump_get_resume_path (xlator_t *this)
 static int
 pump_update_resume_state (xlator_t *this, const char *path)
 {
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-
         pump_state_t state;
         const char *resume_path = NULL;
 
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
         state = pump_get_state ();
 
         if (state == PUMP_STATE_RESUME) {
@@ -350,16 +392,10 @@ pump_update_resume_state (xlator_t *this, const char *path)
 static gf_boolean_t
 is_pump_traversal_allowed (xlator_t *this, const char *path)
 {
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-
         pump_state_t state;
         const char *resume_path = NULL;
         gf_boolean_t ret = _gf_true;
 
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
         state = pump_get_state ();
 
         if (state == PUMP_STATE_RESUME) {
@@ -402,38 +438,33 @@ pump_save_file_stats (xlator_t *this, const char *path)
 static int
 gf_pump_traverse_directory (loc_t *loc)
 {
-        xlator_t *this = NULL;
-        afr_private_t *priv = NULL;
-        fd_t     *fd   = NULL;
-
-        off_t       offset   = 0;
-        loc_t       entry_loc;
-        gf_dirent_t *entry = NULL;
-        gf_dirent_t *tmp = NULL;
-        gf_dirent_t entries;
-
-	struct iatt iatt, parent;
-	dict_t *xattr_rsp;
-
-        int source = 0;
-
-        char *file_path = NULL;
-        int ret = 0;
+        xlator_t        *this              = NULL;
+        fd_t            *fd                = NULL;
+        off_t           offset             = 0;
+        loc_t           entry_loc          = {0};
+        gf_dirent_t     *entry             = NULL;
+        gf_dirent_t     *tmp               = NULL;
+        gf_dirent_t     entries;
+	struct iatt     iatt               = {0};
+        struct iatt     parent             = {0};
+	dict_t          *xattr_rsp         = NULL;
+        int             ret                = 0;
+        gf_boolean_t    is_directory_empty = _gf_true;
+        gf_boolean_t    free_entries       = _gf_false;
 
         INIT_LIST_HEAD (&entries.list);
         this = THIS;
-        priv = this->private;
 
-        assert (loc->inode);
+        GF_ASSERT (loc->inode);
 
-	fd = fd_create (loc->inode, PUMP_PID);
+	fd = fd_create (loc->inode, pump_pid);
         if (!fd) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "Failed to create fd for %s", loc->path);
                 goto out;
         }
 
-        ret = syncop_opendir (priv->children[source], loc, fd);
+        ret = syncop_opendir (this, loc, fd);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "opendir failed on %s", loc->path);
@@ -444,7 +475,8 @@ gf_pump_traverse_directory (loc_t *loc)
                 "pump opendir on %s returned=%d",
                 loc->path, ret);
 
-        while (syncop_readdirp (priv->children[source], fd, 131072, offset, &entries)) {
+        while (syncop_readdirp (this, fd, 131072, offset, NULL, &entries)) {
+                free_entries = _gf_true;
 
                 if (list_empty (&entries.list)) {
                         gf_log (this->name, GF_LOG_TRACE,
@@ -456,105 +488,108 @@ gf_pump_traverse_directory (loc_t *loc)
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "found readdir entry=%s", entry->d_name);
 
-                        file_path = build_file_path (loc, entry);
-                        if (!file_path) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "file path construction failed");
-                                goto out;
+                        offset = entry->d_off;
+                        if (uuid_is_null (entry->d_stat.ia_gfid)) {
+                                gf_log (this->name, GF_LOG_WARNING, "%s/%s: No "
+                                        "gfid present skipping",
+                                        loc->path, entry->d_name);
+                                continue;
                         }
+                        loc_wipe (&entry_loc);
+                        ret = afr_build_child_loc (this, &entry_loc, loc,
+                                                   entry->d_name);
+                        if (ret)
+                                goto out;
 
-                        build_child_loc (loc, &entry_loc, file_path, entry->d_name);
-
-                        if (!IS_ENTRY_CWD (entry->d_name) &&
-                                           !IS_ENTRY_PARENT (entry->d_name)) {
-
-                                    ret = syncop_lookup (this, &entry_loc, NULL,
-                                                         &iatt, &xattr_rsp, &parent);
+			if ((strcmp (entry->d_name, ".") == 0) ||
+			    (strcmp (entry->d_name, "..") == 0))
+				continue;
 
-                                    entry_loc.ino = iatt.ia_ino;
-                                    entry_loc.inode->ino = iatt.ia_ino;
-                                    memcpy (entry_loc.inode->gfid, iatt.ia_gfid, 16);
+			is_directory_empty = _gf_false;
+			gf_log (this->name, GF_LOG_DEBUG,
+				"lookup %s => %"PRId64,
+				entry_loc.path,
+				iatt.ia_ino);
 
-                                    gf_log (this->name, GF_LOG_DEBUG,
-                                            "lookup %s => %"PRId64,
-                                            entry_loc.path,
-                                            iatt.ia_ino);
+			ret = syncop_lookup (this, &entry_loc, NULL, &iatt,
+					     &xattr_rsp, &parent);
 
-                                    ret = syncop_lookup (this, &entry_loc, NULL,
-                                                         &iatt, &xattr_rsp, &parent);
+			if (ret) {
+				gf_log (this->name, GF_LOG_ERROR,
+					"%s: lookup failed", entry_loc.path);
+				continue;
+			}
 
+			ret = afr_selfheal_name (this, loc->gfid, entry->d_name);
+			if (ret) {
+				gf_log (this->name, GF_LOG_ERROR,
+					"%s: name self-heal failed (%s/%s)",
+					entry_loc.path, uuid_utoa (loc->gfid),
+					entry->d_name);
+				continue;
+			}
 
-                                    gf_log (this->name, GF_LOG_DEBUG,
-                                            "second lookup ret=%d: %s => %"PRId64,
-                                            ret,
-                                            entry_loc.path,
-                                            iatt.ia_ino);
+			ret = afr_selfheal (this, iatt.ia_gfid);
+			if (ret) {
+				gf_log (this->name, GF_LOG_ERROR,
+					"%s: self-heal failed (%s)",
+					entry_loc.path, uuid_utoa (iatt.ia_gfid));
+				continue;
+			}
 
-                                    pump_update_resume_state (this, entry_loc.path);
+			pump_fill_loc_info (&entry_loc, &iatt, &parent);
 
-                                    pump_save_path (this, entry_loc.path);
-                                    pump_save_file_stats (this, entry_loc.path);
+			pump_update_resume_state (this, entry_loc.path);
 
-                                    ret = pump_check_and_update_status (this);
-                                    if (ret < 0) {
-                                            gf_log (this->name, GF_LOG_DEBUG,
-                                                    "Pump beginning to exit out");
-                                            goto out;
-                                    }
+			pump_save_path (this, entry_loc.path);
+			pump_save_file_stats (this, entry_loc.path);
 
-                                    gf_log (this->name, GF_LOG_TRACE,
-                                            "type of file=%d, IFDIR=%d",
-                                            iatt.ia_type, IA_IFDIR);
+			ret = pump_check_and_update_status (this);
+			if (ret < 0) {
+				gf_log (this->name, GF_LOG_DEBUG,
+					"Pump beginning to exit out");
+				goto out;
+			}
 
-                                    if (IA_ISDIR (iatt.ia_type)) {
-                                            if (is_pump_traversal_allowed (this, entry_loc.path)) {
-                                                    gf_log (this->name, GF_LOG_TRACE,
-                                                            "entering dir=%s",
-                                                            entry->d_name);
-                                                    gf_pump_traverse_directory (&entry_loc);
-                                            }
-                                    }
-                            }
-                        offset = entry->d_off;
-                        loc_wipe (&entry_loc);
+			if (IA_ISDIR (iatt.ia_type)) {
+				if (is_pump_traversal_allowed (this, entry_loc.path)) {
+					gf_log (this->name, GF_LOG_TRACE,
+						"entering dir=%s", entry->d_name);
+					gf_pump_traverse_directory (&entry_loc);
+				}
+                        }
                 }
 
                 gf_dirent_free (&entries);
-                gf_log (this->name, GF_LOG_TRACE,
-                        "offset incremented to %d",
+                free_entries = _gf_false;
+                gf_log (this->name, GF_LOG_TRACE, "offset incremented to %d",
                         (int32_t ) offset);
 
         }
 
-out:
-        return 0;
-
-}
+        ret = syncop_close (fd);
+        if (ret < 0)
+                gf_log (this->name, GF_LOG_DEBUG, "closing the fd failed");
 
-void
-build_root_loc (inode_t *inode, loc_t *loc)
-{
-        loc->path = "/";
-        loc->name = "";
-        loc->inode = inode;
-        loc->ino = 1;
-        loc->inode->ino = 1;
-        memset (loc->inode->gfid, 0, 16);
-        loc->inode->gfid[15] = 1;
+        if (is_directory_empty && (strcmp (loc->path, "/") == 0)) {
+               pump_change_state (this, PUMP_STATE_RUNNING);
+               gf_log (this->name, GF_LOG_INFO, "Empty source brick. "
+                                "Nothing to be done.");
+        }
 
+out:
+        if (entry_loc.path)
+                loc_wipe (&entry_loc);
+        if (free_entries)
+                gf_dirent_free (&entries);
+        return 0;
 }
 
 static int
 pump_update_resume_path (xlator_t *this)
 {
-        afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
-
         const char *resume_path = NULL;
 
-        priv = this->private;
-        pump_priv = priv->pump_private;
-
         resume_path = pump_get_resume_path (this);
 
         if (resume_path) {
@@ -573,6 +608,40 @@ pump_update_resume_path (xlator_t *this)
         return 0;
 }
 
+static int32_t
+pump_xattr_cleaner (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        afr_private_t  *priv      = NULL;
+        loc_t           loc       = {0};
+        int             i         = 0;
+        int             ret       = 0;
+        int             source    = 0;
+        int             sink      = 1;
+
+        priv      = this->private;
+
+        afr_build_root_loc (this, &loc);
+
+        ret = syncop_removexattr (priv->children[source], &loc,
+				  PUMP_PATH, 0);
+
+        ret = syncop_removexattr (priv->children[sink], &loc,
+                                  PUMP_SINK_COMPLETE, 0);
+
+        for (i = 0; i < priv->child_count; i++) {
+                ret = syncop_removexattr (priv->children[i], &loc,
+                                          PUMP_SOURCE_COMPLETE, 0);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG, "removexattr "
+                                "failed with %s", strerror (-ret));
+                }
+        }
+
+        loc_wipe (&loc);
+        return pump_command_reply (frame, this);
+}
+
 static int
 pump_complete_migration (xlator_t *this)
 {
@@ -580,16 +649,16 @@ pump_complete_migration (xlator_t *this)
         pump_private_t *pump_priv = NULL;
         dict_t *dict = NULL;
         pump_state_t state;
-        loc_t  loc;
+        loc_t  loc = {0};
         int dict_ret = 0;
         int ret = -1;
 
         priv = this->private;
         pump_priv = priv->pump_private;
 
-        assert (priv->root_inode);
+        GF_ASSERT (priv->root_inode);
 
-        build_root_loc (priv->root_inode, &loc);
+        afr_build_root_loc (this, &loc);
 
         dict = dict_new ();
 
@@ -601,6 +670,10 @@ pump_complete_migration (xlator_t *this)
                 pump_priv->pump_finished = _gf_true;
 
                 dict_ret = dict_set_str (dict, PUMP_SOURCE_COMPLETE, "jargon");
+                if (dict_ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: failed to set the key %s",
+                                loc.path, PUMP_SOURCE_COMPLETE);
 
                 ret = syncop_setxattr (PUMP_SOURCE_CHILD (this), &loc, dict, 0);
                 if (ret < 0) {
@@ -608,29 +681,27 @@ pump_complete_migration (xlator_t *this)
                                 "setxattr failed - while  notifying source complete");
                 }
                 dict_ret = dict_set_str (dict, PUMP_SINK_COMPLETE, "jargon");
+                if (dict_ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: failed to set the key %s",
+                                loc.path, PUMP_SINK_COMPLETE);
 
                 ret = syncop_setxattr (PUMP_SINK_CHILD (this), &loc, dict, 0);
                 if (ret < 0) {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "setxattr failed - while notifying sink complete");
                 }
-        }
-
-        return 0;
-}
-
-static int
-pump_set_root_gfid (dict_t *dict)
-{
-        uuid_t gfid;
-        int ret = 0;
 
-        memset (gfid, 0, 16);
-        gfid[15] = 1;
+                pump_save_path (this, "/");
 
-        ret = afr_set_dict_gfid (dict, gfid);
+        } else if (state == PUMP_STATE_ABORT) {
+                gf_log (this->name, GF_LOG_DEBUG, "Starting cleanup "
+                        "of pump internal xattrs");
+                call_resume (pump_priv->cleaner);
+        }
 
-        return ret;
+        loc_wipe (&loc);
+        return 0;
 }
 
 static int
@@ -646,7 +717,7 @@ pump_lookup_sink (loc_t *loc)
 
         xattr_req = dict_new ();
 
-        ret = pump_set_root_gfid (xattr_req);
+        ret = afr_set_root_gfid (xattr_req);
         if (ret)
                 goto out;
 
@@ -656,6 +727,7 @@ pump_lookup_sink (loc_t *loc)
         if (ret) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "Lookup on sink child failed");
+                ret = -1;
                 goto out;
         }
 
@@ -673,7 +745,7 @@ pump_task (void *data)
         afr_private_t *priv = NULL;
 
 
-        loc_t loc;
+        loc_t loc = {0};
 	struct iatt iatt, parent;
 	dict_t *xattr_rsp = NULL;
         dict_t *xattr_req = NULL;
@@ -683,9 +755,9 @@ pump_task (void *data)
         this = THIS;
         priv = this->private;
 
-        assert (priv->root_inode);
+        GF_ASSERT (priv->root_inode);
 
-        build_root_loc (priv->root_inode, &loc);
+        afr_build_root_loc (this, &loc);
         xattr_req = dict_new ();
         if (!xattr_req) {
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -694,14 +766,13 @@ pump_task (void *data)
                 goto out;
         }
 
-        pump_set_root_gfid (xattr_req);
+        afr_set_root_gfid (xattr_req);
         ret = syncop_lookup (this, &loc, xattr_req,
                              &iatt, &xattr_rsp, &parent);
 
         gf_log (this->name, GF_LOG_TRACE,
-                "lookup: ino=%"PRId64", path=%s",
-                loc.ino,
-                loc.path);
+                "lookup: path=%s gfid=%s",
+                loc.path, uuid_utoa (loc.inode->gfid));
 
         ret = pump_check_and_update_status (this);
         if (ret < 0) {
@@ -710,7 +781,7 @@ pump_task (void *data)
 
         pump_update_resume_path (this);
 
-        pump_set_root_gfid (xattr_req);
+        afr_set_root_gfid (xattr_req);
         ret = pump_lookup_sink (&loc);
         if (ret) {
                 pump_update_resume_path (this);
@@ -724,26 +795,23 @@ out:
         if (xattr_req)
                 dict_unref (xattr_req);
 
+        loc_wipe (&loc);
 	return 0;
 }
 
 
 static int
-pump_task_completion (int ret, void *data)
+pump_task_completion (int ret, call_frame_t *sync_frame, void *data)
 {
         xlator_t *this = NULL;
-        call_frame_t *frame = NULL;
         afr_private_t *priv = NULL;
-        pump_private_t *pump_priv = NULL;
 
         this = THIS;
 
-        frame = (call_frame_t *) data;
-
         priv = this->private;
-        pump_priv = priv->pump_private;
 
         inode_unref (priv->root_inode);
+        STACK_DESTROY (sync_frame->root);
 
         gf_log (this->name, GF_LOG_DEBUG,
                 "Pump xlator exiting");
@@ -761,21 +829,22 @@ pump_start (call_frame_t *pump_frame, xlator_t *this)
 	priv = this->private;
         pump_priv = priv->pump_private;
 
-        if (!pump_frame->root->lk_owner)
-                pump_frame->root->lk_owner = PUMP_LK_OWNER;
+        afr_set_lk_owner (pump_frame, this, pump_frame->root);
+	pump_pid = (uint64_t) (unsigned long)pump_frame->root;
 
 	ret = synctask_new (pump_priv->env, pump_task,
                             pump_task_completion,
-                            pump_frame);
+                            pump_frame, NULL);
         if (ret == -1) {
-                gf_log (this->name, GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_ERROR,
                         "starting pump failed");
                 pump_change_state (this, PUMP_STATE_ABORT);
                 goto out;
         }
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "setting pump as started");
+        gf_log (this->name, GF_LOG_DEBUG,
+                "setting pump as started lk_owner: %s %"PRIu64,
+                lkowner_utoa (&pump_frame->root->lk_owner), pump_pid);
 
         priv->use_afr_in_pump = 1;
 out:
@@ -809,9 +878,10 @@ pump_cmd_start_setxattr_cbk (call_frame_t *frame,
                              void *cookie,
                              xlator_t *this,
                              int32_t op_ret,
-                             int32_t op_errno)
+                             int32_t op_errno, dict_t *xdata)
 
 {
+        call_frame_t *prev = NULL;
         afr_local_t *local = NULL;
         int ret = 0;
 
@@ -831,6 +901,11 @@ pump_cmd_start_setxattr_cbk (call_frame_t *frame,
 
         pump_mark_start_pending (this);
 
+        /* send the PARENT_UP as pump is ready now */
+        prev = cookie;
+        if (prev && prev->this)
+                prev->this->notify (prev->this, GF_EVENT_PARENT_UP, this);
+
 out:
         local->op_ret = ret;
         pump_command_reply (frame, this);
@@ -844,8 +919,9 @@ pump_initiate_sink_connect (call_frame_t *frame, xlator_t *this)
         afr_local_t   *local     = NULL;
         afr_private_t *priv      = NULL;
         dict_t        *dict      = NULL;
-        char          *dst_brick = NULL;
-        loc_t loc;
+        data_t        *data      = NULL;
+        char          *clnt_cmd  = NULL;
+        loc_t loc = {0};
 
         int ret = 0;
 
@@ -854,10 +930,11 @@ pump_initiate_sink_connect (call_frame_t *frame, xlator_t *this)
 
         GF_ASSERT (priv->root_inode);
 
-        build_root_loc (priv->root_inode, &loc);
+        afr_build_root_loc (this, &loc);
 
-        ret = dict_get_str (local->dict, PUMP_CMD_START, &dst_brick);
-        if (ret < 0) {
+        data = data_ref (dict_get (local->dict, RB_PUMP_CMD_START));
+        if (!data) {
+                ret = -1;
                 gf_log (this->name, GF_LOG_ERROR,
                         "Could not get destination brick value");
                 goto out;
@@ -865,17 +942,22 @@ pump_initiate_sink_connect (call_frame_t *frame, xlator_t *this)
 
         dict = dict_new ();
         if (!dict) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
                 ret = -1;
                 goto out;
         }
 
-        GF_ASSERT (dst_brick);
-        gf_log (this->name, GF_LOG_DEBUG,
-                "Got destination brick as %s", dst_brick);
+        clnt_cmd = GF_CALLOC (1, data->len+1, gf_common_mt_char);
+        if (!clnt_cmd) {
+                ret = -1;
+                goto out;
+        }
+
+        memcpy (clnt_cmd, data->data, data->len);
+        clnt_cmd[data->len] = '\0';
+        gf_log (this->name, GF_LOG_DEBUG, "Got destination brick %s\n",
+                        clnt_cmd);
 
-        ret = dict_set_str (dict, CLIENT_CMD_CONNECT, dst_brick);
+        ret = dict_set_dynstr (dict, CLIENT_CMD_CONNECT, clnt_cmd);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "Could not inititiate destination brick "
@@ -889,12 +971,21 @@ pump_initiate_sink_connect (call_frame_t *frame, xlator_t *this)
 		    PUMP_SINK_CHILD(this)->fops->setxattr,
 		    &loc,
 		    dict,
-		    0);
+		    0, NULL);
 
         ret = 0;
 
-        dict_unref (dict);
 out:
+        if (dict)
+                dict_unref (dict);
+
+        if (data)
+                data_unref (data);
+
+        if (ret && clnt_cmd)
+                GF_FREE (clnt_cmd);
+
+        loc_wipe (&loc);
         return ret;
 }
 
@@ -914,7 +1005,7 @@ pump_cmd_start_getxattr_cbk (call_frame_t *frame,
                              xlator_t *this,
                              int32_t op_ret,
                              int32_t op_errno,
-                             dict_t *dict)
+                             dict_t *dict, dict_t *xdata)
 {
         afr_local_t *local = NULL;
         char *path = NULL;
@@ -981,6 +1072,7 @@ pump_execute_status (call_frame_t *frame, xlator_t *this)
         uint64_t number_files = 0;
 
         char filename[PATH_MAX];
+        char summary[PATH_MAX+256];
         char *dict_str = NULL;
 
         int32_t op_ret = 0;
@@ -1009,28 +1101,35 @@ pump_execute_status (call_frame_t *frame, xlator_t *this)
         }
 
         if (pump_priv->pump_finished) {
-        snprintf (dict_str, PATH_MAX + 256, "Number of files migrated = %"PRIu64"        Migration complete ",
-                  number_files);
+                snprintf (summary, PATH_MAX+256,
+                          "no_of_files=%"PRIu64, number_files);
         } else {
-        snprintf (dict_str, PATH_MAX + 256, "Number of files migrated = %"PRIu64"       Current file= %s ",
-                  number_files, filename);
+                snprintf (summary, PATH_MAX+256,
+                          "no_of_files=%"PRIu64":current_file=%s",
+                          number_files, filename);
         }
+        snprintf (dict_str, PATH_MAX+256, "status=%d:%s",
+                  (pump_priv->pump_finished)?1:0, summary);
 
         dict = dict_new ();
 
-        ret = dict_set_str (dict, PUMP_CMD_STATUS, dict_str);
+        ret = dict_set_dynstr (dict, RB_PUMP_CMD_STATUS, dict_str);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
-                        "dict_set_str returned negative value");
+                        "dict_set_dynstr returned negative value");
+        } else {
+                dict_str = NULL;
         }
 
         op_ret = 0;
 
 out:
 
-        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+        AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, NULL);
+
+        if (dict)
+                dict_unref (dict);
 
-        dict_unref (dict);
         GF_FREE (dict_str);
 
         return 0;
@@ -1058,7 +1157,7 @@ pump_execute_start (call_frame_t *frame, xlator_t *this)
         afr_local_t   *local = NULL;
 
         int ret = 0;
-        loc_t loc;
+        loc_t loc = {0};
 
         priv = this->private;
         local = frame->local;
@@ -1073,14 +1172,14 @@ pump_execute_start (call_frame_t *frame, xlator_t *this)
 
         GF_ASSERT (priv->root_inode);
 
-        build_root_loc (priv->root_inode, &loc);
+        afr_build_root_loc (this, &loc);
 
 	STACK_WIND (frame,
 		    pump_cmd_start_getxattr_cbk,
 		    PUMP_SOURCE_CHILD(this),
 		    PUMP_SOURCE_CHILD(this)->fops->getxattr,
 		    &loc,
-		    PUMP_PATH);
+		    PUMP_PATH, NULL);
 
         ret = 0;
 
@@ -1090,15 +1189,69 @@ out:
                 pump_command_reply (frame, this);
         }
 
+        loc_wipe (&loc);
 	return 0;
 }
 
+static int
+pump_cleanup_helper (void *data) {
+        call_frame_t *frame = data;
+
+        pump_xattr_cleaner (frame, 0, frame->this, 0, 0, NULL);
+
+        return 0;
+}
+
+static int
+pump_cleanup_done (int ret, call_frame_t *sync_frame, void *data)
+{
+        STACK_DESTROY (sync_frame->root);
+
+        return 0;
+}
+
+int
+pump_execute_commit (call_frame_t *frame, xlator_t *this)
+{
+        afr_private_t  *priv       = NULL;
+        pump_private_t *pump_priv  = NULL;
+        afr_local_t    *local      = NULL;
+        call_frame_t   *sync_frame = NULL;
+        int             ret        = 0;
+
+        priv      = this->private;
+        pump_priv = priv->pump_private;
+        local     = frame->local;
+
+        local->op_ret = 0;
+        if (pump_priv->pump_finished) {
+                pump_change_state (this, PUMP_STATE_COMMIT);
+                sync_frame = create_frame (this, this->ctx->pool);
+                ret = synctask_new (pump_priv->env, pump_cleanup_helper,
+                                    pump_cleanup_done, sync_frame, frame);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG, "Couldn't create "
+                                "synctask for cleaning up xattrs.");
+                }
+
+        } else {
+                gf_log (this->name, GF_LOG_ERROR, "Commit can't proceed. "
+                        "Migration in progress");
+                local->op_ret = -1;
+                local->op_errno = EINPROGRESS;
+                pump_command_reply (frame, this);
+        }
+
+        return 0;
+}
 int
 pump_execute_abort (call_frame_t *frame, xlator_t *this)
 {
-        afr_private_t  *priv      = NULL;
-        pump_private_t *pump_priv = NULL;
-        afr_local_t    *local     = NULL;
+        afr_private_t  *priv       = NULL;
+        pump_private_t *pump_priv  = NULL;
+        afr_local_t    *local      = NULL;
+        call_frame_t   *sync_frame = NULL;
+        int             ret        = 0;
 
         priv      = this->private;
         pump_priv = priv->pump_private;
@@ -1114,7 +1267,20 @@ pump_execute_abort (call_frame_t *frame, xlator_t *this)
         UNLOCK (&pump_priv->resume_path_lock);
 
         local->op_ret = 0;
-        pump_command_reply (frame, this);
+        if (pump_priv->pump_finished) {
+                sync_frame = create_frame (this, this->ctx->pool);
+                ret = synctask_new (pump_priv->env, pump_cleanup_helper,
+                                    pump_cleanup_done, sync_frame, frame);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_DEBUG, "Couldn't create "
+                                "synctask for cleaning up xattrs.");
+                }
+
+        } else {
+                pump_priv->cleaner = fop_setxattr_cbk_stub (frame,
+                                                            pump_xattr_cleaner,
+                                                            0, 0, NULL);
+        }
 
         return 0;
 }
@@ -1126,7 +1292,7 @@ pump_command_status (xlator_t *this, dict_t *dict)
         int dict_ret = -1;
         int ret = _gf_true;
 
-        dict_ret = dict_get_str (dict, PUMP_CMD_STATUS, &cmd);
+        dict_ret = dict_get_str (dict, RB_PUMP_CMD_STATUS, &cmd);
         if (dict_ret < 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "Not a pump status command");
@@ -1150,7 +1316,7 @@ pump_command_pause (xlator_t *this, dict_t *dict)
         int dict_ret = -1;
         int ret = _gf_true;
 
-        dict_ret = dict_get_str (dict, PUMP_CMD_PAUSE, &cmd);
+        dict_ret = dict_get_str (dict, RB_PUMP_CMD_PAUSE, &cmd);
         if (dict_ret < 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "Not a pump pause command");
@@ -1168,22 +1334,22 @@ out:
 }
 
 gf_boolean_t
-pump_command_abort (xlator_t *this, dict_t *dict)
+pump_command_commit (xlator_t *this, dict_t *dict)
 {
         char *cmd = NULL;
         int dict_ret = -1;
         int ret = _gf_true;
 
-        dict_ret = dict_get_str (dict, PUMP_CMD_ABORT, &cmd);
+        dict_ret = dict_get_str (dict, RB_PUMP_CMD_COMMIT, &cmd);
         if (dict_ret < 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
-                        "Not a pump abort command");
+                        "Not a pump commit command");
                 ret = _gf_false;
                 goto out;
         }
 
         gf_log (this->name, GF_LOG_DEBUG,
-                "Hit a pump command - abort");
+                "Hit a pump command - commit");
         ret = _gf_true;
 
 out:
@@ -1192,22 +1358,22 @@ out:
 }
 
 gf_boolean_t
-pump_command_start (xlator_t *this, dict_t *dict)
+pump_command_abort (xlator_t *this, dict_t *dict)
 {
         char *cmd = NULL;
         int dict_ret = -1;
         int ret = _gf_true;
 
-        dict_ret = dict_get_str (dict, PUMP_CMD_START, &cmd);
+        dict_ret = dict_get_str (dict, RB_PUMP_CMD_ABORT, &cmd);
         if (dict_ret < 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
-                        "Not a pump start command");
+                        "Not a pump abort command");
                 ret = _gf_false;
                 goto out;
         }
 
         gf_log (this->name, GF_LOG_DEBUG,
-                "Hit a pump command - start");
+                "Hit a pump command - abort");
         ret = _gf_true;
 
 out:
@@ -1215,134 +1381,47 @@ out:
 
 }
 
-struct _xattr_key {
-        char *key;
-        struct list_head list;
-};
-
-static void
-__gather_xattr_keys (dict_t *dict, char *key, data_t *value,
-                     void *data)
-{
-        struct list_head *  list  = data;
-        struct _xattr_key * xkey  = NULL;
-
-        if (!strncmp (key, AFR_XATTR_PREFIX,
-                      strlen (AFR_XATTR_PREFIX))) {
-
-                xkey = GF_CALLOC (1, sizeof (*xkey), gf_afr_mt_xattr_key);
-                if (!xkey)
-                        return;
-
-                xkey->key = key;
-                INIT_LIST_HEAD (&xkey->list);
-
-                list_add_tail (&xkey->list, list);
-        }
-}
-
-static void
-__filter_xattrs (dict_t *dict)
+gf_boolean_t
+pump_command_start (xlator_t *this, dict_t *dict)
 {
-        struct list_head keys;
-
-        struct _xattr_key *key;
-        struct _xattr_key *tmp;
-
-        INIT_LIST_HEAD (&keys);
-
-        dict_foreach (dict, __gather_xattr_keys,
-                      (void *) &keys);
-
-        list_for_each_entry_safe (key, tmp, &keys, list) {
-                dict_del (dict, key->key);
-
-                list_del_init (&key->list);
+        char *cmd = NULL;
+        int dict_ret = -1;
+        int ret = _gf_true;
 
-                GF_FREE (key);
+        dict_ret = dict_get_str (dict, RB_PUMP_CMD_START, &cmd);
+        if (dict_ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Not a pump start command");
+                ret = _gf_false;
+                goto out;
         }
-}
-
-int32_t
-pump_getxattr_cbk (call_frame_t *frame, void *cookie,
-		  xlator_t *this, int32_t op_ret, int32_t op_errno,
-		  dict_t *dict)
-{
-	afr_private_t * priv     = NULL;
-	afr_local_t *   local    = NULL;
-	xlator_t **     children = NULL;
-
-	int unwind     = 1;
-	int last_tried = -1;
-	int this_try = -1;
-        int read_child = -1;
-
-	priv     = this->private;
-	children = priv->children;
-
-	local = frame->local;
-
-        read_child = (long) cookie;
-
-	if (op_ret == -1) {
-        retry:
-		last_tried = local->cont.getxattr.last_tried;
-
-		if (all_tried (last_tried, priv->child_count)) {
-			goto out;
-		}
-		this_try = ++local->cont.getxattr.last_tried;
-
-                if (this_try == read_child) {
-                        goto retry;
-                }
 
-		unwind = 0;
-		STACK_WIND_COOKIE (frame, pump_getxattr_cbk,
-				   (void *) (long) read_child,
-				   children[this_try],
-				   children[this_try]->fops->getxattr,
-				   &local->loc,
-				   local->cont.getxattr.name);
-	}
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Hit a pump command - start");
+        ret = _gf_true;
 
 out:
-	if (unwind) {
-                if (op_ret >= 0 && dict)
-                        __filter_xattrs (dict);
-
-		AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
-	}
+        return ret;
 
-	return 0;
 }
 
-int32_t
-pump_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *name)
+int
+pump_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+	       const char *name, dict_t *xdata)
 {
-	afr_private_t *   priv       = NULL;
-	xlator_t **       children   = NULL;
-	int               call_child = 0;
-	afr_local_t     * local      = NULL;
-
-        int               read_child = -1;
-
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
-
-	priv     = this->private;
-	VALIDATE_OR_GOTO (priv->children, out);
+	afr_private_t *priv = NULL;
+	int op_errno = 0;
+	int ret = 0;
 
-	children = priv->children;
+	priv = this->private;
 
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-	frame->local = local;
+        if (!priv->use_afr_in_pump) {
+                STACK_WIND (frame, default_getxattr_cbk,
+                            FIRST_CHILD (this),
+                            (FIRST_CHILD (this))->fops->getxattr,
+                            loc, name, xdata);
+                return 0;
+        }
 
         if (name) {
                 if (!strncmp (name, AFR_XATTR_PREFIX,
@@ -1352,188 +1431,21 @@ pump_getxattr (call_frame_t *frame, xlator_t *this,
                         goto out;
                 }
 
-                if (!strcmp (name, PUMP_CMD_STATUS)) {
+                if (!strcmp (name, RB_PUMP_CMD_STATUS)) {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "Hit pump command - status");
                         pump_execute_status (frame, this);
-                        op_ret = 0;
-                        goto out;
-                }
-        }
-
-        if (!priv->use_afr_in_pump) {
-                STACK_WIND (frame, default_getxattr_cbk,
-                            FIRST_CHILD (this),
-                            (FIRST_CHILD (this))->fops->getxattr,
-                            loc, name);
-                return 0;
-        }
-
-        read_child = afr_read_child (this, loc->inode);
-
-        if (read_child >= 0) {
-                call_child = read_child;
-
-                local->cont.getxattr.last_tried = -1;
-        } else {
-                call_child = afr_first_up_child (priv);
-
-                if (call_child == -1) {
-                        op_errno = ENOTCONN;
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "no child is up");
+                        ret = 0;
                         goto out;
                 }
-
-                local->cont.getxattr.last_tried = call_child;
         }
 
-	loc_copy (&local->loc, loc);
-	if (name)
-	  local->cont.getxattr.name       = gf_strdup (name);
-
-	STACK_WIND_COOKIE (frame, pump_getxattr_cbk,
-			   (void *) (long) call_child,
-			   children[call_child], children[call_child]->fops->getxattr,
-			   loc, name);
+	afr_getxattr (frame, this, loc, name, xdata);
 
-	op_ret = 0;
+	ret = 0;
 out:
-	if (op_ret == -1) {
-		AFR_STACK_UNWIND (getxattr, frame, op_ret, op_errno, NULL);
-	}
-	return 0;
-}
-
-static int
-afr_setxattr_unwind (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-	call_frame_t   *main_frame = NULL;
-
-	local = frame->local;
-	priv  = this->private;
-
-	LOCK (&frame->lock);
-	{
-		if (local->transaction.main_frame)
-			main_frame = local->transaction.main_frame;
-		local->transaction.main_frame = NULL;
-	}
-	UNLOCK (&frame->lock);
-
-	if (main_frame) {
-		AFR_STACK_UNWIND (setxattr, main_frame,
-                                  local->op_ret, local->op_errno)
-	}
-	return 0;
-}
-
-static int
-afr_setxattr_wind_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno)
-{
-	afr_local_t *   local = NULL;
-	afr_private_t * priv  = NULL;
-
-	int call_count  = -1;
-	int need_unwind = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret != -1) {
-			if (local->success_count == 0) {
-				local->op_ret = op_ret;
-			}
-			local->success_count++;
-
-			if (local->success_count == priv->child_count) {
-				need_unwind = 1;
-			}
-		}
-
-		local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	if (need_unwind)
-		local->transaction.unwind (frame, this);
-
-	call_count = afr_frame_return (frame);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-	}
-
-	return 0;
-}
-
-static int
-afr_setxattr_wind (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t *local = NULL;
-	afr_private_t *priv = NULL;
-
-	int call_count = -1;
-	int i = 0;
-
-	local = frame->local;
-	priv = this->private;
-
-	call_count = afr_up_children_count (priv->child_count, local->child_up);
-
-	if (call_count == 0) {
-		local->transaction.resume (frame, this);
-		return 0;
-	}
-
-	local->call_count = call_count;
-
-	for (i = 0; i < priv->child_count; i++) {
-		if (local->child_up[i]) {
-			STACK_WIND_COOKIE (frame, afr_setxattr_wind_cbk,
-					   (void *) (long) i,
-					   priv->children[i],
-					   priv->children[i]->fops->setxattr,
-					   &local->loc,
-					   local->cont.setxattr.dict,
-					   local->cont.setxattr.flags);
-
-			if (!--call_count)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-static int
-afr_setxattr_done (call_frame_t *frame, xlator_t *this)
-{
-	afr_local_t * local = frame->local;
-
-	local->transaction.unwind (frame, this);
-
-	AFR_STACK_DESTROY (frame);
-
-	return 0;
-}
-
-int32_t
-pump_setxattr_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno)
-{
-	STACK_UNWIND (frame,
-		      op_ret,
-		      op_errno);
+	if (ret < 0)
+		AFR_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
 	return 0;
 }
 
@@ -1545,121 +1457,91 @@ pump_command_reply (call_frame_t *frame, xlator_t *this)
         local = frame->local;
 
         if (local->op_ret < 0)
-                gf_log (this->name, GF_LOG_NORMAL,
+                gf_log (this->name, GF_LOG_INFO,
                         "Command failed");
         else
-                gf_log (this->name, GF_LOG_NORMAL,
+                gf_log (this->name, GF_LOG_INFO,
                         "Command succeeded");
 
-        dict_unref (local->dict);
-
         AFR_STACK_UNWIND (setxattr,
                           frame,
                           local->op_ret,
-                          local->op_errno);
+                          local->op_errno, NULL);
 
         return 0;
 }
 
 int
-pump_parse_command (call_frame_t *frame, xlator_t *this,
-                    afr_local_t *local, dict_t *dict)
+pump_parse_command (call_frame_t *frame, xlator_t *this, dict_t *dict,
+		    int *op_errno_p)
 {
-
+	afr_local_t *local = NULL;
         int ret = -1;
+	int op_errno = 0;
 
         if (pump_command_start (this, dict)) {
-                frame->local = local;
+                local = AFR_FRAME_INIT (frame, op_errno);
+		if (!local)
+			goto out;
                 local->dict = dict_ref (dict);
                 ret = pump_execute_start (frame, this);
 
         } else if (pump_command_pause (this, dict)) {
-                frame->local = local;
+		local = AFR_FRAME_INIT (frame, op_errno);
+		if (!local)
+			goto out;
                 local->dict = dict_ref (dict);
                 ret = pump_execute_pause (frame, this);
 
         } else if (pump_command_abort (this, dict)) {
-                frame->local = local;
+		local = AFR_FRAME_INIT (frame, op_errno);
+		if (!local)
+			goto out;
                 local->dict = dict_ref (dict);
                 ret = pump_execute_abort (frame, this);
+
+        } else if (pump_command_commit (this, dict)) {
+		local = AFR_FRAME_INIT (frame, op_errno);
+		if (!local)
+			goto out;
+                local->dict = dict_ref (dict);
+                ret = pump_execute_commit (frame, this);
         }
+out:
+	if (op_errno_p)
+		*op_errno_p = op_errno;
         return ret;
 }
 
 int
-pump_setxattr (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, dict_t *dict, int32_t flags)
+pump_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+	       int32_t flags, dict_t *xdata)
 {
-	afr_private_t * priv  = NULL;
-	afr_local_t   * local = NULL;
-	call_frame_t   *transaction_frame = NULL;
-
+	afr_private_t *priv = NULL;
 	int ret = -1;
-
-	int op_ret   = -1;
 	int op_errno = 0;
 
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (this, out);
-	VALIDATE_OR_GOTO (this->private, out);
+        GF_IF_INTERNAL_XATTR_GOTO ("trusted.glusterfs.pump*", dict, op_errno, out);
 
 	priv = this->private;
-
-	ALLOC_OR_GOTO (local, afr_local_t, out);
-
-	ret = AFR_LOCAL_INIT (local, priv);
-	if (ret < 0) {
-		op_errno = -ret;
-		goto out;
-	}
-
-        ret = pump_parse_command (frame, this,
-                                  local, dict);
-        if (ret >= 0) {
-                op_ret = 0;
-                goto out;
-        }
-
         if (!priv->use_afr_in_pump) {
                 STACK_WIND (frame, default_setxattr_cbk,
                             FIRST_CHILD (this),
                             (FIRST_CHILD (this))->fops->setxattr,
-                            loc, dict, flags);
+                            loc, dict, flags, xdata);
                 return 0;
         }
 
-	transaction_frame = copy_frame (frame);
-	if (!transaction_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		goto out;
-	}
-
-	transaction_frame->local = local;
-
-	local->op_ret = -1;
-
-	local->cont.setxattr.dict  = dict_ref (dict);
-	local->cont.setxattr.flags = flags;
-
-	local->transaction.fop    = afr_setxattr_wind;
-	local->transaction.done   = afr_setxattr_done;
-	local->transaction.unwind = afr_setxattr_unwind;
-
-	loc_copy (&local->loc, loc);
-
-	local->transaction.main_frame = frame;
-	local->transaction.start   = LLONG_MAX - 1;
-	local->transaction.len     = 0;
+        ret = pump_parse_command (frame, this, dict, &op_errno);
+        if (ret >= 0)
+                goto out;
 
-	afr_transaction (transaction_frame, this, AFR_METADATA_TRANSACTION);
+	afr_setxattr (frame, this, loc, dict, flags, xdata);
 
-	op_ret = 0;
+	ret = 0;
 out:
-	if (op_ret == -1) {
-		if (transaction_frame)
-			AFR_STACK_DESTROY (transaction_frame);
-		AFR_STACK_UNWIND (setxattr, frame, op_ret, op_errno);
+	if (ret < 0) {
+		AFR_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
 	}
 
 	return 0;
@@ -1693,7 +1575,7 @@ static int32_t
 pump_truncate (call_frame_t *frame,
                xlator_t *this,
                loc_t *loc,
-               off_t offset)
+               off_t offset, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1703,11 +1585,11 @@ pump_truncate (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->truncate,
                             loc,
-                            offset);
+                            offset, xdata);
                 return 0;
         }
 
-        afr_truncate (frame, this, loc, offset);
+        afr_truncate (frame, this, loc, offset, xdata);
         return 0;
 }
 
@@ -1716,7 +1598,7 @@ static int32_t
 pump_ftruncate (call_frame_t *frame,
                 xlator_t *this,
                 fd_t *fd,
-                off_t offset)
+                off_t offset, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1726,11 +1608,11 @@ pump_ftruncate (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->ftruncate,
                             fd,
-                            offset);
+                            offset, xdata);
                 return 0;
         }
 
-        afr_ftruncate (frame, this, fd, offset);
+        afr_ftruncate (frame, this, fd, offset, xdata);
         return 0;
 }
 
@@ -1739,7 +1621,7 @@ pump_ftruncate (call_frame_t *frame,
 
 int
 pump_mknod (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, mode_t mode, dev_t rdev, dict_t *parms)
+            loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1747,10 +1629,10 @@ pump_mknod (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_mknod_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->mknod,
-                            loc, mode, rdev, parms);
+                            loc, mode, rdev, umask, xdata);
                 return 0;
         }
-        afr_mknod (frame, this, loc, mode, rdev, parms);
+        afr_mknod (frame, this, loc, mode, rdev, umask, xdata);
         return 0;
 
 }
@@ -1759,7 +1641,7 @@ pump_mknod (call_frame_t *frame, xlator_t *this,
 
 int
 pump_mkdir (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, mode_t mode, dict_t *params)
+            loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1767,10 +1649,10 @@ pump_mkdir (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_mkdir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->mkdir,
-                            loc, mode, params);
+                            loc, mode, umask, xdata);
                 return 0;
         }
-        afr_mkdir (frame, this, loc, mode, params);
+        afr_mkdir (frame, this, loc, mode, umask, xdata);
         return 0;
 
 }
@@ -1779,7 +1661,7 @@ pump_mkdir (call_frame_t *frame, xlator_t *this,
 static int32_t
 pump_unlink (call_frame_t *frame,
              xlator_t *this,
-             loc_t *loc)
+             loc_t *loc, int xflag, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1788,10 +1670,10 @@ pump_unlink (call_frame_t *frame,
                             default_unlink_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->unlink,
-                            loc);
+                            loc, xflag, xdata);
                 return 0;
         }
-        afr_unlink (frame, this, loc);
+        afr_unlink (frame, this, loc, xflag, xdata);
         return 0;
 
 }
@@ -1799,7 +1681,7 @@ pump_unlink (call_frame_t *frame,
 
 static int
 pump_rmdir (call_frame_t *frame, xlator_t *this,
-            loc_t *loc, int flags)
+            loc_t *loc, int flags, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 
@@ -1809,11 +1691,11 @@ pump_rmdir (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_rmdir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->rmdir,
-                            loc, flags);
+                            loc, flags, xdata);
                 return 0;
         }
 
-        afr_rmdir (frame, this, loc, flags);
+        afr_rmdir (frame, this, loc, flags, xdata);
         return 0;
 
 }
@@ -1822,7 +1704,7 @@ pump_rmdir (call_frame_t *frame, xlator_t *this,
 
 int
 pump_symlink (call_frame_t *frame, xlator_t *this,
-              const char *linkpath, loc_t *loc, dict_t *params)
+              const char *linkpath, loc_t *loc, mode_t umask, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1830,10 +1712,10 @@ pump_symlink (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_symlink_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->symlink,
-                            linkpath, loc, params);
+                            linkpath, loc, umask, xdata);
                 return 0;
         }
-        afr_symlink (frame, this, linkpath, loc, params);
+        afr_symlink (frame, this, linkpath, loc, umask, xdata);
         return 0;
 
 }
@@ -1843,7 +1725,7 @@ static int32_t
 pump_rename (call_frame_t *frame,
              xlator_t *this,
              loc_t *oldloc,
-             loc_t *newloc)
+             loc_t *newloc, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1852,10 +1734,10 @@ pump_rename (call_frame_t *frame,
                             default_rename_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->rename,
-                            oldloc, newloc);
+                            oldloc, newloc, xdata);
                 return 0;
         }
-        afr_rename (frame, this, oldloc, newloc);
+        afr_rename (frame, this, oldloc, newloc, xdata);
         return 0;
 
 }
@@ -1865,7 +1747,7 @@ static int32_t
 pump_link (call_frame_t *frame,
            xlator_t *this,
            loc_t *oldloc,
-           loc_t *newloc)
+           loc_t *newloc, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1874,10 +1756,10 @@ pump_link (call_frame_t *frame,
                             default_link_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->link,
-                            oldloc, newloc);
+                            oldloc, newloc, xdata);
                 return 0;
         }
-        afr_link (frame, this, oldloc, newloc);
+        afr_link (frame, this, oldloc, newloc, xdata);
         return 0;
 
 }
@@ -1886,7 +1768,7 @@ pump_link (call_frame_t *frame,
 static int32_t
 pump_create (call_frame_t *frame, xlator_t *this,
              loc_t *loc, int32_t flags, mode_t mode,
-             fd_t *fd, dict_t *params)
+             mode_t umask, fd_t *fd, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1894,10 +1776,10 @@ pump_create (call_frame_t *frame, xlator_t *this,
                 STACK_WIND (frame, default_create_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->create,
-                            loc, flags, mode, fd, params);
+                            loc, flags, mode, umask, fd, xdata);
                 return 0;
         }
-        afr_create (frame, this, loc, flags, mode, fd, params);
+        afr_create (frame, this, loc, flags, mode, umask, fd, xdata);
         return 0;
 
 }
@@ -1907,8 +1789,7 @@ static int32_t
 pump_open (call_frame_t *frame,
            xlator_t *this,
            loc_t *loc,
-           int32_t flags, fd_t *fd,
-           int32_t wbflags)
+           int32_t flags, fd_t *fd, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1917,10 +1798,10 @@ pump_open (call_frame_t *frame,
                             default_open_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->open,
-                            loc, flags, fd, wbflags);
+                            loc, flags, fd, xdata);
                 return 0;
         }
-        afr_open (frame, this, loc, flags, fd, wbflags);
+        afr_open (frame, this, loc, flags, fd, xdata);
         return 0;
 
 }
@@ -1932,8 +1813,8 @@ pump_writev (call_frame_t *frame,
              fd_t *fd,
              struct iovec *vector,
              int32_t count,
-             off_t off,
-             struct iobref *iobref)
+             off_t off, uint32_t flags,
+             struct iobref *iobref, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1945,20 +1826,20 @@ pump_writev (call_frame_t *frame,
                             fd,
                             vector,
                             count,
-                            off,
-                            iobref);
+                            off, flags,
+                            iobref, xdata);
                 return 0;
         }
-        afr_writev (frame, this, fd, vector, count, off, iobref);
-        return 0;
 
+        afr_writev (frame, this, fd, vector, count, off, flags, iobref, xdata);
+        return 0;
 }
 
 
 static int32_t
 pump_flush (call_frame_t *frame,
             xlator_t *this,
-            fd_t *fd)
+            fd_t *fd, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1967,10 +1848,10 @@ pump_flush (call_frame_t *frame,
                             default_flush_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->flush,
-                            fd);
+                            fd, xdata);
                 return 0;
         }
-        afr_flush (frame, this, fd);
+        afr_flush (frame, this, fd, xdata);
         return 0;
 
 }
@@ -1980,7 +1861,7 @@ static int32_t
 pump_fsync (call_frame_t *frame,
             xlator_t *this,
             fd_t *fd,
-            int32_t flags)
+            int32_t flags, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -1990,10 +1871,10 @@ pump_fsync (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->fsync,
                             fd,
-                            flags);
+                            flags, xdata);
                 return 0;
         }
-        afr_fsync (frame, this, fd, flags);
+        afr_fsync (frame, this, fd, flags, xdata);
         return 0;
 
 }
@@ -2002,7 +1883,7 @@ pump_fsync (call_frame_t *frame,
 static int32_t
 pump_opendir (call_frame_t *frame,
               xlator_t *this,
-              loc_t *loc, fd_t *fd)
+              loc_t *loc, fd_t *fd, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2011,10 +1892,10 @@ pump_opendir (call_frame_t *frame,
                             default_opendir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->opendir,
-                            loc, fd);
+                            loc, fd, xdata);
                 return 0;
         }
-        afr_opendir (frame, this, loc, fd);
+        afr_opendir (frame, this, loc, fd, xdata);
         return 0;
 
 }
@@ -2024,7 +1905,7 @@ static int32_t
 pump_fsyncdir (call_frame_t *frame,
                xlator_t *this,
                fd_t *fd,
-               int32_t flags)
+               int32_t flags, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2034,10 +1915,10 @@ pump_fsyncdir (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->fsyncdir,
                             fd,
-                            flags);
+                            flags, xdata);
                 return 0;
         }
-        afr_fsyncdir (frame, this, fd, flags);
+        afr_fsyncdir (frame, this, fd, flags, xdata);
         return 0;
 
 }
@@ -2048,7 +1929,7 @@ pump_xattrop (call_frame_t *frame,
               xlator_t *this,
               loc_t *loc,
               gf_xattrop_flags_t flags,
-              dict_t *dict)
+              dict_t *dict, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2059,10 +1940,10 @@ pump_xattrop (call_frame_t *frame,
                             FIRST_CHILD(this)->fops->xattrop,
                             loc,
                             flags,
-                            dict);
+                            dict, xdata);
                 return 0;
         }
-        afr_xattrop (frame, this, loc, flags, dict);
+        afr_xattrop (frame, this, loc, flags, dict, xdata);
         return 0;
 
 }
@@ -2072,7 +1953,7 @@ pump_fxattrop (call_frame_t *frame,
                xlator_t *this,
                fd_t *fd,
                gf_xattrop_flags_t flags,
-               dict_t *dict)
+               dict_t *dict, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2083,10 +1964,10 @@ pump_fxattrop (call_frame_t *frame,
                             FIRST_CHILD(this)->fops->fxattrop,
                             fd,
                             flags,
-                            dict);
+                            dict, xdata);
                 return 0;
         }
-        afr_fxattrop (frame, this, fd, flags, dict);
+        afr_fxattrop (frame, this, fd, flags, dict, xdata);
         return 0;
 
 }
@@ -2096,9 +1977,17 @@ static int32_t
 pump_removexattr (call_frame_t *frame,
                   xlator_t *this,
                   loc_t *loc,
-                  const char *name)
+                  const char *name, dict_t *xdata)
 {
-        afr_private_t *priv  = NULL;
+        afr_private_t *priv     = NULL;
+        int            op_errno = -1;
+
+        VALIDATE_OR_GOTO (this, out);
+
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.glusterfs.pump*",
+                                 name, op_errno, out);
+
+        op_errno = 0;
 	priv = this->private;
         if (!priv->use_afr_in_pump) {
                 STACK_WIND (frame,
@@ -2106,10 +1995,14 @@ pump_removexattr (call_frame_t *frame,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->removexattr,
                             loc,
-                            name);
+                            name, xdata);
                 return 0;
         }
-        afr_removexattr (frame, this, loc, name);
+        afr_removexattr (frame, this, loc, name, xdata);
+
+ out:
+        if (op_errno)
+                AFR_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
         return 0;
 
 }
@@ -2121,7 +2014,7 @@ pump_readdir (call_frame_t *frame,
               xlator_t *this,
               fd_t *fd,
               size_t size,
-              off_t off)
+              off_t off, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2130,21 +2023,18 @@ pump_readdir (call_frame_t *frame,
                             default_readdir_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->readdir,
-                            fd, size, off);
+                            fd, size, off, xdata);
                 return 0;
         }
-        afr_readdir (frame, this, fd, size, off);
+        afr_readdir (frame, this, fd, size, off, xdata);
         return 0;
 
 }
 
 
 static int32_t
-pump_readdirp (call_frame_t *frame,
-               xlator_t *this,
-               fd_t *fd,
-               size_t size,
-               off_t off)
+pump_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
+               size_t size, off_t off, dict_t *dict)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2153,10 +2043,10 @@ pump_readdirp (call_frame_t *frame,
                             default_readdirp_cbk,
                             FIRST_CHILD(this),
                             FIRST_CHILD(this)->fops->readdirp,
-                            fd, size, off);
+                            fd, size, off, dict);
                 return 0;
         }
-        afr_readdirp (frame, this, fd, size, off);
+        afr_readdirp (frame, this, fd, size, off, dict);
         return 0;
 
 }
@@ -2187,13 +2077,24 @@ pump_release (xlator_t *this,
 
 }
 
+static int32_t
+pump_forget (xlator_t *this, inode_t *inode)
+{
+        afr_private_t  *priv  = NULL;
+
+        priv = this->private;
+        if (priv->use_afr_in_pump)
+                afr_forget (this, inode);
+
+        return 0;
+}
 
 static int32_t
 pump_setattr (call_frame_t *frame,
               xlator_t *this,
               loc_t *loc,
               struct iatt *stbuf,
-              int32_t valid)
+              int32_t valid, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2202,10 +2103,10 @@ pump_setattr (call_frame_t *frame,
                             default_setattr_cbk,
                             FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->setattr,
-                            loc, stbuf, valid);
+                            loc, stbuf, valid, xdata);
                 return 0;
         }
-        afr_setattr (frame, this, loc, stbuf, valid);
+        afr_setattr (frame, this, loc, stbuf, valid, xdata);
         return 0;
 
 }
@@ -2216,7 +2117,7 @@ pump_fsetattr (call_frame_t *frame,
                xlator_t *this,
                fd_t *fd,
                struct iatt *stbuf,
-               int32_t valid)
+               int32_t valid, dict_t *xdata)
 {
         afr_private_t *priv  = NULL;
 	priv = this->private;
@@ -2225,10 +2126,10 @@ pump_fsetattr (call_frame_t *frame,
                             default_fsetattr_cbk,
                             FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->fsetattr,
-                            fd, stbuf, valid);
+                            fd, stbuf, valid, xdata);
                 return 0;
         }
-        afr_fsetattr (frame, this, fd, stbuf, valid);
+        afr_fsetattr (frame, this, fd, stbuf, valid, xdata);
         return 0;
 
 }
@@ -2277,7 +2178,7 @@ notify (xlator_t *this, int32_t event,
 
         child_xl = (xlator_t *) data;
 
-        ret = afr_notify (this, event, data);
+        ret = afr_notify (this, event, data, NULL);
 
 	switch (event) {
 	case GF_EVENT_CHILD_DOWN:
@@ -2307,12 +2208,12 @@ int32_t
 init (xlator_t *this)
 {
 	afr_private_t * priv        = NULL;
-        pump_private_t *pump_priv = NULL;
+        pump_private_t *pump_priv   = NULL;
 	int             child_count = 0;
 	xlator_list_t * trav        = NULL;
 	int             i           = 0;
 	int             ret         = -1;
-	int             op_errno    = 0;
+	GF_UNUSED int   op_errno    = 0;
 
         int source_child = 0;
 
@@ -2328,26 +2229,36 @@ init (xlator_t *this)
 			"Volume is dangling.");
 	}
 
-	ALLOC_OR_GOTO (this->private, afr_private_t, out);
+	priv = GF_CALLOC (1, sizeof (afr_private_t), gf_afr_mt_afr_private_t);
+        if (!priv)
+                goto out;
+
+        LOCK_INIT (&priv->lock);
 
-	priv = this->private;
+        child_count = xlator_subvolume_count (this);
+        if (child_count != 2) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "There should be exactly 2 children - one source "
+                        "and one sink");
+                return -1;
+        }
+	priv->child_count = child_count;
 
         priv->read_child = source_child;
         priv->favorite_child = source_child;
         priv->background_self_heal_count = 0;
 
-	priv->data_self_heal     = 1;
+	priv->data_self_heal     = "on";
 	priv->metadata_self_heal = 1;
 	priv->entry_self_heal    = 1;
 
-        priv->data_self_heal_algorithm = "";
-
         priv->data_self_heal_window_size = 16;
 
 	priv->data_change_log     = 1;
 	priv->metadata_change_log = 1;
 	priv->entry_change_log    = 1;
         priv->use_afr_in_pump = 1;
+        priv->sh_readdir_size = 65536;
 
 	/* Locking options */
 
@@ -2356,31 +2267,6 @@ init (xlator_t *this)
            and the sink.
         */
 
-	priv->data_lock_server_count = 2;
-	priv->metadata_lock_server_count = 2;
-	priv->entry_lock_server_count = 2;
-
-	priv->strict_readdir = _gf_false;
-
-        trav = this->children;
-        while (trav) {
-                child_count++;
-                trav = trav->next;
-        }
-
-	priv->wait_count = 1;
-
-        if (child_count != 2) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "There should be exactly 2 children - one source "
-                        "and one sink");
-                return -1;
-        }
-	priv->child_count = child_count;
-
-	LOCK_INIT (&priv->lock);
-        LOCK_INIT (&priv->read_child_lock);
-
 	priv->child_up = GF_CALLOC (sizeof (unsigned char), child_count,
                                  gf_afr_mt_char);
 	if (!priv->child_up) {
@@ -2414,8 +2300,9 @@ init (xlator_t *this)
 	while (i < child_count) {
 		priv->children[i] = trav->xlator;
 
-                ret = asprintf (&priv->pending_key[i], "%s.%s", AFR_XATTR_PREFIX,
-                                trav->xlator->name);
+                ret = gf_asprintf (&priv->pending_key[i], "%s.%s",
+                                   AFR_XATTR_PREFIX,
+                                   trav->xlator->name);
                 if (-1 == ret) {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "asprintf failed to set pending key");
@@ -2427,9 +2314,21 @@ init (xlator_t *this)
 		i++;
 	}
 
-        priv->first_lookup = 1;
+        ret = gf_asprintf (&priv->sh_domain, "%s-self-heal", this->name);
+        if (-1 == ret) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
         priv->root_inode = NULL;
 
+        priv->last_event = GF_CALLOC (child_count, sizeof (*priv->last_event),
+                                      gf_afr_mt_int32_t);
+        if (!priv->last_event) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
 	pump_priv = GF_CALLOC (1, sizeof (*pump_priv),
                             gf_afr_mt_pump_priv);
 	if (!pump_priv) {
@@ -2445,13 +2344,12 @@ init (xlator_t *this)
         pump_priv->resume_path = GF_CALLOC (1, PATH_MAX,
                                             gf_afr_mt_char);
         if (!pump_priv->resume_path) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
+                gf_log (this->name, GF_LOG_ERROR, "Out of memory");
                 ret = -1;
                 goto out;
         }
 
-	pump_priv->env = syncenv_new (0);
+	pump_priv->env = this->ctx->env;
         if (!pump_priv->env) {
                 gf_log (this->name, GF_LOG_ERROR,
                         "Could not create new sync-environment");
@@ -2459,21 +2357,67 @@ init (xlator_t *this)
                 goto out;
         }
 
+        /* keep more local here as we may need them for self-heal etc */
+        this->local_pool = mem_pool_new (afr_local_t, 128);
+        if (!this->local_pool) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto out;
+        }
+
 	priv->pump_private = pump_priv;
+        pump_priv = NULL;
 
-        pthread_mutex_init (&priv->mutex, NULL);
-        INIT_LIST_HEAD (&priv->saved_fds);
+        this->private = priv;
+        priv = NULL;
 
         pump_change_state (this, PUMP_STATE_ABORT);
 
 	ret = 0;
 out:
+
+        if (pump_priv) {
+                GF_FREE (pump_priv->resume_path);
+                LOCK_DESTROY (&pump_priv->resume_path_lock);
+                LOCK_DESTROY (&pump_priv->pump_state_lock);
+                GF_FREE (pump_priv);
+        }
+
+        if (priv) {
+                GF_FREE (priv->child_up);
+                GF_FREE (priv->children);
+                GF_FREE (priv->pending_key);
+                GF_FREE (priv->last_event);
+                LOCK_DESTROY (&priv->lock);
+                GF_FREE (priv);
+        }
+
 	return ret;
 }
 
 int
 fini (xlator_t *this)
 {
+        afr_private_t * priv        = NULL;
+        pump_private_t *pump_priv = NULL;
+
+        priv      = this->private;
+        this->private = NULL;
+        if (!priv)
+                goto out;
+
+        pump_priv = priv->pump_private;
+        if (!pump_priv)
+                goto afr_priv;
+
+        GF_FREE (pump_priv->resume_path);
+        LOCK_DESTROY (&pump_priv->resume_path_lock);
+        LOCK_DESTROY (&pump_priv->pump_state_lock);
+        GF_FREE (pump_priv);
+afr_priv:
+        afr_priv_destroy (priv);
+out:
 	return 0;
 }
 
@@ -2521,6 +2465,7 @@ struct xlator_dumpops dumpops = {
 struct xlator_cbks cbks = {
 	.release     = pump_release,
 	.releasedir  = pump_releasedir,
+        .forget      = pump_forget,
 };
 
 struct volume_options options[] = {
diff --git a/xlators/cluster/afr/src/pump.h b/xlators/cluster/afr/src/pump.h
index 0512290f2..9d0b6db6a 100644
--- a/xlators/cluster/afr/src/pump.h
+++ b/xlators/cluster/afr/src/pump.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2007-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef __PUMP_H__
@@ -26,18 +17,6 @@
 #define CLIENT_CMD_CONNECT "trusted.glusterfs.client-connect"
 #define CLIENT_CMD_DISCONNECT "trusted.glusterfs.client-disconnect"
 
-#define PUMP_PID 696969
-#define PUMP_LK_OWNER 696969
-
-#define IS_ROOT_PATH(path) (!strcmp (path, "/"))
-#define IS_ENTRY_CWD(entry) (!strcmp (entry, "."))
-#define IS_ENTRY_PARENT(entry) (!strcmp (entry, ".."))
-
-#define PUMP_CMD_START  "trusted.glusterfs.pump.start"
-#define PUMP_CMD_ABORT  "trusted.glusterfs.pump.abort"
-#define PUMP_CMD_PAUSE  "trusted.glusterfs.pump.pause"
-#define PUMP_CMD_STATUS "trusted.glusterfs.pump.status"
-
 #define PUMP_SOURCE_COMPLETE "trusted.glusterfs.pump-source-complete"
 #define PUMP_SINK_COMPLETE "trusted.glusterfs.pump-sink-complete"
 
@@ -51,11 +30,12 @@ typedef enum {
         PUMP_STATE_RESUME,              /* Pump is resuming from a previous pause */
         PUMP_STATE_PAUSE,               /* Pump is paused */
         PUMP_STATE_ABORT,               /* Pump is aborted */
+        PUMP_STATE_COMMIT,              /* Pump is commited */
 } pump_state_t;
 
 typedef struct _pump_private {
 	struct syncenv *env;            /* The env pointer to the pump synctask */
-        const char *resume_path;        /* path to resume from the last pause */
+        char *resume_path;              /* path to resume from the last pause */
         gf_lock_t resume_path_lock;     /* Synchronize resume_path changes */
         gf_lock_t pump_state_lock;      /* Synchronize pump_state changes */
         pump_state_t pump_state;        /* State of pump */
@@ -64,6 +44,7 @@ typedef struct _pump_private {
         gf_boolean_t pump_finished;     /* Boolean to indicate pump termination */
         char pump_start_pending;        /* Boolean to mark start pending until
                                            CHILD_UP */
+        call_stub_t *cleaner;
 } pump_private_t;
 
 void
@@ -94,4 +75,7 @@ pump_command_status (xlator_t *this, dict_t *dict);
 int
 pump_execute_status (call_frame_t *frame, xlator_t *this);
 
+int
+pump_command_reply (call_frame_t *frame, xlator_t *this);
+
 #endif /* __PUMP_H__ */
diff --git a/xlators/cluster/dht/src/Makefile.am b/xlators/cluster/dht/src/Makefile.am
index 4b69aa071..3fc29bf81 100644
--- a/xlators/cluster/dht/src/Makefile.am
+++ b/xlators/cluster/dht/src/Makefile.am
@@ -1,34 +1,37 @@
-
 xlator_LTLIBRARIES = dht.la nufa.la switch.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
+dht_common_source = dht-layout.c dht-helper.c dht-linkfile.c dht-rebalance.c \
+	dht-selfheal.c dht-rename.c dht-hashfn.c dht-diskusage.c \
+	dht-common.c dht-inode-write.c dht-inode-read.c dht-shared.c \
+	$(top_builddir)/xlators/lib/src/libxlator.c
 
-dht_common_source = dht-layout.c dht-helper.c dht-linkfile.c \
-		dht-selfheal.c dht-rename.c dht-hashfn.c dht-diskusage.c
-
-dht_la_SOURCES = $(dht_common_source) dht.c 
+dht_la_SOURCES = $(dht_common_source) dht.c
 
 nufa_la_SOURCES = $(dht_common_source) nufa.c
 switch_la_SOURCES = $(dht_common_source) switch.c
 
-dht_la_LDFLAGS = -module -avoidversion
+dht_la_LDFLAGS = -module -avoid-version
 dht_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-nufa_la_LDFLAGS = -module -avoidversion
+nufa_la_LDFLAGS = -module -avoid-version
 nufa_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-switch_la_LDFLAGS = -module -avoidversion
+switch_la_LDFLAGS = -module -avoid-version
 switch_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = dht-common.h dht-common.c dht-mem-types.h
+noinst_HEADERS = dht-common.h dht-mem-types.h \
+	$(top_builddir)/xlators/lib/src/libxlator.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/xlators/lib/src
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
-CLEANFILES = 
+CLEANFILES =
 
 uninstall-local:
 	rm -f $(DESTDIR)$(xlatordir)/distribute.so
 
 install-data-hook:
-	ln -sf dht.so $(DESTDIR)$(xlatordir)/distribute.so
-\ No newline at end of file
+	ln -sf dht.so $(DESTDIR)$(xlatordir)/distribute.so
diff --git a/xlators/cluster/dht/src/dht-common.c b/xlators/cluster/dht/src/dht-common.c
index d537c1178..3868fc38f 100644
--- a/xlators/cluster/dht/src/dht-common.c
+++ b/xlators/cluster/dht/src/dht-common.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2009-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -27,12 +18,88 @@
 
 #include "glusterfs.h"
 #include "xlator.h"
+#include "libxlator.h"
 #include "dht-common.h"
 #include "defaults.h"
+#include "byte-order.h"
+#include "glusterfs-acl.h"
 
 #include <sys/time.h>
 #include <libgen.h>
 
+int
+dht_aggregate (dict_t *this, char *key, data_t *value, void *data)
+{
+        dict_t  *dst  = NULL;
+        int64_t *ptr  = 0, *size = NULL;
+        int32_t  ret  = -1;
+        data_t  *dict_data = NULL;
+
+        dst = data;
+
+        if (strcmp (key, GF_XATTR_QUOTA_SIZE_KEY) == 0) {
+                ret = dict_get_bin (dst, key, (void **)&size);
+                if (ret < 0) {
+                        size = GF_CALLOC (1, sizeof (int64_t),
+                                          gf_common_mt_char);
+                        if (size == NULL) {
+                                gf_log ("dht", GF_LOG_WARNING,
+                                        "memory allocation failed");
+                                return -1;
+                        }
+                        ret = dict_set_bin (dst, key, size, sizeof (int64_t));
+                        if (ret < 0) {
+                                gf_log ("dht", GF_LOG_WARNING,
+                                        "dht aggregate dict set failed");
+                                GF_FREE (size);
+                                return -1;
+                        }
+                }
+
+                ptr = data_to_bin (value);
+                if (ptr == NULL) {
+                        gf_log ("dht", GF_LOG_WARNING, "data to bin failed");
+                        return -1;
+                }
+
+                *size = hton64 (ntoh64 (*size) + ntoh64 (*ptr));
+
+        } else if (fnmatch (GF_XATTR_STIME_PATTERN, key, FNM_NOESCAPE) == 0) {
+                ret = gf_get_min_stime (THIS, dst, key, value);
+                if (ret < 0)
+                        return ret;
+        } else {
+                /* compare user xattrs only */
+                if (!strncmp (key, "user.", strlen ("user."))) {
+                        ret = dict_lookup (dst, key, &dict_data);
+                        if (!ret && dict_data && value) {
+                                ret = is_data_equal (dict_data, value);
+                                if (!ret)
+                                        gf_log ("dht", GF_LOG_DEBUG,
+                                                "xattr mismatch for %s", key);
+                        }
+                }
+                ret = dict_set (dst, key, value);
+                if (ret)
+                        gf_log ("dht", GF_LOG_WARNING, "xattr dict set failed");
+        }
+
+        return 0;
+}
+
+
+void
+dht_aggregate_xattr (dict_t *dst, dict_t *src)
+{
+        if ((dst == NULL) || (src == NULL)) {
+                goto out;
+        }
+
+        dict_foreach (src, dht_aggregate, dst);
+out:
+        return;
+}
+
 /* TODO:
    - use volumename in xattr instead of "dht"
    - use NS locks
@@ -43,219 +110,369 @@
 
 int
 dht_lookup_selfheal_cbk (call_frame_t *frame, void *cookie,
-			 xlator_t *this,
-			 int op_ret, int op_errno)
+                         xlator_t *this,
+                         int op_ret, int op_errno, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	dht_layout_t *layout = NULL;
-	int           ret = 0;
+        dht_local_t  *local = NULL;
+        dht_layout_t *layout = NULL;
+	int           ret = -1;
 
-	local = frame->local;
-	ret = op_ret;
+        GF_VALIDATE_OR_GOTO ("dht", frame, out);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
 
-        dht_frame_su_undo (frame);
+        local = frame->local;
+        ret = op_ret;
 
-	if (ret == 0) {
-		layout = local->selfheal.layout;
-		ret = dht_layout_set (this, local->inode, layout);
+        FRAME_SU_UNDO (frame, dht_local_t);
 
-		if (local->ia_ino) {
-			local->stbuf.ia_ino = local->ia_ino;
-		} else {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"could not find hashed subvolume for %s",
-				local->loc.path);
-		}
+        if (ret == 0) {
+                layout = local->selfheal.layout;
+                ret = dht_layout_set (this, local->inode, layout);
+        }
 
-                if (local->loc.parent)
-                        local->postparent.ia_ino = local->loc.parent->ino;
-	}
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           &local->postparent, 1);
+        }
 
-        WIPE (&local->postparent);
+        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
 
-	DHT_STACK_UNWIND (lookup, frame, ret, local->op_errno, local->inode,
-			  &local->stbuf, local->xattr, &local->postparent);
+        DHT_STACK_UNWIND (lookup, frame, ret, local->op_errno, local->inode,
+                          &local->stbuf, local->xattr, &local->postparent);
 
-	return 0;
+out:
+        return ret;
 }
 
 
 int
-dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int op_ret, int op_errno,
-                    inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                    struct iatt *postparent)
+dht_discover_complete (xlator_t *this, call_frame_t *discover_frame)
+{
+        dht_local_t     *local = NULL;
+        call_frame_t    *main_frame = NULL;
+        int              op_errno = 0;
+        int              ret = -1;
+        dht_layout_t    *layout = NULL;
+        dht_conf_t      *conf = NULL;
+
+        local = discover_frame->local;
+        layout = local->layout;
+        conf = this->private;
+
+        LOCK(&discover_frame->lock);
+        {
+                main_frame = local->main_frame;
+                local->main_frame = NULL;
+        }
+        UNLOCK(&discover_frame->lock);
+
+        if (!main_frame)
+                return 0;
+
+        if (local->file_count && local->dir_count) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "path %s exists as a file on one subvolume "
+                        "and directory on another. "
+                        "Please fix it manually",
+                        local->loc.path);
+                op_errno = EIO;
+                goto out;
+        }
+
+        if (local->cached_subvol) {
+                ret = dht_layout_preset (this, local->cached_subvol,
+                                         local->inode);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to set layout for subvolume %s",
+                                local->cached_subvol ? local->cached_subvol->name : "<nil>");
+                        op_errno = EINVAL;
+                        goto out;
+                }
+        } else {
+                ret = dht_layout_normalize (this, &local->loc, layout);
+                if ((ret < 0) || ((ret > 0) && (local->op_ret != 0))) {
+                        /* either the layout is incorrect or the directory is
+                         * not found even in one subvolume.
+                         */
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "normalizing failed on %s "
+                                "(overlaps/holes present: %s, "
+                                "ENOENT errors: %d)", local->loc.path,
+                                (ret < 0) ? "yes" : "no", (ret > 0) ? ret : 0);
+                        if ((ret > 0) && (ret == conf->subvolume_cnt)) {
+                                op_errno = ESTALE;
+                                goto out;
+                        }
+                }
+
+                if (local->inode)
+                        dht_layout_set (this, local->inode, layout);
+        }
+
+        DHT_STACK_UNWIND (lookup, main_frame, local->op_ret, local->op_errno,
+                          local->inode, &local->stbuf, local->xattr,
+                          &local->postparent);
+        return 0;
+out:
+        DHT_STACK_UNWIND (lookup, main_frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL);
+
+        return ret;
+}
+
+
+int
+dht_discover_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int op_ret, int op_errno,
+                  inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+                  struct iatt *postparent)
 {
-	dht_conf_t   *conf                    = NULL;
         dht_local_t  *local                   = NULL;
         int           this_call_cnt           = 0;
         call_frame_t *prev                    = NULL;
-	dht_layout_t *layout                  = NULL;
-	int           ret                     = 0;
-	int           is_dir                  = 0;
+        dht_layout_t *layout                  = NULL;
+        int           ret                     = -1;
+        int           is_dir                  = 0;
+        int           is_linkfile             = 0;
+        int           attempt_unwind          = 0;
+        dht_conf_t   *conf                    = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, out);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
 
-	conf  = this->private;
         local = frame->local;
         prev  = cookie;
+        conf = this->private;
 
-	layout = local->layout;
+        layout = local->layout;
+
+        /* Check if the gfid is different for file from other node */
+        if (!op_ret && uuid_compare (local->gfid, stbuf->ia_gfid)) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: gfid different on %s",
+                        local->loc.path, prev->this->name);
+        }
 
-        if (!op_ret && uuid_is_null (local->gfid))
-                memcpy (local->gfid, stbuf->ia_gfid, 16);
 
         LOCK (&frame->lock);
         {
                 /* TODO: assert equal mode on stbuf->st_mode and
-		   local->stbuf->st_mode
-
-		   else mkdir/chmod/chown and fix
-		*/
-		ret = dht_layout_merge (this, layout, prev->this,
-					op_ret, op_errno, xattr);
-
-		if (op_ret == -1) {
-			local->op_errno = ENOENT;
-			gf_log (this->name, GF_LOG_DEBUG,
-				"lookup of %s on %s returned error (%s)",
-				local->loc.path, prev->this->name,
-				strerror (op_errno));
+                   local->stbuf->st_mode
 
-			goto unlock;
-		}
+                   else mkdir/chmod/chown and fix
+                */
+                ret = dht_layout_merge (this, layout, prev->this,
+                                        op_ret, op_errno, xattr);
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: failed to merge layouts", local->loc.path);
 
- 		is_dir = check_is_dir (inode, stbuf, xattr);
- 		if (!is_dir) {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
                         gf_log (this->name, GF_LOG_DEBUG,
-                                "lookup of %s on %s returned non dir 0%o",
+                                "lookup of %s on %s returned error (%s)",
                                 local->loc.path, prev->this->name,
-                                stbuf->ia_type);
-                        local->need_selfheal = 1;
- 			goto unlock;
+                                strerror (op_errno));
+
+                        goto unlock;
                 }
 
- 		local->op_ret = 0;
- 		if (local->xattr == NULL)
- 			local->xattr = dict_ref (xattr);
- 		if (local->inode == NULL)
- 			local->inode = inode_ref (inode);
+                is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                                 conf->link_xattr_name);
+                is_dir = check_is_dir (inode, stbuf, xattr);
 
-		dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-                dht_iatt_merge (this, &local->postparent, postparent,
-                                prev->this);
+                if (is_dir) {
+                        local->dir_count ++;
+                } else {
+                        local->file_count ++;
+
+                        if (!is_linkfile) {
+                                /* real file */
+                                local->cached_subvol = prev->this;
+                                attempt_unwind = 1;
+                        } else {
+                                goto unlock;
+                        }
+                }
 
-                if (prev->this == dht_first_up_subvol (this)) {
-			local->ia_ino = local->stbuf.ia_ino;
+                local->op_ret = 0;
+
+                if (local->xattr == NULL) {
+                        local->xattr = dict_ref (xattr);
+                } else {
+                        dht_aggregate_xattr (local->xattr, xattr);
                 }
 
+                if (local->inode == NULL)
+                        local->inode = inode_ref (inode);
+
+                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+                dht_iatt_merge (this, &local->postparent, postparent,
+                                prev->this);
         }
 unlock:
         UNLOCK (&frame->lock);
+out:
+        this_call_cnt = dht_frame_return (frame);
 
+        if (is_last_call (this_call_cnt) || attempt_unwind) {
+                dht_discover_complete (this, frame);
+        }
 
-        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt))
+                DHT_STACK_DESTROY (frame);
 
-        if (is_last_call (this_call_cnt)) {
-                if (local->need_selfheal) {
-                        local->need_selfheal = 0;
-                        dht_lookup_everywhere (frame, this, &local->loc);
-                        return 0;
-                }
+        return 0;
+}
 
-		if (local->op_ret == 0) {
-			ret = dht_layout_normalize (this, &local->loc, layout);
 
-			if (ret != 0) {
-				gf_log (this->name, GF_LOG_DEBUG,
-					"fixing assignment on %s",
-					local->loc.path);
-				goto selfheal;
-			}
+int
+dht_discover (call_frame_t *frame, xlator_t *this, loc_t *loc)
+{
+        int          ret;
+        dht_local_t *local = NULL;
+        dht_conf_t  *conf = NULL;
+        int          call_cnt = 0;
+        int          op_errno = EINVAL;
+        int          i = 0;
+        call_frame_t *discover_frame = NULL;
 
-			dht_layout_set (this, local->inode, layout);
+        conf = this->private;
+        local = frame->local;
 
-			if (local->ia_ino) {
-				local->stbuf.ia_ino = local->ia_ino;
-			} else {
-				gf_log (this->name, GF_LOG_DEBUG,
-					"could not find hashed subvol for %s",
-					local->loc.path);
-			}
+        ret = dict_set_uint32 (local->xattr_req, conf->xattr_name, 4 * 4);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to set '%s' key",
+                        loc->path, conf->xattr_name);
 
-                        if (local->loc.parent)
-                                local->postparent.ia_ino =
-                                        local->loc.parent->ino;
-		}
+        ret = dict_set_uint32 (local->xattr_req, conf->link_xattr_name, 256);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to set '%s' key",
+                        loc->path, conf->link_xattr_name);
 
-		DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-				  local->inode, &local->stbuf, local->xattr,
-                                  &local->postparent);
+        call_cnt        = conf->subvolume_cnt;
+        local->call_cnt = call_cnt;
+
+        local->layout = dht_layout_new (this, conf->subvolume_cnt);
+
+        if (!local->layout) {
+                op_errno = ENOMEM;
+                goto err;
         }
 
-	return 0;
+        uuid_copy (local->gfid, loc->gfid);
 
-selfheal:
-        dht_frame_su_do (frame);
-	ret = dht_selfheal_directory (frame, dht_lookup_selfheal_cbk,
-				      &local->loc, layout);
+        discover_frame = copy_frame (frame);
+        if (!discover_frame) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	return 0;
+        discover_frame->local = local;
+        frame->local = NULL;
+        local->main_frame = frame;
+
+        for (i = 0; i < call_cnt; i++) {
+                STACK_WIND (discover_frame, dht_discover_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->lookup,
+                            &local->loc, local->xattr_req);
+        }
+
+        return 0;
+
+err:
+        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL);
+
+        return 0;
 }
 
+
 int
-dht_lookup_root_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno,
-                         inode_t *inode, struct iatt *stbuf, dict_t *xattr,
-                         struct iatt *postparent)
+dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno,
+                    inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+                    struct iatt *postparent)
 {
-	dht_conf_t   *conf                    = NULL;
         dht_local_t  *local                   = NULL;
         int           this_call_cnt           = 0;
         call_frame_t *prev                    = NULL;
-	dht_layout_t *layout                  = NULL;
-	int           ret                     = 0;
-	int           is_dir                  = 0;
+        dht_layout_t *layout                  = NULL;
+        int           ret                     = -1;
+        int           is_dir                  = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, out);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
 
-	conf  = this->private;
         local = frame->local;
         prev  = cookie;
 
-	layout = local->layout;
+        layout = local->layout;
+
+        if (!op_ret && uuid_is_null (local->gfid))
+                memcpy (local->gfid, stbuf->ia_gfid, 16);
+
+        /* Check if the gfid is different for file from other node */
+        if (!op_ret && uuid_compare (local->gfid, stbuf->ia_gfid)) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: gfid different on %s",
+                        local->loc.path, prev->this->name);
+        }
 
         LOCK (&frame->lock);
         {
-		ret = dht_layout_merge (this, layout, prev->this,
-					op_ret, op_errno, xattr);
-
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			gf_log (this->name, GF_LOG_ERROR,
-				"lookup of %s on %s returned error (%s)",
-				local->loc.path, prev->this->name,
-				strerror (op_errno));
-			goto unlock;
-		}
+                /* TODO: assert equal mode on stbuf->st_mode and
+                   local->stbuf->st_mode
+
+                   else mkdir/chmod/chown and fix
+                */
+                ret = dht_layout_merge (this, layout, prev->this,
+                                        op_ret, op_errno, xattr);
+
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "lookup of %s on %s returned error (%s)",
+                                local->loc.path, prev->this->name,
+                                strerror (op_errno));
+
+                        goto unlock;
+                }
 
                 is_dir = check_is_dir (inode, stbuf, xattr);
                 if (!is_dir) {
-                        gf_log (this->name, GF_LOG_CRITICAL,
+                        gf_log (this->name, GF_LOG_DEBUG,
                                 "lookup of %s on %s returned non dir 0%o",
                                 local->loc.path, prev->this->name,
                                 stbuf->ia_type);
+                        local->need_selfheal = 1;
                         goto unlock;
                 }
 
                 local->op_ret = 0;
-                if (local->xattr == NULL)
+                if (local->xattr == NULL) {
                         local->xattr = dict_ref (xattr);
+                } else {
+                        dht_aggregate_xattr (local->xattr, xattr);
+                }
+
                 if (local->inode == NULL)
                         local->inode = inode_ref (inode);
 
-		dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-
-                if (prev->this == dht_first_up_subvol (this)) {
-			local->ia_ino = local->stbuf.ia_ino;
-                }
-
+                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+                dht_iatt_merge (this, &local->postparent, postparent,
+                                prev->this);
         }
 unlock:
         UNLOCK (&frame->lock);
@@ -264,74 +481,45 @@ unlock:
         this_call_cnt = dht_frame_return (frame);
 
         if (is_last_call (this_call_cnt)) {
-		if (local->op_ret == 0) {
-			ret = dht_layout_normalize (this, &local->loc, layout);
-			if (ret != 0) {
-				gf_log (this->name, GF_LOG_INFO,
-					"fixing assignment on %s",
-					local->loc.path);
-			}
-
-			dht_layout_set (this, local->inode, layout);
-		}
-
-		DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-				  local->inode, &local->stbuf, local->xattr,
-                                  &local->postparent);
-        }
-
-	return 0;
-}
-
-static int
-dht_do_fresh_lookup_on_root (xlator_t *this, call_frame_t *frame)
-{
-        dht_local_t  *local    = NULL;
-	dht_conf_t   *conf     = NULL;
-	int           ret      = -1;
-        int           call_cnt = 0;
-        int           i        = 0;
-
-        local = frame->local;
-        conf = this->private;
-        if (!conf)
-                goto err;
+                if (local->need_selfheal) {
+                        local->need_selfheal = 0;
+                        dht_lookup_everywhere (frame, this, &local->loc);
+                        return 0;
+                }
 
-        if (local->layout) {
-                dht_layout_unref (this, local->layout);
-                local->layout = NULL;
-        }
+                if (local->op_ret == 0) {
+                        ret = dht_layout_normalize (this, &local->loc, layout);
 
-        ret = dict_set_uint32 (local->xattr_req,
-                               "trusted.glusterfs.dht", 4 * 4);
-        if (ret)
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failed to set the dict entry for dht");
+                        if (ret != 0) {
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "fixing assignment on %s",
+                                        local->loc.path);
+                                goto selfheal;
+                        }
 
-        call_cnt = local->call_cnt = conf->subvolume_cnt;
+                        dht_layout_set (this, local->inode, layout);
+                }
 
-        local->layout = dht_layout_new (this,
-                                        conf->subvolume_cnt);
-        if (!local->layout) {
-                local->op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                goto err;
-        }
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
 
-        for (i = 0; i < call_cnt; i++) {
-                STACK_WIND (frame, dht_lookup_root_dir_cbk,
-                            conf->subvolumes[i],
-                            conf->subvolumes[i]->fops->lookup,
-                            &local->loc, local->xattr_req);
+                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+                DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
+                                  local->inode, &local->stbuf, local->xattr,
+                                  &local->postparent);
         }
 
         return 0;
-err:
-        DHT_STACK_UNWIND (lookup, frame, -1, local->op_errno,
-                          local->inode, &local->stbuf, local->xattr,
-                          &local->postparent);
-        return 0;
+
+selfheal:
+        FRAME_SU_DO (frame, dht_local_t);
+        uuid_copy (local->loc.gfid, local->gfid);
+        ret = dht_selfheal_directory (frame, dht_lookup_selfheal_cbk,
+                                      &local->loc, layout);
+out:
+        return ret;
 }
 
 int
@@ -343,152 +531,226 @@ dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         dht_local_t  *local         = NULL;
         int           this_call_cnt = 0;
         call_frame_t *prev          = NULL;
-	dht_layout_t *layout        = NULL;
-	dht_conf_t   *conf          = NULL;
-	int           ret  = -1;
-	int           is_dir = 0;
-	int           is_linkfile = 0;
-        unsigned char root_gfid[16] = {0,};
+        dht_layout_t *layout        = NULL;
+        dht_conf_t   *conf          = NULL;
+        int           ret  = -1;
+        int           is_dir = 0;
+        int           is_linkfile = 0;
+        call_frame_t *copy          = NULL;
+        dht_local_t  *copy_local    = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, err);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, err);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, err);
 
         local = frame->local;
         prev  = cookie;
-	conf = this->private;
+        conf = this->private;
         if (!conf)
                 goto out;
 
         LOCK (&frame->lock);
         {
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
 
-			if ((op_errno != ENOTCONN) 
+                        if ((op_errno != ENOTCONN)
                             && (op_errno != ENOENT)
                             && (op_errno != ESTALE)) {
 				gf_log (this->name, GF_LOG_INFO,
-					"subvolume %s returned -1 (%s)",
-					prev->this->name, strerror (op_errno));
+					"subvolume %s for %s returned -1 (%s)",
+					prev->this->name, local->loc.path,
+                                        strerror (op_errno));
 			}
-                        
                         if (op_errno == ESTALE) {
-                                /* propogate the ESTALE to parent. 
-                                 * setting local->layout_mismatch would send
+                                /* propagate the ESTALE to parent.
+                                 * setting local->return_estale would send
                                  * ESTALE to parent. */
-                                local->layout_mismatch = 1;
+                                local->return_estale = 1;
                         }
 
-			goto unlock;
-		}
+                        /* if it is ENOENT, we may have to do a
+                         * 'lookup_everywhere()' to make sure
+                         * the file is not migrated */
+                        if (op_errno == ENOENT) {
+                                if (IA_ISREG (local->loc.inode->ia_type)) {
+                                        local->need_lookup_everywhere = 1;
+                                }
+                        }
+                        goto unlock;
+                }
 
-		if (stbuf->ia_type != local->inode->ia_type) {
-			gf_log (this->name, GF_LOG_INFO,
-				"mismatching filetypes 0%o v/s 0%o for %s",
-				(stbuf->ia_type), (local->inode->ia_type),
-				local->loc.path);
+                if (stbuf->ia_type != local->inode->ia_type) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "mismatching filetypes 0%o v/s 0%o for %s",
+                                (stbuf->ia_type), (local->inode->ia_type),
+                                local->loc.path);
 
-			local->op_ret = -1;
-			local->op_errno = EINVAL;
+                        local->op_ret = -1;
+                        local->op_errno = EINVAL;
 
-			goto unlock;
-		}
+                        goto unlock;
+                }
 
-		layout = local->layout;
-		
-		is_dir = check_is_dir (inode, stbuf, xattr);
-		is_linkfile = check_is_linkfile (inode, stbuf, xattr);
-		
-		if (is_linkfile) {
-			gf_log (this->name, GF_LOG_INFO,
-				"linkfile found in revalidate for %s",
-				local->loc.path);
-			local->layout_mismatch = 1;
-
-			goto unlock;
-		}
+                layout = local->layout;
 
-		if (is_dir) {
-			ret = dht_layout_dir_mismatch (this, layout,
-						       prev->this, &local->loc,
-						       xattr);
-			if (ret != 0) {
-				gf_log (this->name, GF_LOG_INFO,
-					"mismatching layouts for %s", 
-					local->loc.path);
-			
-				local->layout_mismatch = 1;
+                is_dir = check_is_dir (inode, stbuf, xattr);
+                is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                                 conf->link_xattr_name);
 
-				goto unlock;
-			}
-		} 
-		
-		dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+                if (is_linkfile) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "linkfile found in revalidate for %s",
+                                local->loc.path);
+                        local->return_estale = 1;
+
+                        goto unlock;
+                }
+
+                if (is_dir) {
+                        ret = dht_dir_has_layout (xattr, conf->xattr_name);
+                        if (ret >= 0) {
+                                if (is_greater_time(local->stbuf.ia_ctime,
+                                                    local->stbuf.ia_ctime_nsec,
+                                                    stbuf->ia_ctime,
+                                                    stbuf->ia_ctime_nsec)) {
+                                        local->prebuf.ia_gid = stbuf->ia_gid;
+                                        local->prebuf.ia_uid = stbuf->ia_uid;
+                                }
+                        }
+                        if (local->stbuf.ia_type != IA_INVAL)
+                        {
+                                if ((local->stbuf.ia_gid != stbuf->ia_gid) ||
+                                    (local->stbuf.ia_uid != stbuf->ia_uid)) {
+                                        local->need_selfheal = 1;
+                                }
+                        }
+                        ret = dht_layout_dir_mismatch (this, layout,
+                                                       prev->this, &local->loc,
+                                                       xattr);
+                        if (ret != 0) {
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "mismatching layouts for %s",
+                                        local->loc.path);
+
+                                local->layout_mismatch = 1;
+
+                                goto unlock;
+                        }
+                }
+
+                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
                 dht_iatt_merge (this, &local->postparent, postparent,
                                 prev->this);
-		
-		local->op_ret = 0;
-		local->stbuf.ia_ino = local->ia_ino;
 
-                if (local->loc.parent)
-                        local->postparent.ia_ino = local->loc.parent->ino;
+                local->op_ret = 0;
 
-		if (!local->xattr)
-			local->xattr = dict_ref (xattr);
-	}
+                if (!local->xattr) {
+                        local->xattr = dict_ref (xattr);
+                } else if (is_dir) {
+                        dht_aggregate_xattr (local->xattr, xattr);
+                }
+        }
 unlock:
-	UNLOCK (&frame->lock);
+        UNLOCK (&frame->lock);
 out:
         this_call_cnt = dht_frame_return (frame);
 
         if (is_last_call (this_call_cnt)) {
-		if (!IA_ISDIR (local->stbuf.ia_type)
-		    && (local->hashed_subvol != local->cached_subvol)
-		    && (local->stbuf.ia_nlink == 1)
+                if (!IA_ISDIR (local->stbuf.ia_type)
+                    && (local->hashed_subvol != local->cached_subvol)
+                    && (local->stbuf.ia_nlink == 1)
                     && (conf && conf->unhashed_sticky_bit)) {
-			local->stbuf.ia_prot.sticky = 1;
-		}
-
-		if (local->layout_mismatch) {
-			local->op_ret = -1;
-			local->op_errno = ESTALE;
-
-                        /* Because for 'root' inode, there is no FRESH lookup
-                         * sent from FUSE layer upon ESTALE, we need to handle
-                         * that one case here */
-                        root_gfid[15] = 1;
-                        if (!local->loc.parent &&
-                            !uuid_compare (local->loc.inode->gfid, root_gfid)) {
-                                dht_do_fresh_lookup_on_root (this, frame);
-                                return 0;
+                        local->stbuf.ia_prot.sticky = 1;
+                }
+                if (local->need_selfheal) {
+                        local->need_selfheal = 0;
+                        uuid_copy (local->gfid, local->stbuf.ia_gfid);
+                        local->stbuf.ia_gid = local->prebuf.ia_gid;
+                        local->stbuf.ia_uid = local->prebuf.ia_uid;
+                        copy = create_frame (this, this->ctx->pool);
+                        if (copy) {
+                                copy_local = dht_local_init (copy, &local->loc,
+                                                             NULL, 0);
+                                if (!copy_local)
+                                        goto cont;
+                                copy_local->stbuf = local->stbuf;
+                                copy->local = copy_local;
+                                FRAME_SU_DO (copy, dht_local_t);
+                                ret = synctask_new (this->ctx->env,
+                                                    dht_dir_attr_heal,
+                                                    dht_dir_attr_heal_done,
+                                                    copy, copy);
                         }
-		}
+                }
+cont:
+		if (local->layout_mismatch) {
+                        /* Found layout mismatch in the directory, need to
+                           fix this in the inode context */
+                        dht_layout_unref (this, local->layout);
+                        local->layout = NULL;
+                        dht_lookup_directory (frame, this, &local->loc);
+                        return 0;
+                }
+
+                if (local->need_lookup_everywhere) {
+                        /* As the current layout gave ENOENT error, we would
+                           need a new layout */
+                        dht_layout_unref (this, local->layout);
+                        local->layout = NULL;
+
+                        /* We know that current cached subvol is no more
+                           valid, get the new one */
+                        local->cached_subvol = NULL;
+                        dht_lookup_everywhere (frame, this, &local->loc);
+                        return 0;
+                }
+                if (local->return_estale) {
+                        local->op_ret = -1;
+                        local->op_errno = ESTALE;
+                }
 
-                WIPE (&local->postparent);
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
 
+                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
                 DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-				  local->inode, &local->stbuf, local->xattr,
+                                  local->inode, &local->stbuf, local->xattr,
                                   &local->postparent);
-	}
+        }
 
-        return 0;
+err:
+        return ret;
 }
 
 
 int
 dht_lookup_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-				xlator_t *this,
-				int32_t op_ret, int32_t op_errno,
+                                xlator_t *this,
+                                int32_t op_ret, int32_t op_errno,
                                 inode_t *inode, struct iatt *stbuf,
-                                struct iatt *preparent, struct iatt *postparent)
+                                struct iatt *preparent, struct iatt *postparent,
+                                dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	xlator_t     *cached_subvol = NULL;
-	dht_conf_t   *conf = NULL;
+        dht_local_t  *local = NULL;
+        xlator_t     *cached_subvol = NULL;
+        dht_conf_t   *conf = NULL;
         int           ret = -1;
 
-	local = frame->local;
-	cached_subvol = local->cached_subvol;
-	conf = this->private;
+        GF_VALIDATE_OR_GOTO ("dht", frame, out);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
 
-        ret = dht_layout_preset (this, local->cached_subvol, inode);
+        local = frame->local;
+        cached_subvol = local->cached_subvol;
+        conf = this->private;
+
+        ret = dht_layout_preset (this, local->cached_subvol, local->loc.inode);
         if (ret < 0) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "failed to set layout for subvolume %s",
@@ -498,74 +760,232 @@ dht_lookup_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                 goto unwind;
         }
 
-	local->op_ret = 0;
-	if ((local->stbuf.ia_nlink == 1)
-	    && (conf && conf->unhashed_sticky_bit)) {
-		local->stbuf.ia_prot.sticky = 1;
-	}
+        local->op_ret = 0;
+        if ((local->stbuf.ia_nlink == 1)
+            && (conf && conf->unhashed_sticky_bit)) {
+                local->stbuf.ia_prot.sticky = 1;
+        }
 
-        if (local->loc.parent)
-                local->postparent.ia_ino = local->loc.parent->ino;
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
 
 unwind:
-        WIPE (&local->postparent);
+        if (local->linked == _gf_true)
+                dht_linkfile_attr_heal (frame, this);
 
-	DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
-			  local->inode, &local->stbuf, local->xattr,
+        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+        DHT_STACK_UNWIND (lookup, frame, local->op_ret, local->op_errno,
+                          local->inode, &local->stbuf, local->xattr,
                           &local->postparent);
-	return 0;
+out:
+        return ret;
+}
+
+
+int
+dht_lookup_everywhere_done (call_frame_t *frame, xlator_t *this)
+{
+        int           ret = 0;
+        dht_local_t  *local = NULL;
+        xlator_t     *hashed_subvol = NULL;
+        xlator_t     *cached_subvol = NULL;
+        dht_layout_t *layout = NULL;
+
+        local = frame->local;
+        hashed_subvol = local->hashed_subvol;
+        cached_subvol = local->cached_subvol;
+
+        if (local->file_count && local->dir_count) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "path %s exists as a file on one subvolume "
+                        "and directory on another. "
+                        "Please fix it manually",
+                        local->loc.path);
+                DHT_STACK_UNWIND (lookup, frame, -1, EIO, NULL, NULL, NULL,
+                                  NULL);
+                return 0;
+        }
+
+        if (local->dir_count) {
+                dht_lookup_directory (frame, this, &local->loc);
+                return 0;
+        }
+
+        if (!cached_subvol) {
+                DHT_STACK_UNWIND (lookup, frame, -1, ENOENT, NULL, NULL, NULL,
+                                  NULL);
+                return 0;
+        }
+
+        if (local->need_lookup_everywhere) {
+                if (uuid_compare (local->gfid, local->inode->gfid)) {
+                        /* GFID different, return error */
+                        DHT_STACK_UNWIND (lookup, frame, -1, ENOENT, NULL,
+                                          NULL, NULL, NULL);
+                        return 0;
+                }
+                local->op_ret = 0;
+                local->op_errno = 0;
+                layout = dht_layout_for_subvol (this, cached_subvol);
+                if (!layout) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "%s: no pre-set layout for subvolume %s",
+                                local->loc.path, (cached_subvol ?
+                                                  cached_subvol->name :
+                                                  "<nil>"));
+                }
+
+                ret = dht_layout_set (this, local->inode, layout);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "%s: failed to set layout for subvol %s",
+                                local->loc.path, (cached_subvol ?
+                                                  cached_subvol->name :
+                                                  "<nil>"));
+                }
+
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
+
+                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+                DHT_STACK_UNWIND (lookup, frame, local->op_ret,
+                                  local->op_errno, local->inode,
+                                  &local->stbuf, local->xattr,
+                                  &local->postparent);
+                return 0;
+        }
+
+        if (!hashed_subvol) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "cannot create linkfile file for %s on %s: "
+                        "hashed subvolume cannot be found.",
+                        local->loc.path, cached_subvol->name);
+
+                local->op_ret = 0;
+                local->op_errno = 0;
+
+                ret = dht_layout_preset (frame->this, cached_subvol,
+                                         local->inode);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "failed to set layout for subvol %s",
+                                cached_subvol ? cached_subvol->name :
+                                "<nil>");
+                        local->op_ret = -1;
+                        local->op_errno = EINVAL;
+                }
+
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
+
+                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+                DHT_STACK_UNWIND (lookup, frame, local->op_ret,
+                                  local->op_errno, local->inode,
+                                  &local->stbuf, local->xattr,
+                                  &local->postparent);
+                return 0;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "linking file %s existing on %s to %s (hash)",
+                local->loc.path, cached_subvol->name,
+                hashed_subvol->name);
+
+        ret = dht_linkfile_create (frame,
+                                   dht_lookup_linkfile_create_cbk, this,
+                                   cached_subvol, hashed_subvol, &local->loc);
+
+        return ret;
+}
+
+
+int
+dht_lookup_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno,
+                       struct iatt *preparent, struct iatt *postparent,
+                       dict_t *xdata)
+{
+        int  this_call_cnt = 0;
+
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+                dht_lookup_everywhere_done (frame, this);
+        }
+
+        return 0;
 }
 
 
 int
 dht_lookup_everywhere_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int32_t op_ret, int32_t op_errno,
+                           int32_t op_ret, int32_t op_errno,
                            inode_t *inode, struct iatt *buf, dict_t *xattr,
                            struct iatt *postparent)
 {
-	dht_conf_t   *conf          = NULL;
         dht_local_t  *local         = NULL;
         int           this_call_cnt = 0;
         call_frame_t *prev          = NULL;
-	int           is_linkfile   = 0;
-	int           is_dir        = 0;
-	xlator_t     *subvol        = NULL;
-	loc_t        *loc           = NULL;
-	xlator_t     *link_subvol   = NULL;
-	xlator_t     *hashed_subvol = NULL;
-	xlator_t     *cached_subvol = NULL;
-        int           ret = -1;
+        int           is_linkfile   = 0;
+        int           is_dir        = 0;
+        xlator_t     *subvol        = NULL;
+        loc_t        *loc           = NULL;
+        xlator_t     *link_subvol   = NULL;
+        int           ret           = -1;
+        int32_t       fd_count      = 0;
+        dht_conf_t   *conf          = NULL;
 
-	conf   = this->private;
+        GF_VALIDATE_OR_GOTO ("dht", frame, out);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
 
-	local  = frame->local;
-	loc    = &local->loc;
+        local  = frame->local;
+        loc    = &local->loc;
+        conf   = this->private;
 
-	prev   = cookie;
-	subvol = prev->this;
+        prev   = cookie;
+        subvol = prev->this;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        if (op_errno != ENOENT)
+                                local->op_errno = op_errno;
+                        goto unlock;
+                }
 
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			if (op_errno != ENOENT)
-				local->op_errno = op_errno;
-			goto unlock;
-		}
                 if (uuid_is_null (local->gfid))
-                        memcpy (local->gfid, buf->ia_gfid, 16);
-
-		is_linkfile = check_is_linkfile (inode, buf, xattr);
-		is_dir = check_is_dir (inode, buf, xattr);
-
-		if (is_linkfile) {
-			link_subvol = dht_linkfile_subvol (this, inode, buf,
-							   xattr);
-			gf_log (this->name, GF_LOG_DEBUG,
-				"found on %s linkfile %s (-> %s)",
-				subvol->name, loc->path,
-				link_subvol ? link_subvol->name : "''");
-			goto unlock;
-		}
+                        uuid_copy (local->gfid, buf->ia_gfid);
+
+                if (uuid_compare (local->gfid, buf->ia_gfid)) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: gfid differs on subvolume %s",
+                                loc->path, prev->this->name);
+                }
+
+                is_linkfile = check_is_linkfile (inode, buf, xattr,
+                                                 conf->link_xattr_name);
+                is_dir = check_is_dir (inode, buf, xattr);
+
+                if (is_linkfile) {
+                        link_subvol = dht_linkfile_subvol (this, inode, buf,
+                                                           xattr);
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "found on %s linkfile %s (-> %s)",
+                                subvol->name, loc->path,
+                                link_subvol ? link_subvol->name : "''");
+                        goto unlock;
+                }
+
+                /* non linkfile GFID takes precedence */
+                uuid_copy (local->gfid, buf->ia_gfid);
 
                 if (is_dir) {
                         local->dir_count++;
@@ -585,131 +1005,82 @@ dht_lookup_everywhere_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                 gf_log (this->name, GF_LOG_DEBUG,
                                         "found on %s file %s",
                                         subvol->name, loc->path);
-                                
+
                                 dht_iatt_merge (this, &local->postparent,
                                                 postparent, subvol);
                         } else {
-                                gf_log (this->name, GF_LOG_DEBUG,
+                                /* This is where we need 'rename' both entries logic */
+                                gf_log (this->name, GF_LOG_WARNING,
                                         "multiple subvolumes (%s and %s) have "
-                                        "file %s", local->cached_subvol->name,
+                                        "file %s (preferably rename the file "
+                                        "in the backend, and do a fresh lookup)",
+                                        local->cached_subvol->name,
                                         subvol->name, local->loc.path);
                         }
                 }
-	}
+        }
 unlock:
-	UNLOCK (&frame->lock);
-
-	if (is_linkfile) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"deleting stale linkfile %s on %s",
-			loc->path, subvol->name);
-		dht_linkfile_unlink (frame, this, subvol, loc);
-	}
-
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt)) {
-		hashed_subvol = local->hashed_subvol;
-		cached_subvol = local->cached_subvol;
-
-                if (local->file_count && local->dir_count) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "path %s exists as a file on one subvolume " 
-                                "and directory on another. "
-                                "Please fix it manually",
-                                loc->path);
-                        DHT_STACK_UNWIND (lookup, frame, -1, EIO, NULL, NULL, NULL,
-                                          NULL);
-                        return 0;
-                }
-
-                if (local->dir_count) {
-                        dht_lookup_directory (frame, this, &local->loc);
-                        return 0;
-                }
-
-		if (!cached_subvol) {
-			DHT_STACK_UNWIND (lookup, frame, -1, ENOENT, NULL, NULL, NULL,
-                                          NULL);
-			return 0;
-		}
-
-                if (!hashed_subvol) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "cannot create linkfile file for %s on %s: "
-                                "hashed subvolume cannot be found.",
-                                loc->path, cached_subvol->name);
-                        
-                        local->op_ret = 0;
-                        local->op_errno = 0;
-
-                        ret = dht_layout_preset (frame->this, cached_subvol,
-                                                 local->inode);
-                        if (ret < 0) {
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "failed to set layout for subvol %s",
-                                        cached_subvol ? cached_subvol->name :
-                                        "<nil>");
-                                local->op_ret = -1;
-                                local->op_errno = EINVAL;
-                        }
-
-                        if (local->loc.parent)
-                                local->postparent.ia_ino =
-                                        local->loc.parent->ino;
-
-                        WIPE (&local->postparent);
+        UNLOCK (&frame->lock);
 
-                        DHT_STACK_UNWIND (lookup, frame, local->op_ret,
-                                          local->op_errno, local->inode,
-                                          &local->stbuf, local->xattr,
-                                          &local->postparent);
+        if (is_linkfile) {
+                ret = dict_get_int32 (xattr, GLUSTERFS_OPEN_FD_COUNT, &fd_count);
+                /* Delete the linkfile only if there are no open fds on it.
+                   if there is a open-fd, it may be in migration */
+                if (!ret && (fd_count == 0)) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "deleting stale linkfile %s on %s",
+                                loc->path, subvol->name);
+                        STACK_WIND (frame, dht_lookup_unlink_cbk,
+                                    subvol, subvol->fops->unlink, loc, 0, NULL);
                         return 0;
                 }
+        }
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "linking file %s existing on %s to %s (hash)",
-                        loc->path, cached_subvol->name,
-                        hashed_subvol->name);
-                        
-                dht_linkfile_create (frame, 
-                                     dht_lookup_linkfile_create_cbk,
-                                     cached_subvol, hashed_subvol, loc);
-	}
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+                dht_lookup_everywhere_done (frame, this);
+        }
 
-	return 0;
+out:
+        return ret;
 }
 
 
 int
 dht_lookup_everywhere (call_frame_t *frame, xlator_t *this, loc_t *loc)
 {
-	dht_conf_t     *conf = NULL;
-	dht_local_t    *local = NULL;
-	int             i = 0;
-	int             call_cnt = 0;
+        dht_conf_t     *conf = NULL;
+        dht_local_t    *local = NULL;
+        int             i = 0;
+        int             call_cnt = 0;
 
-	conf = this->private;
-	local = frame->local;
-        if (!conf)
-                goto out;
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
+        GF_VALIDATE_OR_GOTO ("dht", loc, out);
 
-	call_cnt = conf->subvolume_cnt;
-	local->call_cnt = call_cnt;
+        conf = this->private;
+        local = frame->local;
 
-	if (!local->inode)
-		local->inode = inode_ref (loc->inode);
+        call_cnt = conf->subvolume_cnt;
+        local->call_cnt = call_cnt;
 
-	for (i = 0; i < call_cnt; i++) {
-		STACK_WIND (frame, dht_lookup_everywhere_cbk,
-			    conf->subvolumes[i],
-			    conf->subvolumes[i]->fops->lookup,
-			    loc, local->xattr_req);
-	}
+        if (!local->inode)
+                local->inode = inode_ref (loc->inode);
+
+        for (i = 0; i < call_cnt; i++) {
+                STACK_WIND (frame, dht_lookup_everywhere_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->lookup,
+                            loc, local->xattr_req);
+        }
 
         return 0;
 out:
         DHT_STACK_UNWIND (lookup, frame, -1, EINVAL, NULL, NULL, NULL, NULL);
-        return 0;
+err:
+        return -1;
 }
 
 
@@ -720,60 +1091,82 @@ dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
                          struct iatt *postparent)
 {
         call_frame_t *prev          = NULL;
-	dht_local_t  *local         = NULL;
-	xlator_t     *subvol        = NULL;
-	loc_t        *loc           = NULL;
-	dht_conf_t   *conf          = NULL;
+        dht_local_t  *local         = NULL;
+        xlator_t     *subvol        = NULL;
+        loc_t        *loc           = NULL;
+        dht_conf_t   *conf          = NULL;
         int           ret           = 0;
 
+        GF_VALIDATE_OR_GOTO ("dht", frame, out);
+        GF_VALIDATE_OR_GOTO ("dht", this, unwind);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, unwind);
+        GF_VALIDATE_OR_GOTO ("dht", this->private, unwind);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, unwind);
+
         prev   = cookie;
-	subvol = prev->this;
-	conf   = this->private;
-	local  = frame->local;
-	loc    = &local->loc;
+        subvol = prev->this;
+        conf   = this->private;
+        local  = frame->local;
+        loc    = &local->loc;
 
         if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"lookup of %s on %s (following linkfile) failed (%s)",
-			local->loc.path, subvol->name, strerror (op_errno));
-                goto err;
-	}
+                gf_log (this->name, GF_LOG_INFO,
+                        "lookup of %s on %s (following linkfile) failed (%s)",
+                        local->loc.path, subvol->name, strerror (op_errno));
+
+                /* If cached subvol returned ENOTCONN, do not do
+                lookup_everywhere. We need to make sure linkfile does not get
+                removed, which can take away the namespace, and subvol is
+                anyways down. */
+
+                if (op_errno != ENOTCONN)
+                        goto err;
+                else
+                        goto unwind;
+        }
 
         if (check_is_dir (inode, stbuf, xattr)) {
-                gf_log (this->name, GF_LOG_DEBUG,
+                gf_log (this->name, GF_LOG_INFO,
                         "lookup of %s on %s (following linkfile) reached dir",
                         local->loc.path, subvol->name);
                 goto err;
         }
 
-        if (check_is_linkfile (inode, stbuf, xattr)) {
-                gf_log (this->name, GF_LOG_DEBUG,
+        if (check_is_linkfile (inode, stbuf, xattr, conf->link_xattr_name)) {
+                gf_log (this->name, GF_LOG_INFO,
                         "lookup of %s on %s (following linkfile) reached link",
                         local->loc.path, subvol->name);
                 goto err;
         }
 
-	if ((stbuf->ia_nlink == 1)
-	    && (conf && conf->unhashed_sticky_bit)) {
-		stbuf->ia_prot.sticky = 1;
-	}
-        dht_itransform (this, prev->this, stbuf->ia_ino, &stbuf->ia_ino);
-        if (local->loc.parent)
-                postparent->ia_ino = local->loc.parent->ino;
-        
-	ret = dht_layout_preset (this, prev->this, inode);
-	if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
+        if (uuid_compare (local->gfid, stbuf->ia_gfid)) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: gfid different on data file on %s",
+                        local->loc.path, subvol->name);
+                goto err;
+        }
+
+        if ((stbuf->ia_nlink == 1)
+            && (conf && conf->unhashed_sticky_bit)) {
+                stbuf->ia_prot.sticky = 1;
+        }
+
+        ret = dht_layout_preset (this, prev->this, inode);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_INFO,
                         "failed to set layout for subvolume %s",
                         prev->this->name);
-		op_ret   = -1;
-		op_errno = EINVAL;
-		goto out;
-	}
+                op_ret   = -1;
+                op_errno = EINVAL;
+        }
 
-out:
-        WIPE (postparent);
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
 
+unwind:
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
                           postparent);
 
@@ -781,7 +1174,7 @@ out:
 
 err:
         dht_lookup_everywhere (frame, this, loc);
-
+out:
         return 0;
 }
 
@@ -793,23 +1186,38 @@ dht_lookup_directory (call_frame_t *frame, xlator_t *this, loc_t *loc)
         int           i = 0;
         dht_conf_t   *conf = NULL;
         dht_local_t  *local = NULL;
+        int           ret = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, out);
+        GF_VALIDATE_OR_GOTO ("dht", this, unwind);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, unwind);
+        GF_VALIDATE_OR_GOTO ("dht", this->private, unwind);
+        GF_VALIDATE_OR_GOTO ("dht", loc, unwind);
 
         conf = this->private;
         local = frame->local;
 
-        if (!conf)
-                goto unwind;
-
         call_cnt        = conf->subvolume_cnt;
         local->call_cnt = call_cnt;
 
         local->layout = dht_layout_new (this, conf->subvolume_cnt);
         if (!local->layout) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
                 goto unwind;
         }
 
+        if (local->xattr != NULL) {
+                dict_unref (local->xattr);
+                local->xattr = NULL;
+        }
+
+        if (!uuid_is_null (local->gfid)) {
+                ret = dict_set_static_bin (local->xattr_req, "gfid-req",
+                                           local->gfid, 16);
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: failed to set gfid", local->loc.path);
+        }
+
         for (i = 0; i < call_cnt; i++) {
                 STACK_WIND (frame, dht_lookup_dir_cbk,
                             conf->subvolumes[i],
@@ -819,6 +1227,7 @@ dht_lookup_directory (call_frame_t *frame, xlator_t *this, loc_t *loc)
         return 0;
 unwind:
         DHT_STACK_UNWIND (lookup, frame, -1, ENOMEM, NULL, NULL, NULL, NULL);
+out:
         return 0;
 
 }
@@ -838,12 +1247,15 @@ dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         loc_t        *loc           = NULL;
         call_frame_t *prev          = NULL;
         int           ret           = 0;
-        uint64_t      tmp_layout    = 0;
         dht_layout_t *parent_layout = NULL;
 
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+        GF_VALIDATE_OR_GOTO ("dht", this->private, out);
+
         conf  = this->private;
-        if (!conf)
-                goto out;
 
         prev  = cookie;
         local = frame->local;
@@ -852,99 +1264,137 @@ dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         /* This is required for handling stale linkfile deletion,
          * or any more call which happens from this 'loc'.
          */
-        if (uuid_is_null (local->gfid) && !op_ret)
+        if (!op_ret && uuid_is_null (local->gfid))
                 memcpy (local->gfid, stbuf->ia_gfid, 16);
 
-	if (ENTRY_MISSING (op_ret, op_errno)) {
+        if (ENTRY_MISSING (op_ret, op_errno)) {
+                gf_log (this->name, GF_LOG_TRACE, "Entry %s missing on subvol"
+                        " %s", loc->path, prev->this->name);
                 if (conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_ON) {
-			local->op_errno = ENOENT;
-			dht_lookup_everywhere (frame, this, loc);
-			return 0;
-		}
+                        local->op_errno = ENOENT;
+                        dht_lookup_everywhere (frame, this, loc);
+                        return 0;
+                }
                 if ((conf->search_unhashed == GF_DHT_LOOKUP_UNHASHED_AUTO) &&
                     (loc->parent)) {
-                        ret = inode_ctx_get (loc->parent, this, &tmp_layout);
-                        parent_layout = (dht_layout_t *)(long)tmp_layout;
+                        ret = dht_inode_ctx_layout_get (loc->parent, this,
+                                                        &parent_layout);
+                        if (ret || !parent_layout)
+                                goto out;
                         if (parent_layout->search_unhashed) {
                                 local->op_errno = ENOENT;
                                 dht_lookup_everywhere (frame, this, loc);
                                 return 0;
                         }
                 }
-	}
+        }
 
- 	if (op_ret == 0) {
- 		is_dir      = check_is_dir (inode, stbuf, xattr);
- 		if (is_dir) {
- 			local->inode = inode_ref (inode);
- 			local->xattr = dict_ref (xattr);
- 		}
- 	}
+        if (op_ret == 0) {
+                is_dir      = check_is_dir (inode, stbuf, xattr);
+                if (is_dir) {
+                        local->inode = inode_ref (inode);
+                        local->xattr = dict_ref (xattr);
+                }
+        }
 
- 	if (is_dir || (op_ret == -1 && op_errno == ENOTCONN)) {
+        if (is_dir || (op_ret == -1 && op_errno == ENOTCONN)) {
                 dht_lookup_directory (frame, this, &local->loc);
                 return 0;
- 	}
- 
-        if (op_ret == -1)
+        }
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG, "Lookup of %s for subvolume"
+                       " %s failed with error %s", loc->path, prev->this->name,
+                       strerror (op_errno));
                 goto out;
+        }
 
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr);
-        is_dir      = check_is_dir (inode, stbuf, xattr);
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
 
-        if (!is_dir && !is_linkfile) {
+        if (!is_linkfile) {
                 /* non-directory and not a linkfile */
 
-		dht_itransform (this, prev->this, stbuf->ia_ino,
-				&stbuf->ia_ino);
-                if (loc->parent)
-                        postparent->ia_ino = loc->parent->ino;
-
-		ret = dht_layout_preset (this, prev->this, inode);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"could not set pre-set layout for subvolume %s",
-				prev->this->name);
-			op_ret   = -1;
-			op_errno = EINVAL;
-			goto out;
-		}
-                goto out; 
-	}
-
-        if (is_linkfile) {
-                subvol = dht_linkfile_subvol (this, inode, stbuf, xattr);
-
-                if (!subvol) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "linkfile not having link subvolume. path=%s",
-                                loc->path);
-			dht_lookup_everywhere (frame, this, loc);
-			return 0;
+                ret = dht_layout_preset (this, prev->this, inode);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "could not set pre-set layout for subvolume %s",
+                                prev->this->name);
+                        op_ret   = -1;
+                        op_errno = EINVAL;
+                        goto out;
                 }
+                goto out;
+        }
 
-		STACK_WIND (frame, dht_lookup_linkfile_cbk,
-			    subvol, subvol->fops->lookup,
-			    &local->loc, local->xattr_req);
+        subvol = dht_linkfile_subvol (this, inode, stbuf, xattr);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "linkfile not having link subvolume. path=%s",
+                        loc->path);
+                dht_lookup_everywhere (frame, this, loc);
+                return 0;
         }
 
+        STACK_WIND (frame, dht_lookup_linkfile_cbk,
+                    subvol, subvol->fops->lookup,
+                    &local->loc, local->xattr_req);
+
         return 0;
 
 out:
-        /* 
-         * FIXME: postparent->ia_size and postparent->st_blocks do not have 
-         * correct values. since, postparent corresponds to a directory these 
+        /*
+         * FIXME: postparent->ia_size and postparent->st_blocks do not have
+         * correct values. since, postparent corresponds to a directory these
          * two members should have values equal to sum of corresponding values
          * from each of the subvolume. See dht_iatt_merge for reference.
          */
 
-        WIPE (postparent);
+        if (!op_ret && local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
 
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, stbuf, xattr,
                           postparent);
+err:
         return 0;
 }
 
+/* For directories, check if acl xattrs have been requested (by the acl xlator),
+ * if not, request for them. These xattrs are needed for dht dir self-heal to
+ * perform proper self-healing of dirs
+ */
+void
+dht_check_and_set_acl_xattr_req (inode_t *inode, dict_t *xattr_req)
+{
+        int     ret = 0;
+
+        GF_ASSERT (inode);
+        GF_ASSERT (xattr_req);
+
+        if (inode->ia_type != IA_IFDIR)
+                return;
+
+        if (!dict_get (xattr_req, POSIX_ACL_ACCESS_XATTR)) {
+                ret = dict_set_int8 (xattr_req, POSIX_ACL_ACCESS_XATTR, 0);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "failed to set key %s",
+                                POSIX_ACL_ACCESS_XATTR);
+        }
+
+        if (!dict_get (xattr_req, POSIX_ACL_DEFAULT_XATTR)) {
+                ret = dict_set_int8 (xattr_req, POSIX_ACL_DEFAULT_XATTR, 0);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "failed to set key %s",
+                                POSIX_ACL_DEFAULT_XATTR);
+        }
+
+        return;
+}
 
 int
 dht_lookup (call_frame_t *frame, xlator_t *this,
@@ -952,36 +1402,40 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
 {
         xlator_t     *subvol = NULL;
         xlator_t     *hashed_subvol = NULL;
-        xlator_t     *cached_subvol = NULL;
         dht_local_t  *local  = NULL;
-	dht_conf_t   *conf = NULL;
+        dht_conf_t   *conf = NULL;
         int           ret    = -1;
         int           op_errno = -1;
-	dht_layout_t *layout = NULL;
-	int           i = 0;
-	int           call_cnt = 0;
-
+        dht_layout_t *layout = NULL;
+        int           i = 0;
+        int           call_cnt = 0;
+        loc_t         new_loc = {0,};
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
-	conf = this->private;
+        conf = this->private;
         if (!conf)
                 goto err;
 
-        local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
-        if (!dht_filter_loc_subvol_key (this, loc, &local->loc,
-                                        &hashed_subvol)) {
-                ret = loc_dup (loc, &local->loc);
+        local = dht_local_init (frame, loc, NULL, GF_FOP_LOOKUP);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        ret = dht_filter_loc_subvol_key (this, loc, &new_loc,
+                                         &hashed_subvol);
+        if (ret) {
+                loc_wipe (&local->loc);
+                ret = loc_dup (&new_loc, &local->loc);
+
+                /* we no more need 'new_loc' entries */
+                loc_wipe (&new_loc);
+
+                /* check if loc_dup() is successful */
                 if (ret == -1) {
                         op_errno = errno;
                         gf_log (this->name, GF_LOG_DEBUG,
@@ -991,22 +1445,25 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
                 }
         }
 
-	if (xattr_req) {
-		local->xattr_req = dict_ref (xattr_req);
-	} else {
-		local->xattr_req = dict_new ();
-	}
+        if (xattr_req) {
+                local->xattr_req = dict_ref (xattr_req);
+        } else {
+                local->xattr_req = dict_new ();
+        }
+
+        if (uuid_is_null (loc->pargfid) && !uuid_is_null (loc->gfid) &&
+            !__is_root_gfid (loc->inode->gfid)) {
+                local->cached_subvol = NULL;
+                dht_discover (frame, this, loc);
+                return 0;
+        }
 
         if (!hashed_subvol)
                 hashed_subvol = dht_subvol_get_hashed (this, loc);
-	cached_subvol = dht_subvol_get_cached (this, loc->inode);
-
-	local->cached_subvol = cached_subvol;
-	local->hashed_subvol = hashed_subvol;
+        local->hashed_subvol = hashed_subvol;
 
         if (is_revalidate (loc)) {
-		local->layout = layout = dht_layout_get (this, loc->inode);
-
+                layout = local->layout;
                 if (!layout) {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "revalidate without cache. path=%s",
@@ -1015,71 +1472,93 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
                         goto err;
                 }
 
-		if (layout->gen && (layout->gen < conf->gen)) {
-			gf_log (this->name, GF_LOG_TRACE,
-				"incomplete layout failure for path=%s",
-				loc->path);
+                if (layout->gen && (layout->gen < conf->gen)) {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "incomplete layout failure for path=%s",
+                                loc->path);
 
                         dht_layout_unref (this, local->layout);
                         local->layout = NULL;
-			goto do_fresh_lookup;
-		}
+                        local->cached_subvol = NULL;
+                        goto do_fresh_lookup;
+                }
+
+                local->inode = inode_ref (loc->inode);
+
+                /* NOTE: we don't require 'trusted.glusterfs.dht.linkto' attribute,
+                 *       revalidates directly go to the cached-subvolume.
+                 */
+                ret = dict_set_uint32 (local->xattr_req,
+                                       conf->xattr_name, 4 * 4);
+
+                if (IA_ISDIR (local->inode->ia_type)) {
+                        local->call_cnt = call_cnt = conf->subvolume_cnt;
+                        for (i = 0; i < call_cnt; i++) {
+                                STACK_WIND (frame, dht_revalidate_cbk,
+                                            conf->subvolumes[i],
+                                            conf->subvolumes[i]->fops->lookup,
+                                            loc, local->xattr_req);
+                        }
+                        return 0;
+                }
 
-		local->inode    = inode_ref (loc->inode);
-		local->ia_ino   = loc->inode->ino;
-		
-		local->call_cnt = layout->cnt;
-		call_cnt = local->call_cnt;
-		
-		/* NOTE: we don't require 'trusted.glusterfs.dht.linkto' attribute,
-		 *       revalidates directly go to the cached-subvolume.
-		 */
-		ret = dict_set_uint32 (local->xattr_req, 
-				       "trusted.glusterfs.dht", 4 * 4);
-
-		for (i = 0; i < layout->cnt; i++) {
+                call_cnt = local->call_cnt = layout->cnt;
+
+                /* need it for self-healing linkfiles which is
+                   'in-migration' state */
+                ret = dict_set_uint32 (local->xattr_req,
+                                       GLUSTERFS_OPEN_FD_COUNT, 4);
+
+                /* need it for dir self-heal */
+                dht_check_and_set_acl_xattr_req (loc->inode, local->xattr_req);
+
+		for (i = 0; i < call_cnt; i++) {
 			subvol = layout->list[i].xlator;
-			
+
 			STACK_WIND (frame, dht_revalidate_cbk,
 				    subvol, subvol->fops->lookup,
 				    &local->loc, local->xattr_req);
 
-			if (!--call_cnt)
-				break;
 		}
         } else {
         do_fresh_lookup:
-		/* TODO: remove the hard-coding */
-		ret = dict_set_uint32 (local->xattr_req, 
-				       "trusted.glusterfs.dht", 4 * 4);
+                /* TODO: remove the hard-coding */
+                ret = dict_set_uint32 (local->xattr_req,
+                                       conf->xattr_name, 4 * 4);
 
-		ret = dict_set_uint32 (local->xattr_req, 
-				       "trusted.glusterfs.dht.linkto", 256);
+                ret = dict_set_uint32 (local->xattr_req,
+                                       conf->link_xattr_name, 256);
+
+                /* need it for self-healing linkfiles which is
+                   'in-migration' state */
+                ret = dict_set_uint32 (local->xattr_req,
+                                       GLUSTERFS_OPEN_FD_COUNT, 4);
+
+                /* need it for dir self-heal */
+                dht_check_and_set_acl_xattr_req (loc->inode, local->xattr_req);
 
                 if (!hashed_subvol) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"no subvolume in layout for path=%s, "
-				"checking on all the subvols to see if "
-				"it is a directory", loc->path);
- 			call_cnt        = conf->subvolume_cnt;
- 			local->call_cnt = call_cnt;
- 			
- 			local->layout = dht_layout_new (this,
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "no subvolume in layout for path=%s, "
+                                "checking on all the subvols to see if "
+                                "it is a directory", loc->path);
+                        call_cnt        = conf->subvolume_cnt;
+                        local->call_cnt = call_cnt;
+
+                        local->layout = dht_layout_new (this,
                                                         conf->subvolume_cnt);
- 			if (!local->layout) {
- 				op_errno = ENOMEM;
- 				gf_log (this->name, GF_LOG_ERROR,
- 					"Out of memory");
- 				goto err;
- 			}
-
-			for (i = 0; i < call_cnt; i++) {
- 				STACK_WIND (frame, dht_lookup_dir_cbk,
- 					    conf->subvolumes[i],
- 					    conf->subvolumes[i]->fops->lookup,
- 					    &local->loc, local->xattr_req);
- 			}
- 			return 0;
+                        if (!local->layout) {
+                                op_errno = ENOMEM;
+                                goto err;
+                        }
+
+                        for (i = 0; i < call_cnt; i++) {
+                                STACK_WIND (frame, dht_lookup_dir_cbk,
+                                            conf->subvolumes[i],
+                                            conf->subvolumes[i]->fops->lookup,
+                                            &local->loc, local->xattr_req);
+                        }
+                        return 0;
                 }
 
                 STACK_WIND (frame, dht_lookup_cbk,
@@ -1090,336 +1569,52 @@ dht_lookup (call_frame_t *frame, xlator_t *this,
         return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
-        return 0;
-}
-
-
-int
-dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	      int op_ret, int op_errno, struct iatt *prebuf,
-              struct iatt *postbuf)
-{
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = 0;
-	call_frame_t *prev = NULL;
-
-
-	local = frame->local;
-	prev = cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-                        local->op_ret = -1;
-			gf_log (this->name, GF_LOG_DEBUG,
-				"subvolume %s returned -1 (%s)",
-				prev->this->name, strerror (op_errno));
-			goto unlock;
-		}
-
-                dht_iatt_merge (this, &local->prebuf, prebuf, prev->this);
-		dht_iatt_merge (this, &local->stbuf, postbuf, prev->this);
-
-		if (local->inode) {
-			local->stbuf.ia_ino = local->inode->ino;
-                        local->prebuf.ia_ino = local->inode->ino;
-                }
-
-		local->op_ret = 0;
-	}
-unlock:
-	UNLOCK (&frame->lock);
-
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt))
-		DHT_STACK_UNWIND (truncate, frame, local->op_ret, local->op_errno,
-				  &local->prebuf, &local->stbuf);
-
-        return 0;
-}
-
-
-
-int
-dht_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	      int op_ret, int op_errno, struct iatt *stbuf)
-{
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = 0;
-	call_frame_t *prev = NULL;
-
-
-	local = frame->local;
-	prev = cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			gf_log (this->name, GF_LOG_DEBUG,
-				"subvolume %s returned -1 (%s)",
-				prev->this->name, strerror (op_errno));
-			goto unlock;
-		}
-
-		dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-		
-		if (local->inode)
-			local->stbuf.ia_ino = local->inode->ino;
-		local->op_ret = 0;
-	}
-unlock:
-	UNLOCK (&frame->lock);
-
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt))
-		DHT_STACK_UNWIND (stat, frame, local->op_ret, local->op_errno,
-				  &local->stbuf);
-
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL);
         return 0;
 }
 
 
 int
-dht_stat (call_frame_t *frame, xlator_t *this,
-	  loc_t *loc)
-{
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-	dht_local_t  *local = NULL;
-	dht_layout_t *layout = NULL;
-	int           i = 0;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
-
-	local->layout = layout = dht_layout_get (this, loc->inode);
-	if (!layout) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no layout for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	local->inode = inode_ref (loc->inode);
-	local->call_cnt = layout->cnt;
-
-	for (i = 0; i < layout->cnt; i++) {
-		subvol = layout->list[i].xlator;
-
-		STACK_WIND (frame, dht_attr_cbk,
-			    subvol, subvol->fops->stat,
-			    loc);
-	}
-
-	return 0;
-
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL);
-
-	return 0;
-}
-
-
-int
-dht_fstat (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd)
-{
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-	dht_local_t  *local = NULL;
-	dht_layout_t *layout = NULL;
-	int           i = 0;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
-
-	local->layout = layout = dht_layout_get (this, fd->inode);
-	if (!layout) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no layout for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	local->inode    = inode_ref (fd->inode);
-	local->call_cnt = layout->cnt;;
-
-	for (i = 0; i < layout->cnt; i++) {
-		subvol = layout->list[i].xlator;
-		STACK_WIND (frame, dht_attr_cbk,
-			    subvol, subvol->fops->fstat,
-			    fd);
-	}
-
-	return 0;
-
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (fstat, frame, -1, op_errno, NULL);
-
-	return 0;
-}
-
-
-int
-dht_truncate (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, off_t offset)
-{
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-	dht_local_t  *local = NULL;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = dht_subvol_get_cached (this, loc->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
-
-	local->inode = inode_ref (loc->inode);
-	local->call_cnt = 1;
-
-	STACK_WIND (frame, dht_truncate_cbk,
-		    subvol, subvol->fops->truncate,
-		    loc, offset);
-
-	return 0;
-
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int
-dht_ftruncate (call_frame_t *frame, xlator_t *this,
-	       fd_t *fd, off_t offset)
-{
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-	dht_local_t  *local = NULL;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
-
-	local->inode = inode_ref (fd->inode);
-	local->call_cnt = 1;
-
-	STACK_WIND (frame, dht_truncate_cbk,
-		    subvol, subvol->fops->ftruncate,
-		    fd, offset);
-
-	return 0;
-
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
-
-
-int
 dht_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int op_ret, int op_errno, struct iatt *preparent,
-                struct iatt *postparent)
+                struct iatt *postparent, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	call_frame_t *prev  = NULL;
+        dht_local_t  *local = NULL;
+        call_frame_t *prev  = NULL;
 
-	local = frame->local;
-	prev  = cookie;
+        local = frame->local;
+        prev  = cookie;
 
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
                         local->op_ret   = -1;
-			local->op_errno = op_errno;
-			gf_log (this->name, GF_LOG_DEBUG,
-				"subvolume %s returned -1 (%s)",
-				prev->this->name, strerror (op_errno));
-			goto unlock;
-		}
+                        local->op_errno = op_errno;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "subvolume %s returned -1 (%s)",
+                                prev->this->name, strerror (op_errno));
+                        goto unlock;
+                }
 
-                preparent->ia_ino = local->loc.parent->ino;
-                postparent->ia_ino = local->loc.parent->ino;
-		local->op_ret = 0;
+                local->op_ret = 0;
 
                 local->postparent = *postparent;
                 local->preparent = *preparent;
 
-                WIPE (&local->postparent);
-                WIPE (&local->preparent);
-	}
+                if (local->loc.parent) {
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->preparent, 0);
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
+                }
+        }
 unlock:
-	UNLOCK (&frame->lock);
+        UNLOCK (&frame->lock);
 
         DHT_STACK_UNWIND (unlink, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent);
+                          &local->preparent, &local->postparent, NULL);
 
         return 0;
 }
@@ -1428,32 +1623,33 @@ unlock:
 int
 dht_unlink_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int op_ret, int op_errno, struct iatt *preparent,
-                         struct iatt *postparent)
+                         struct iatt *postparent, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	call_frame_t *prev = NULL;
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
 
         xlator_t *cached_subvol = NULL;
 
-	local = frame->local;
-	prev  = cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			gf_log (this->name, GF_LOG_DEBUG,
-				"subvolume %s returned -1 (%s)",
-				prev->this->name, strerror (op_errno));
-			goto unlock;
-		}
+        local = frame->local;
+        prev  = cookie;
 
-		local->op_ret = 0;
-	}
+        LOCK (&frame->lock);
+        {
+                if ((op_ret == -1) && !((op_errno == ENOENT) ||
+                                        (op_errno == ENOTCONN))) {
+                        local->op_errno = op_errno;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "subvolume %s returned -1 (%s)",
+                                prev->this->name, strerror (op_errno));
+                        goto unlock;
+                }
+
+                local->op_ret = 0;
+        }
 unlock:
-	UNLOCK (&frame->lock);
+        UNLOCK (&frame->lock);
 
-        if (op_ret == -1)
+        if (local->op_ret == -1)
                 goto err;
 
         cached_subvol = dht_subvol_get_cached (this, local->loc.inode);
@@ -1467,321 +1663,314 @@ unlock:
 
         STACK_WIND (frame, dht_unlink_cbk,
                     cached_subvol, cached_subvol->fops->unlink,
-                    &local->loc);
+                    &local->loc, local->flags, NULL);
 
         return 0;
 
 err:
         DHT_STACK_UNWIND (unlink, frame, -1, local->op_errno,
-                          NULL, NULL);
+                          NULL, NULL, NULL);
         return 0;
 }
 
-
 int
-dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-               int op_errno, struct iatt *prebuf, struct iatt *postbuf)
+dht_err_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+             int op_ret, int op_errno, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = 0;
-	call_frame_t *prev = NULL;
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        call_frame_t *prev = NULL;
 
+        local = frame->local;
+        prev = cookie;
 
-	local = frame->local;
-	prev = cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			gf_log (this->name, GF_LOG_DEBUG,
-				"subvolume %s returned -1 (%s)",
-				prev->this->name, strerror (op_errno));
-			goto unlock;
-		}
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "subvolume %s returned -1 (%s)",
+                                prev->this->name, strerror (op_errno));
+                        goto unlock;
+                }
 
-		local->op_ret = 0;
-	}
+                local->op_ret = 0;
+        }
 unlock:
-	UNLOCK (&frame->lock);
+        UNLOCK (&frame->lock);
 
-        if (local && (op_ret == 0)) {
-                prebuf->ia_ino = local->ia_ino;
-                postbuf->ia_ino = local->ia_ino;
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+                DHT_STACK_UNWIND (setxattr, frame, local->op_ret,
+                                  local->op_errno, NULL);
         }
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt))
-		DHT_STACK_UNWIND (fsync, frame, local->op_ret, local->op_errno,
-                                  prebuf, postbuf);
-
         return 0;
 }
 
-
-
-int
-dht_err_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	     int op_ret, int op_errno)
+static void
+fill_layout_info (dht_layout_t *layout, char *buf)
 {
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = 0;
-	call_frame_t *prev = NULL;
-
-
-	local = frame->local;
-	prev = cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			gf_log (this->name, GF_LOG_DEBUG,
-				"subvolume %s returned -1 (%s)",
-				prev->this->name, strerror (op_errno));
-			goto unlock;
-		}
-
-		local->op_ret = 0;
-	}
-unlock:
-	UNLOCK (&frame->lock);
+        int i = 0;
+        char tmp_buf[128] = {0,};
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt)) {
-		DHT_STACK_UNWIND (setxattr, frame, local->op_ret, local->op_errno);
+        for (i = 0; i < layout->cnt; i++) {
+                snprintf (tmp_buf, 128, "(%s %u %u)",
+                          layout->list[i].xlator->name,
+                          layout->list[i].start,
+                          layout->list[i].stop);
+                if (i)
+                        strcat (buf, " ");
+                strcat (buf, tmp_buf);
         }
-
-        return 0;
 }
 
+void
+dht_fill_pathinfo_xattr (xlator_t *this, dht_local_t *local,
+                         char *xattr_buf, int32_t alloc_len,
+                         int flag, char *layout_buf)
+{
+        if (flag && local->xattr_val)
+                snprintf (xattr_buf, alloc_len,
+                          "((<"DHT_PATHINFO_HEADER"%s> %s) (%s-layout %s))",
+                          this->name, local->xattr_val, this->name,
+                          layout_buf);
+        else if (local->xattr_val)
+                snprintf (xattr_buf, alloc_len,
+                          "(<"DHT_PATHINFO_HEADER"%s> %s)",
+                          this->name, local->xattr_val);
+        else if (flag)
+                snprintf (xattr_buf, alloc_len, "(%s-layout %s)",
+                          this->name, layout_buf);
+}
 
 int
-dht_access (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t mask)
+dht_vgetxattr_alloc_and_fill (dht_local_t *local, dict_t *xattr, xlator_t *this,
+                              int op_errno)
 {
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-	dht_local_t  *local = NULL;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
+        int      ret       = -1;
+        char    *value     = NULL;
+        int32_t  plen      = 0;
 
-	subvol = dht_subvol_get_cached (this, loc->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+        ret = dict_get_str (xattr, local->xsel, &value);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Subvolume %s returned -1 (%s)", this->name,
+                        strerror (op_errno));
+                local->op_ret = -1;
+                local->op_errno = op_errno;
+                goto out;
+        }
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
+        local->alloc_len += strlen(value);
 
-	local->call_cnt = 1;
+        if (!local->xattr_val) {
+                local->alloc_len += (strlen (DHT_PATHINFO_HEADER) + 10);
+                local->xattr_val = GF_CALLOC (local->alloc_len, sizeof (char),
+                                              gf_common_mt_char);
+                if (!local->xattr_val) {
+                        ret = -1;
+                        goto out;
+                }
+        }
 
-	STACK_WIND (frame, dht_err_cbk,
-		    subvol, subvol->fops->access,
-		    loc, mask);
+        if (local->xattr_val) {
+                plen = strlen (local->xattr_val);
+                if (plen) {
+                        /* extra byte(s) for \0 to be safe */
+                        local->alloc_len += (plen + 2);
+                        local->xattr_val = GF_REALLOC (local->xattr_val,
+                                                       local->alloc_len);
+                        if (!local->xattr_val) {
+                                ret = -1;
+                                goto out;
+                        }
+                }
 
-	return 0;
+                (void) strcat (local->xattr_val, value);
+                (void) strcat (local->xattr_val, " ");
+                local->op_ret = 0;
+        }
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (access, frame, -1, op_errno);
+        ret = 0;
 
-	return 0;
+ out:
+        return ret;
 }
 
-
 int
-dht_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int op_ret, int op_errno, const char *path, struct iatt *sbuf)
+dht_vgetxattr_fill_and_set (dht_local_t *local, dict_t **dict, xlator_t *this,
+                            gf_boolean_t flag)
 {
-        dht_local_t *local = NULL;
+        int   ret             = -1;
+        char *xattr_buf       = NULL;
+        char layout_buf[8192] = {0,};
 
-        local = frame->local;
-        if (op_ret == -1)
-                goto err;
+        if (flag)
+                fill_layout_info (local->layout, layout_buf);
 
-        if (local) {
-                sbuf->ia_ino = local->ia_ino;
+        *dict = dict_new ();
+        if (!*dict)
+                goto out;
+
+        local->xattr_val[strlen (local->xattr_val) - 1] = '\0';
+
+        /* we would need max this many bytes to create xattr string
+         * extra 40 bytes is just an estimated amount of additional
+         * space required as we include translator name and some
+         * spaces, brackets etc. when forming the pathinfo string.
+         *
+         * For node-uuid we just don't have all the pretty formatting,
+         * but since this is a generic routine for pathinfo & node-uuid
+         * we dont have conditional space allocation and try to be
+         * generic
+         */
+        local->alloc_len += (2 * strlen (this->name))
+                + strlen (layout_buf)
+                + 40;
+        xattr_buf = GF_CALLOC (local->alloc_len, sizeof (char),
+                               gf_common_mt_char);
+        if (!xattr_buf)
+                goto out;
+
+        if (XATTR_IS_PATHINFO (local->xsel)) {
+                (void) dht_fill_pathinfo_xattr (this, local, xattr_buf,
+                                                local->alloc_len, flag,
+                                                layout_buf);
+        } else if (XATTR_IS_NODE_UUID (local->xsel)) {
+                (void) snprintf (xattr_buf, local->alloc_len, "%s",
+                                 local->xattr_val);
         } else {
-                op_ret = -1;
-                op_errno = EINVAL;
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Unknown local->xsel (%s)", local->xsel);
+                GF_FREE (xattr_buf);
+                goto out;
         }
 
-err:
-        DHT_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, sbuf);
+        ret = dict_set_dynstr (*dict, local->xsel, xattr_buf);
+        if (ret)
+                GF_FREE (xattr_buf);
+        GF_FREE (local->xattr_val);
 
-        return 0;
+ out:
+        return ret;
 }
 
-
 int
-dht_readlink (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, size_t size)
+dht_vgetxattr_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
+        int          ret           = 0;
+        dht_local_t *local         = NULL;
+        int          this_call_cnt = 0;
+        dict_t      *dict          = NULL;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (frame->local, out);
 
-	subvol = dht_subvol_get_cached (this, loc->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
+        local = frame->local;
 
-        local->ia_ino = loc->inode->ino;
- 
-	STACK_WIND (frame, dht_readlink_cbk,
-		    subvol, subvol->fops->readlink,
-		    loc, size);
+        LOCK (&frame->lock);
+        {
+                this_call_cnt = --local->call_cnt;
+                if (op_ret < 0) {
+                        if (op_errno != ENOTCONN) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "getxattr err (%s) for dir",
+                                        strerror (op_errno));
+				local->op_ret = -1;
+				local->op_errno = op_errno;
+                        }
 
-	return 0;
+                        goto unlock;
+                }
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL);
+                ret = dht_vgetxattr_alloc_and_fill (local, xattr, this,
+                                                    op_errno);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "alloc or fill failure");
+        }
+ unlock:
+        UNLOCK (&frame->lock);
 
-	return 0;
-}
+        if (!is_last_call (this_call_cnt))
+                goto out;
 
+        /* -- last call: do patch ups -- */
 
-int
-dht_fix_layout_cbk (call_frame_t *frame, void *cookie,
-                    xlator_t *this, int32_t op_ret, int32_t op_errno)
-{
-        DHT_STACK_UNWIND (getxattr, frame, -1, ENODATA, NULL);
+        if (local->op_ret == -1) {
+                goto unwind;
+        }
 
-        return 0;
-}
+        ret = dht_vgetxattr_fill_and_set (local, &dict, this, _gf_true);
+        if (ret)
+                goto unwind;
 
-static void
-fill_layout_info (dht_layout_t *layout, char *buf)
-{
-        int i = 0;
-        char tmp_buf[128] = {0,};
+        DHT_STACK_UNWIND (getxattr, frame, 0, 0, dict, xdata);
+        goto cleanup;
 
-        for (i = 0; i < layout->cnt; i++) {
-                snprintf (tmp_buf, 128, "(%s %u %u)",
-                          layout->list[i].xlator->name,
-                          layout->list[i].start,
-                          layout->list[i].stop);
-                if (i)
-                        strcat (buf, " ");
-                strcat (buf, tmp_buf);
-        }
+ unwind:
+        DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno, NULL, NULL);
+ cleanup:
+        if (dict)
+                dict_unref (dict);
+ out:
+        return 0;
 }
 
 int
-dht_pathinfo_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr)
+dht_vgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-        int ret = 0;
-        int flag = 0;
-        int this_call_cnt = 0;
-        char *value_got = NULL;
-        char  layout_buf[8192] = {0,};
-        char  xattr_buf[8192 + 1024] = {0,};
-        dict_t *dict = NULL;
+        dht_local_t  *local         = NULL;
+        int           ret           = 0;
+        dict_t       *dict          = NULL;
+        call_frame_t *prev          = NULL;
+        gf_boolean_t  flag          = _gf_true;
 
         local = frame->local;
+        prev = cookie;
 
-        if (op_ret != -1) {
-                ret = dict_get_str (xattr, GF_XATTR_PATHINFO_KEY, &value_got);
-                if (!ret) {
-                        if (!local->pathinfo)
-                                local->pathinfo = GF_CALLOC (8192, sizeof (char),
-                                                             gf_common_mt_char);
-                        if (local->pathinfo)
-                                strcat (local->pathinfo, value_got);
-                }
+        if (op_ret < 0) {
+                local->op_ret = -1;
+                local->op_errno = op_errno;
+                gf_log (this->name, GF_LOG_ERROR, "Subvolume %s returned -1 "
+                        "(%s)", prev->this->name, strerror (op_errno));
+                goto unwind;
         }
 
-        this_call_cnt = dht_frame_return (frame);
-        if (is_last_call (this_call_cnt)) {
-                if (local->layout->cnt > 1) {
-                        /* Set it for directory */
-                        fill_layout_info (local->layout, layout_buf);
-                        flag = 1;
-                }
-
-                dict = dict_new ();
-
-                if (flag && local->pathinfo)
-                        snprintf (xattr_buf, 9216, "((%s %s) (%s-layout %s))",
-                                  this->name, local->pathinfo, this->name,
-                                  layout_buf);
-                else if (local->pathinfo)
-                        snprintf (xattr_buf, 9216, "(%s %s)",
-                                  this->name, local->pathinfo);
-                else if (flag)
-                        snprintf (xattr_buf, 9216, "(%s-layout %s)",
-                                  this->name, layout_buf);
-
-                ret = dict_set_str (dict, GF_XATTR_PATHINFO_KEY,
-                                    xattr_buf);
-
-                if (local->pathinfo)
-                        GF_FREE (local->pathinfo);
-                GF_FREE (local->key);
-
-                DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict);
+        ret = dht_vgetxattr_alloc_and_fill (local, xattr, this,
+                                            op_errno);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "alloc or fill failure");
+                goto unwind;
+        }
 
-                if (dict)
-                        dict_unref (dict);
+        flag = (local->layout->cnt > 1) ? _gf_true : _gf_false;
 
-                return 0;
-        }
+        ret = dht_vgetxattr_fill_and_set (local, &dict, this, flag);
+        if (ret)
+                goto unwind;
 
-        if (local->pathinfo)
-                strcat (local->pathinfo, " Link: ");
-        if (local->hashed_subvol) {
-                /* This will happen if there pending */
-                STACK_WIND (frame, dht_pathinfo_getxattr_cbk, local->hashed_subvol,
-                            local->hashed_subvol->fops->getxattr,
-                            &local->loc, local->key);
+        DHT_STACK_UNWIND (getxattr, frame, 0, 0, dict, xdata);
+        goto cleanup;
 
-                return 0;
-        }
+ unwind:
+        DHT_STACK_UNWIND (getxattr, frame, -1, local->op_errno,
+                          NULL, NULL);
+ cleanup:
+        if (dict)
+                dict_unref (dict);
 
-        gf_log ("this->name", GF_LOG_ERROR, "Unable to find hashed_subvol for path"
-                " %s", local->pathinfo);
-      
-        DHT_STACK_UNWIND (getxattr, frame, -1, op_errno, dict);
         return 0;
 }
 
 int
 dht_linkinfo_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                           int op_ret, int op_errno, dict_t *xattr)
+                           int op_ret, int op_errno, dict_t *xattr,
+                           dict_t *xdata)
 {
         int   ret   = 0;
         char *value = NULL;
@@ -1796,93 +1985,234 @@ dht_linkinfo_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 }
         }
 
-        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr);
+        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
 
         return 0;
 }
 
 int
 dht_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int op_ret, int op_errno, dict_t *xattr)
+                  int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
 {
-        if (op_ret != -1) {
-                if (dict_get (xattr, "trusted.glusterfs.dht")) {
-                        dict_del (xattr, "trusted.glusterfs.dht");
-                }
+        int             this_call_cnt = 0;
+        dht_local_t     *local = NULL;
+        dht_conf_t      *conf = NULL;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (frame->local, out);
+        VALIDATE_OR_GOTO (this->private, out);
+
+        conf = this->private;
+        local = frame->local;
+
+        this_call_cnt = dht_frame_return (frame);
+
+        if (!xattr || (op_ret == -1))
+                goto out;
+
+        if (dict_get (xattr, conf->xattr_name)) {
+                dict_del (xattr, conf->xattr_name);
         }
 
-        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr);
+        if (frame->root->pid >= 0 ) {
+                GF_REMOVE_INTERNAL_XATTR("trusted.glusterfs.quota*", xattr);
+                GF_REMOVE_INTERNAL_XATTR("trusted.pgfid*", xattr);
+        }
 
+        local->op_ret = 0;
+
+        if (!local->xattr) {
+                local->xattr = dict_copy_with_ref (xattr, NULL);
+        } else {
+                dht_aggregate_xattr (local->xattr, xattr);
+        }
+out:
+        if (is_last_call (this_call_cnt)) {
+                DHT_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
+                                  local->xattr, NULL);
+        }
         return 0;
 }
 
+int32_t
+dht_getxattr_unwind (call_frame_t *frame,
+                     int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
+{
+        DHT_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
+}
+
+
+int
+dht_getxattr_get_real_filename_cbk (call_frame_t *frame, void *cookie,
+				    xlator_t *this, int op_ret, int op_errno,
+				    dict_t *xattr, dict_t *xdata)
+{
+        int             this_call_cnt = 0;
+        dht_local_t     *local = NULL;
+
+
+        local = frame->local;
+
+	if (op_ret != -1) {
+		if (local->xattr)
+			dict_unref (local->xattr);
+		local->xattr = dict_ref (xattr);
+
+		if (local->xattr_req)
+			dict_unref (local->xattr_req);
+		local->xattr_req = dict_ref (xdata);
+	}
+
+	this_call_cnt = dht_frame_return (frame);
+	if (is_last_call (this_call_cnt)) {
+		DHT_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
+				  local->xattr, local->xattr_req);
+	}
+
+	return 0;
+}
+
+
+int
+dht_getxattr_get_real_filename (call_frame_t *frame, xlator_t *this,
+				loc_t *loc, const char *key, dict_t *xdata)
+{
+	dht_local_t     *local = NULL;
+	int              i = 0;
+	dht_layout_t    *layout = NULL;
+	int              cnt = 0;
+	xlator_t        *subvol = NULL;
+
+
+	local = frame->local;
+	layout = local->layout;
+
+	cnt = local->call_cnt = layout->cnt;
+
+	local->op_ret = -1;
+	local->op_errno = ENODATA;
+
+	for (i = 0; i < cnt; i++) {
+		subvol = layout->list[i].xlator;
+		STACK_WIND (frame, dht_getxattr_get_real_filename_cbk,
+			    subvol, subvol->fops->getxattr,
+			    loc, key, xdata);
+	}
+
+	return 0;
+}
+
 
 int
 dht_getxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, const char *key)
+              loc_t *loc, const char *key, dict_t *xdata)
+#define DHT_IS_DIR(layout)  (layout->cnt > 1)
 {
-	xlator_t     *subvol        = NULL;
-	xlator_t     *hashed_subvol = NULL;
-	xlator_t     *cached_subvol = NULL;
-	dht_conf_t   *conf          = NULL;
-	dht_local_t  *local         = NULL;
+
+        xlator_t     *subvol        = NULL;
+        xlator_t     *hashed_subvol = NULL;
+        xlator_t     *cached_subvol = NULL;
+        dht_conf_t   *conf          = NULL;
+        dht_local_t  *local         = NULL;
         dht_layout_t *layout        = NULL;
+        xlator_t     **sub_volumes  = NULL;
         int           op_errno      = -1;
-        int           ret           = 0;
-        int           flag          = 0;
         int           i             = 0;
+        int           cnt           = 0;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
         VALIDATE_OR_GOTO (this->private, err);
 
         conf   = this->private;
-        layout = dht_layout_get (this, loc->inode);
-        if (key && (strcmp (key, GF_XATTR_PATHINFO_KEY) == 0)) {
-                hashed_subvol = dht_subvol_get_hashed (this, loc);
-                cached_subvol = dht_subvol_get_cached (this, loc->inode);
 
-                local = dht_local_init (frame);
-                if (!local) {
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
-                        goto err;
-                }
+        local = dht_local_init (frame, loc, NULL, GF_FOP_GETXATTR);
+        if (!local) {
+                op_errno = ENOMEM;
 
-                ret = loc_dup (loc, &local->loc);
-                if (ret == -1) {
-                        op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
-                        goto err;
-                }
+                goto err;
+        }
+
+        layout = local->layout;
+        if (!layout) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "layout is NULL");
+                op_errno = ENOENT;
+                goto err;
+        }
+
+        if (key) {
                 local->key = gf_strdup (key);
                 if (!local->key) {
                         op_errno = ENOMEM;
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
                         goto err;
                 }
-                local->layout = layout;
+        }
 
-                local->call_cnt = 1;
-                if (hashed_subvol != cached_subvol) {
-                        local->call_cnt = 2;
-                        local->hashed_subvol = hashed_subvol;
+	if (key &&
+	    (strncmp (key, GF_XATTR_GET_REAL_FILENAME_KEY,
+		      strlen (GF_XATTR_GET_REAL_FILENAME_KEY)) == 0)
+	    && DHT_IS_DIR(layout)) {
+		dht_getxattr_get_real_filename (frame, this, loc, key, xdata);
+		return 0;
+	}
+
+        /* for file use cached subvolume (obviously!): see if {}
+         * below
+         * for directory:
+         *  wind to all subvolumes and exclude subvolumes which
+         *  return ENOTCONN (in callback)
+         *
+         * NOTE: Don't trust inode here, as that may not be valid
+         *       (until inode_link() happens)
+         */
+        if (key && DHT_IS_DIR(layout) &&
+            (XATTR_IS_PATHINFO (key)
+             || (strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0))) {
+                (void) strncpy (local->xsel, key, 256);
+                cnt = local->call_cnt = layout->cnt;
+                for (i = 0; i < cnt; i++) {
+                        subvol = layout->list[i].xlator;
+                        STACK_WIND (frame, dht_vgetxattr_dir_cbk,
+                                    subvol, subvol->fops->getxattr,
+                                    loc, key, NULL);
                 }
+                return 0;
+        }
+
+        /* node-uuid or pathinfo for files */
+        if (key && ((strcmp (key, GF_XATTR_NODE_UUID_KEY) == 0)
+                    || XATTR_IS_PATHINFO (key))) {
+                cached_subvol = local->cached_subvol;
+                (void) strncpy (local->xsel, key, 256);
 
-                STACK_WIND (frame, dht_pathinfo_getxattr_cbk, cached_subvol,
-                            cached_subvol->fops->getxattr, loc, key);
+                local->call_cnt = 1;
+                STACK_WIND (frame, dht_vgetxattr_cbk, cached_subvol,
+                            cached_subvol->fops->getxattr, loc, key, NULL);
 
                 return 0;
         }
+
         if (key && (strcmp (key, GF_XATTR_LINKINFO_KEY) == 0)) {
                 hashed_subvol = dht_subvol_get_hashed (this, loc);
+                if (!hashed_subvol) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get"
+                                "hashed subvol for %s", loc->path);
+                        op_errno = EINVAL;
+                        goto err;
+                }
+
                 cached_subvol = dht_subvol_get_cached (this, loc->inode);
+                if (!cached_subvol) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get"
+                                "cached subvol for %s", loc->path);
+                        op_errno = EINVAL;
+                        goto err;
+                }
+
                 if (hashed_subvol == cached_subvol) {
                         op_errno = ENODATA;
                         goto err;
@@ -1890,720 +2220,878 @@ dht_getxattr (call_frame_t *frame, xlator_t *this,
                 if (hashed_subvol) {
                         STACK_WIND (frame, dht_linkinfo_getxattr_cbk, hashed_subvol,
                                     hashed_subvol->fops->getxattr, loc,
-                                    GF_XATTR_PATHINFO_KEY);
+                                    GF_XATTR_PATHINFO_KEY, NULL);
                         return 0;
                 }
                 op_errno = ENODATA;
                 goto err;
         }
-        if (key && (strcmp (key, GF_XATTR_FIX_LAYOUT_KEY) == 0)) {
-                for (i = 0; i < layout->cnt; i++) {
-                        if (layout->list[i].start == layout->list[i].stop) {
-                                flag = 1;
-                                break;
-                        }
+
+        if (key && (!strcmp (GF_XATTR_MARKER_KEY, key))
+            && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+                if (DHT_IS_DIR(layout)) {
+                        cnt = layout->cnt;
+                } else {
+                        cnt = 1;
                 }
-                if ((layout->cnt < conf->subvolume_cnt) || flag) {
-                        gf_log (this->name, GF_LOG_INFO,
-                                "expanding layout of %s from %d to %d",
-                                loc->path, layout->cnt, conf->subvolume_cnt);
-                        local = dht_local_init (frame);
-                        if (!local) {
-                                op_errno = ENOMEM;
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Out of memory");
-                                goto err;
-                        }
 
-                        ret = loc_dup (loc, &local->loc);
-                        if (ret == -1) {
-                                op_errno = ENOMEM;
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Out of memory");
+                sub_volumes = alloca ( cnt * sizeof (xlator_t *));
+                for (i = 0; i < cnt; i++)
+                        *(sub_volumes + i) = layout->list[i].xlator;
+
+                if (cluster_getmarkerattr (frame, this, loc, key,
+                                           local, dht_getxattr_unwind,
+                                           sub_volumes, cnt,
+                                           MARKER_UUID_TYPE, marker_uuid_default_gauge,
+                                           conf->vol_uuid)) {
+                        op_errno = EINVAL;
+                        goto err;
+                }
+
+                return 0;
+        }
+
+        if (key && !strcmp (GF_XATTR_QUOTA_LIMIT_LIST, key)) {
+                /* quota hardlimit and aggregated size of a directory is stored
+                 * in inode contexts of each brick. Hence its good enough that
+                 * we send getxattr for this key to any brick.
+                 */
+                local->call_cnt = 1;
+                subvol = dht_first_up_subvol (this);
+                STACK_WIND (frame, dht_getxattr_cbk, subvol,
+                            subvol->fops->getxattr, loc, key, xdata);
+                return 0;
+        }
+
+        if (key && *conf->vol_uuid) {
+                if ((match_uuid_local (key, conf->vol_uuid) == 0) &&
+                    (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+                        if (DHT_IS_DIR(layout)) {
+                                cnt = layout->cnt;
+                        } else {
+                                cnt = 1;
+                        }
+                        sub_volumes = alloca ( cnt * sizeof (xlator_t *));
+                        for (i = 0; i < cnt; i++)
+                                sub_volumes[i] = layout->list[i].xlator;
+
+                        if (cluster_getmarkerattr (frame, this, loc, key,
+                                                   local, dht_getxattr_unwind,
+                                                   sub_volumes, cnt,
+                                                   MARKER_XTIME_TYPE,
+                                                   marker_xtime_default_gauge,
+                                                   conf->vol_uuid)) {
+                                op_errno = EINVAL;
                                 goto err;
                         }
-                        local->layout = layout;
-                        //layout = dht_layout_new (this, conf->subvolume_cnt);
 
-                        dht_selfheal_new_directory (frame, dht_fix_layout_cbk,
-                                                    layout);
                         return 0;
                 }
-                op_errno = ENODATA;
-                goto err;
         }
-	subvol = dht_subvol_get_cached (this, loc->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
 
-	STACK_WIND (frame, dht_getxattr_cbk,
-		    subvol, subvol->fops->getxattr,
-		    loc, key);
+        if (DHT_IS_DIR(layout)) {
+                cnt = local->call_cnt = layout->cnt;
+        } else {
+                cnt = local->call_cnt  = 1;
+        }
 
-	return 0;
+        for (i = 0; i < cnt; i++) {
+                subvol = layout->list[i].xlator;
+                STACK_WIND (frame, dht_getxattr_cbk,
+                            subvol, subvol->fops->getxattr,
+                            loc, key, NULL);
+        }
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
 
-	return 0;
+        return 0;
 }
-
+#undef DHT_IS_DIR
 
 int
-dht_setxattr (call_frame_t *frame, xlator_t *this,
-	      loc_t *loc, dict_t *xattr, int flags)
+dht_fgetxattr (call_frame_t *frame, xlator_t *this,
+               fd_t *fd, const char *key, dict_t *xdata)
 {
-	xlator_t     *subvol   = NULL;
-	dht_local_t  *local    = NULL;
-        dht_conf_t   *conf     = NULL;
-        dht_layout_t *layout   = NULL;
-        int           i        = 0;
-        int           op_errno = EINVAL;
+        xlator_t     *subvol        = NULL;
+        dht_local_t  *local         = NULL;
+        dht_layout_t *layout        = NULL;
+        int           op_errno      = -1;
+        int           i             = 0;
+        int           cnt           = 0;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+        VALIDATE_OR_GOTO (this->private, err);
 
-        conf   = this->private;
-	subvol = dht_subvol_get_cached (this, loc->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FGETXATTR);
+        if (!local) {
+                op_errno = ENOMEM;
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
+                goto err;
+        }
 
-	local->layout = layout = dht_layout_get (this, loc->inode);
-	if (!layout) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no layout for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+        layout = local->layout;
+        if (!layout) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "layout is NULL");
+                op_errno = ENOENT;
+                goto err;
+        }
 
-	local->call_cnt = layout->cnt;
+        if (key) {
+                local->key = gf_strdup (key);
+                if (!local->key) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+        }
 
-	for (i = 0; i < layout->cnt; i++) {
-		STACK_WIND (frame, dht_err_cbk,
-			    layout->list[i].xlator,
-			    layout->list[i].xlator->fops->setxattr,
-			    loc, xattr, flags);
-	}
+        if ((fd->inode->ia_type == IA_IFDIR)
+	    && key
+            && (strncmp (key, GF_XATTR_LOCKINFO_KEY,
+                         strlen (GF_XATTR_LOCKINFO_KEY) != 0))) {
+                cnt = local->call_cnt = layout->cnt;
+        } else {
+                cnt = local->call_cnt  = 1;
+        }
 
-	return 0;
+        for (i = 0; i < cnt; i++) {
+                subvol = layout->list[i].xlator;
+                STACK_WIND (frame, dht_getxattr_cbk,
+                            subvol, subvol->fops->fgetxattr,
+                            fd, key, NULL);
+        }
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (setxattr, frame, -1, op_errno);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fgetxattr, frame, -1, op_errno, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
-
 int
-dht_removexattr (call_frame_t *frame, xlator_t *this,
-		 loc_t *loc, const char *key)
+dht_fsetxattr (call_frame_t *frame, xlator_t *this,
+               fd_t *fd, dict_t *xattr, int flags, dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-	dht_local_t  *local = NULL;
-
+        xlator_t     *subvol   = NULL;
+        dht_local_t  *local    = NULL;
+        int           op_errno = EINVAL;
+        dht_conf_t   *conf     = NULL;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+        VALIDATE_OR_GOTO (this->private, err);
 
-	subvol = dht_subvol_get_cached (this, loc->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+        conf = this->private;
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
+        GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
+                                   op_errno, err);
 
-	local->call_cnt = 1;
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FSETXATTR);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	STACK_WIND (frame, dht_err_cbk,
-		    subvol, subvol->fops->removexattr,
-		    loc, key);
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	return 0;
+        local->call_cnt = 1;
+
+        STACK_WIND (frame, dht_err_cbk, subvol, subvol->fops->fsetxattr,
+                    fd, xattr, flags, NULL);
+
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (removexattr, frame, -1, op_errno);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fsetxattr, frame, -1, op_errno, NULL);
 
-	return 0;
+        return 0;
 }
 
 
+static int
+dht_common_setxattr_cbk (call_frame_t *frame, void *cookie,
+                         xlator_t *this, int32_t op_ret, int32_t op_errno,
+                         dict_t *xdata)
+{
+        DHT_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
 int
-dht_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	      int op_ret, int op_errno, fd_t *fd)
+dht_checking_pathinfo_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int op_ret, int op_errno, dict_t *xattr,
+                           dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = 0;
-	call_frame_t *prev = NULL;
+        int           i     = -1;
+        int           ret   = -1;
+        char         *value = NULL;
+        dht_local_t  *local = NULL;
+        dht_conf_t   *conf  = NULL;
+        call_frame_t *prev  = NULL;
+        int           this_call_cnt = 0;
 
+        local = frame->local;
+        prev = cookie;
+        conf = this->private;
 
-	local = frame->local;
-	prev = cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			gf_log (this->name, GF_LOG_DEBUG,
-				"subvolume %s returned -1 (%s)",
-				prev->this->name, strerror (op_errno));
-			goto unlock;
-		}
+        if (op_ret == -1)
+                goto out;
 
-		local->op_ret = 0;
-	}
-unlock:
-	UNLOCK (&frame->lock);
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt))
-		DHT_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
-				  local->fd);
+        ret = dict_get_str (xattr, GF_XATTR_PATHINFO_KEY, &value);
+        if (ret)
+                goto out;
 
+        if (!strcmp (value, local->key)) {
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        if (conf->subvolumes[i] == prev->this)
+                                conf->decommissioned_bricks[i] = prev->this;
+                }
+        }
+
+out:
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+                DHT_STACK_UNWIND (setxattr, frame, local->op_ret, ENOTSUP, NULL);
+        }
         return 0;
-}
 
+}
 
 int
-dht_open (call_frame_t *frame, xlator_t *this,
-	  loc_t *loc, int flags, fd_t *fd, int wbflags)
+dht_setxattr (call_frame_t *frame, xlator_t *this,
+              loc_t *loc, dict_t *xattr, int flags, dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
-	int           ret = -1;
-        int           op_errno = -1;
-	dht_local_t  *local = NULL;
-
+        xlator_t     *subvol   = NULL;
+        dht_local_t  *local    = NULL;
+        dht_conf_t   *conf     = NULL;
+        dht_layout_t *layout   = NULL;
+        int           i        = 0;
+        int           op_errno = EINVAL;
+        int           ret      = -1;
+        data_t       *tmp      = NULL;
+        uint32_t      dir_spread = 0;
+        char          value[4096] = {0,};
+        gf_dht_migrate_data_type_t forced_rebalance = GF_DHT_MIGRATE_DATA;
+        int           call_cnt = 0;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
 
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+        conf   = this->private;
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
+        GF_IF_INTERNAL_XATTR_GOTO (conf->wild_xattr_name, xattr,
+                                   op_errno, err);
 
-	local->fd = fd_ref (fd);
-	ret = loc_dup (loc, &local->loc);
-	if (ret == -1) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
+        local = dht_local_init (frame, loc, NULL, GF_FOP_SETXATTR);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	STACK_WIND (frame, dht_fd_cbk,
-		    subvol, subvol->fops->open,
-		    loc, flags, fd, wbflags);
+        layout = local->layout;
+        if (!layout) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no layout for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	return 0;
+        local->call_cnt = call_cnt = layout->cnt;
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (open, frame, -1, op_errno, NULL);
+        tmp = dict_get (xattr, "distribute.migrate-data");
+        if (tmp) {
+                if (IA_ISDIR (loc->inode->ia_type)) {
+                        op_errno = ENOTSUP;
+                        goto err;
+                }
 
-	return 0;
-}
+                /* TODO: need to interpret the 'value' for more meaning
+                   (ie, 'target' subvolume given there, etc) */
+                memcpy (value, tmp->data, tmp->len);
+                if (strcmp (value, "force") == 0)
+                       forced_rebalance =
+                                GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS;
 
+                if (conf->decommission_in_progress)
+                        forced_rebalance = GF_DHT_MIGRATE_HARDLINK;
 
-int
-dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	       int op_ret, int op_errno,
-	       struct iovec *vector, int count, struct iatt *stbuf,
-               struct iobref *iobref)
-{
-        dht_local_t     *local = frame->local;
+                local->rebalance.target_node = dht_subvol_get_hashed (this, loc);
+                if (!local->rebalance.target_node) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "hashed subvol for %s", loc->path);
+                        op_errno = EINVAL;
+                        goto err;
+                }
 
-        if (!local) {
-                op_ret = -1;
+                local->rebalance.from_subvol = local->cached_subvol;
+
+                if (local->rebalance.target_node == local->rebalance.from_subvol) {
+                        op_errno = EEXIST;
+                        goto err;
+                }
+                if (local->rebalance.target_node) {
+                        local->flags = forced_rebalance;
+
+                        ret = dht_start_rebalance_task (this, frame);
+                        if (!ret)
+                                return 0;
+
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "%s: failed to create a new synctask",
+                                loc->path);
+                }
                 op_errno = EINVAL;
-                goto out;
-        }
+                goto err;
 
-        if (op_ret != -1)
-                stbuf->ia_ino = local->ia_ino;
-out:
-        DHT_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count, stbuf,
-                          iobref);
+        }
 
-        return 0;
-}
+        tmp = dict_get (xattr, "decommission-brick");
+        if (tmp) {
+                /* This operation should happen only on '/' */
+                if (!__is_root_gfid (loc->inode->gfid)) {
+                        op_errno = ENOTSUP;
+                        goto err;
+                }
 
+                memcpy (value, tmp->data, ((tmp->len < 4095) ? tmp->len : 4095));
+                local->key = gf_strdup (value);
+                local->call_cnt = conf->subvolume_cnt;
 
-int
-dht_readv (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, size_t size, off_t off)
-{
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-        dht_local_t  *local = NULL;
+                for (i = 0 ; i < conf->subvolume_cnt; i++) {
+                        /* Get the pathinfo, and then compare */
+                        STACK_WIND (frame, dht_checking_pathinfo_cbk,
+                                    conf->subvolumes[i],
+                                    conf->subvolumes[i]->fops->getxattr,
+                                    loc, GF_XATTR_PATHINFO_KEY, NULL);
+                }
+                return 0;
+        }
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+        tmp = dict_get (xattr, GF_XATTR_FIX_LAYOUT_KEY);
+        if (tmp) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "fixing the layout of %s", loc->path);
 
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+                ret = dht_fix_directory_layout (frame, dht_common_setxattr_cbk,
+                                                layout);
+                if (ret) {
+                        op_errno = ENOTCONN;
+                        goto err;
+                }
+                return ret;
+        }
 
-        local = dht_local_init (frame);
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR, "Out of memory");
-                op_errno = ENOMEM;
+        tmp = dict_get (xattr, "distribute.directory-spread-count");
+        if (tmp) {
+                /* Setxattr value is packed as 'binary', not string */
+                memcpy (value, tmp->data, ((tmp->len < 4095)?tmp->len:4095));
+                ret = gf_string2uint32 (value, &dir_spread);
+                if (!ret && ((dir_spread <= conf->subvolume_cnt) &&
+                             (dir_spread > 0))) {
+                        layout->spread_cnt = dir_spread;
+
+                        ret = dht_fix_directory_layout (frame,
+                                                        dht_common_setxattr_cbk,
+                                                        layout);
+                        if (ret) {
+                                op_errno = ENOTCONN;
+                                goto err;
+                        }
+                        return ret;
+                }
+                gf_log (this->name, GF_LOG_ERROR,
+                        "wrong 'directory-spread-count' value (%s)", value);
+                op_errno = ENOTSUP;
                 goto err;
         }
 
-        local->ia_ino = fd->inode->ino;
-	STACK_WIND (frame, dht_readv_cbk,
-		    subvol, subvol->fops->readv,
-		    fd, size, off);
+        for (i = 0; i < call_cnt; i++) {
+                STACK_WIND (frame, dht_err_cbk,
+                            layout->list[i].xlator,
+                            layout->list[i].xlator->fops->setxattr,
+                            loc, xattr, flags, xdata);
+        }
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (setxattr, frame, -1, op_errno, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
-dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int op_ret, int op_errno, struct iatt *prebuf,
-                struct iatt *postbuf)
+dht_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xdata)
 {
-        dht_local_t *local = NULL;
-
-        if (op_ret == -1) {
-                goto out;
-        }
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        call_frame_t *prev = NULL;
 
         local = frame->local;
-        if (!local) {
-                op_ret = -1;
-                op_errno = EINVAL;
-                goto out;
-        } 
-        
-        prebuf->ia_ino = local->ia_ino;
-        postbuf->ia_ino = local->ia_ino;
+        prev = cookie;
 
-out:
-        DHT_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "subvolume %s returned -1 (%s)",
+                                prev->this->name, strerror (op_errno));
+                        goto unlock;
+                }
+
+                local->op_ret = 0;
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+                DHT_STACK_UNWIND (removexattr, frame, local->op_ret,
+                                  local->op_errno, NULL);
+        }
 
         return 0;
 }
 
 
 int
-dht_writev (call_frame_t *frame, xlator_t *this,
-	    fd_t *fd, struct iovec *vector, int count, off_t off,
-            struct iobref *iobref)
+dht_removexattr (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, const char *key, dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
+        xlator_t     *subvol = NULL;
         int           op_errno = -1;
         dht_local_t  *local = NULL;
+        dht_layout_t *layout = NULL;
+        int           call_cnt = 0;
+        dht_conf_t   *conf = NULL;
+
+        int i;
 
-        VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (this->private, err);
 
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+        conf = this->private;
+
+        GF_IF_NATIVE_XATTR_GOTO (conf->wild_xattr_name, key, op_errno, err);
 
-        local = dht_local_init (frame);
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+
+        local = dht_local_init (frame, loc, NULL, GF_FOP_REMOVEXATTR);
         if (!local) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
                 op_errno = ENOMEM;
                 goto err;
         }
 
-        local->ia_ino = fd->inode->ino;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        layout = local->layout;
+        if (!local->layout) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no layout for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        local->call_cnt = call_cnt = layout->cnt;
+        local->key = gf_strdup (key);
 
-	STACK_WIND (frame, dht_writev_cbk,
-		    subvol, subvol->fops->writev,
-		    fd, vector, count, off, iobref);
+        for (i = 0; i < call_cnt; i++) {
+                STACK_WIND (frame, dht_removexattr_cbk,
+                            layout->list[i].xlator,
+                            layout->list[i].xlator->fops->removexattr,
+                            loc, key, NULL);
+        }
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (removexattr, frame, -1, op_errno, NULL);
 
-	return 0;
+        return 0;
 }
 
-
 int
-dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+dht_fremovexattr (call_frame_t *frame, xlator_t *this,
+                  fd_t *fd, const char *key, dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
+        xlator_t     *subvol = NULL;
         int           op_errno = -1;
-	dht_local_t  *local = NULL;
+        dht_local_t  *local = NULL;
+        dht_layout_t *layout = NULL;
+        int           call_cnt = 0;
+        dht_conf_t   *conf = 0;
 
+        int i;
 
-        VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
-
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
-
-	local->fd = fd_ref (fd);
-	local->call_cnt = 1;
-
-	STACK_WIND (frame, dht_err_cbk,
-		    subvol, subvol->fops->flush, fd);
-
-	return 0;
-
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (flush, frame, -1, op_errno);
-
-	return 0;
-}
-
+        VALIDATE_OR_GOTO (this->private, err);
 
-int
-dht_fsync (call_frame_t *frame, xlator_t *this,
-	   fd_t *fd, int datasync)
-{
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-	dht_local_t  *local = NULL;
+        conf = this->private;
 
+        GF_IF_NATIVE_XATTR_GOTO (conf->wild_xattr_name, key, op_errno, err);
 
         VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
 
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FREMOVEXATTR);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
-	local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for inode=%s",
+                        uuid_utoa (fd->inode->gfid));
+                op_errno = EINVAL;
+                goto err;
+        }
 
-        local->ia_ino = fd->inode->ino;
+        layout = local->layout;
+        if (!local->layout) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no layout for inode=%s", uuid_utoa (fd->inode->gfid));
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	STACK_WIND (frame, dht_fsync_cbk,
-		    subvol, subvol->fops->fsync,
-		    fd, datasync);
+        local->call_cnt = call_cnt = layout->cnt;
+        local->key = gf_strdup (key);
 
-	return 0;
+        for (i = 0; i < call_cnt; i++) {
+                STACK_WIND (frame, dht_removexattr_cbk,
+                            layout->list[i].xlator,
+                            layout->list[i].xlator->fops->fremovexattr,
+                            fd, key, NULL);
+        }
+
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fremovexattr, frame, -1, op_errno, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
-dht_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	    int op_ret, int op_errno, struct gf_flock *flock)
+dht_fd_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+            int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
-        DHT_STACK_UNWIND (lk, frame, op_ret, op_errno, flock);
-
-        return 0;
-}
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        call_frame_t *prev = NULL;
 
+        local = frame->local;
+        prev = cookie;
 
-int
-dht_lk (call_frame_t *frame, xlator_t *this,
-	fd_t *fd, int cmd, struct gf_flock *flock)
-{
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "subvolume %s returned -1 (%s)",
+                                prev->this->name, strerror (op_errno));
+                        goto unlock;
+                }
 
+                local->op_ret = 0;
+        }
+unlock:
+        UNLOCK (&frame->lock);
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt))
+                DHT_STACK_UNWIND (open, frame, local->op_ret, local->op_errno,
+                                  local->fd, NULL);
 
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+        return 0;
+}
 
-	STACK_WIND (frame, dht_lk_cbk,
-		    subvol, subvol->fops->lk,
-		    fd, cmd, flock);
+/*
+ * dht_normalize_stats -
+ */
+static void
+dht_normalize_stats (struct statvfs *buf, unsigned long bsize,
+                     unsigned long frsize)
+{
+        double factor = 0;
 
-	return 0;
+        if (buf->f_bsize != bsize) {
+                buf->f_bsize = bsize;
+        }
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (lk, frame, -1, op_errno, NULL);
+        if (buf->f_frsize != frsize) {
+                factor = ((double) buf->f_frsize) / frsize;
+                buf->f_frsize = frsize;
+                buf->f_blocks = (fsblkcnt_t) (factor * buf->f_blocks);
+                buf->f_bfree  = (fsblkcnt_t) (factor * buf->f_bfree);
+                buf->f_bavail = (fsblkcnt_t) (factor * buf->f_bavail);
 
-	return 0;
+        }
 }
 
-
 int
 dht_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int op_ret, int op_errno, struct statvfs *statvfs)
+                int op_ret, int op_errno, struct statvfs *statvfs, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = 0;
+        dht_local_t *local              = NULL;
+        int          this_call_cnt      = 0;
+        int          bsize              = 0;
+        int          frsize             = 0;
+        int8_t       quota_deem_statfs  = 0;
+        GF_UNUSED int     ret           = 0;
+        unsigned long     new_usage     = 0;
+        unsigned long     cur_usage     = 0;
 
+        ret = dict_get_int8 (xdata, "quota-deem-statfs", &quota_deem_statfs);
 
-	local = frame->local;
+        local = frame->local;
 
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			goto unlock;
-		}
-		local->op_ret = 0;
-
-		/* TODO: normalize sizes */
-		local->statvfs.f_bsize    = statvfs->f_bsize;
-		local->statvfs.f_frsize   = statvfs->f_frsize;
-
-		local->statvfs.f_blocks  += statvfs->f_blocks;
-		local->statvfs.f_bfree   += statvfs->f_bfree;
-		local->statvfs.f_bavail  += statvfs->f_bavail;
-		local->statvfs.f_files   += statvfs->f_files;
-		local->statvfs.f_ffree   += statvfs->f_ffree;
-		local->statvfs.f_favail  += statvfs->f_favail;
-		local->statvfs.f_fsid     = statvfs->f_fsid;
-		local->statvfs.f_flag     = statvfs->f_flag;
-		local->statvfs.f_namemax  = statvfs->f_namemax;
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        goto unlock;
+                }
+                if (!statvfs) {
+                        op_errno = EINVAL;
+                        local->op_ret = -1;
+                        goto unlock;
+                }
+                local->op_ret = 0;
 
-	}
+                if (quota_deem_statfs) {
+                        new_usage = statvfs->f_blocks - statvfs->f_bfree;
+                        cur_usage = local->statvfs.f_blocks - local->statvfs.f_bfree;
+                        /* We take the maximux of the usage from the subvols */
+                        if (new_usage >= cur_usage)
+                                local->statvfs = *statvfs;
+                        goto unlock;
+                }
+
+                if (local->statvfs.f_bsize != 0) {
+                        bsize = max(local->statvfs.f_bsize, statvfs->f_bsize);
+                        frsize = max(local->statvfs.f_frsize, statvfs->f_frsize);
+                        dht_normalize_stats(&local->statvfs, bsize, frsize);
+                        dht_normalize_stats(statvfs, bsize, frsize);
+                } else {
+                        local->statvfs.f_bsize    = statvfs->f_bsize;
+                        local->statvfs.f_frsize   = statvfs->f_frsize;
+                }
+
+                local->statvfs.f_blocks  += statvfs->f_blocks;
+                local->statvfs.f_bfree   += statvfs->f_bfree;
+                local->statvfs.f_bavail  += statvfs->f_bavail;
+                local->statvfs.f_files   += statvfs->f_files;
+                local->statvfs.f_ffree   += statvfs->f_ffree;
+                local->statvfs.f_favail  += statvfs->f_favail;
+                local->statvfs.f_fsid     = statvfs->f_fsid;
+                local->statvfs.f_flag     = statvfs->f_flag;
+                local->statvfs.f_namemax  = statvfs->f_namemax;
+
+
+        }
 unlock:
-	UNLOCK (&frame->lock);
+        UNLOCK (&frame->lock);
 
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt))
-		DHT_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
-				  &local->statvfs);
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt))
+                DHT_STACK_UNWIND (statfs, frame, local->op_ret, local->op_errno,
+                                  &local->statvfs, xdata);
 
         return 0;
 }
 
 
 int
-dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+dht_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-	dht_local_t  *local  = NULL;
-	dht_conf_t   *conf = NULL;
+        xlator_t     *subvol = NULL;
+        dht_local_t  *local  = NULL;
+        dht_conf_t   *conf = NULL;
         int           op_errno = -1;
-	int           i = -1;
-
+        int           i = -1;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
         VALIDATE_OR_GOTO (this->private, err);
 
-	conf = this->private;
+        conf = this->private;
 
-	local = dht_local_init (frame);
-	local->call_cnt = conf->subvolume_cnt;
+        local = dht_local_init (frame, NULL, NULL, GF_FOP_STATFS);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		STACK_WIND (frame, dht_statfs_cbk,
-			    conf->subvolumes[i],
-			    conf->subvolumes[i]->fops->statfs, loc);
-	}
+        if (IA_ISDIR (loc->inode->ia_type)) {
+                local->call_cnt = conf->subvolume_cnt;
 
-	return 0;
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        STACK_WIND (frame, dht_statfs_cbk,
+                                    conf->subvolumes[i],
+                                    conf->subvolumes[i]->fops->statfs, loc,
+                                    xdata);
+                }
+                return 0;
+        }
+
+        subvol = dht_subvol_get_cached (this, loc->inode);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        local->call_cnt = 1;
+
+        STACK_WIND (frame, dht_statfs_cbk,
+                    subvol, subvol->fops->statfs, loc, xdata);
+
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (statfs, frame, -1, op_errno, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
-dht_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+dht_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+             dict_t *xdata)
 {
-	dht_local_t  *local  = NULL;
-	dht_conf_t   *conf = NULL;
-	int           ret = -1;
+        dht_local_t  *local  = NULL;
+        dht_conf_t   *conf = NULL;
         int           op_errno = -1;
-	int           i = -1;
-
+        int           i = -1;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
         VALIDATE_OR_GOTO (this->private, err);
 
-	conf = this->private;
+        conf = this->private;
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
+        local = dht_local_init (frame, loc, fd, GF_FOP_OPENDIR);
+        if (!local) {
+                op_errno = ENOMEM;
 
-	local->fd = fd_ref (fd);
-	ret = loc_dup (loc, &local->loc);
-	if (ret == -1) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
+                goto err;
+        }
 
-	local->call_cnt = conf->subvolume_cnt;
+        local->call_cnt = conf->subvolume_cnt;
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		STACK_WIND (frame, dht_fd_cbk,
-			    conf->subvolumes[i],
-			    conf->subvolumes[i]->fops->opendir,
-			    loc, fd);
-	}
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                STACK_WIND (frame, dht_fd_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->opendir,
+                            loc, fd, xdata);
+        }
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (opendir, frame, -1, op_errno, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (opendir, frame, -1, op_errno, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
-                  int op_errno, gf_dirent_t *orig_entries)
+                  int op_errno, gf_dirent_t *orig_entries, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	gf_dirent_t   entries;
-	gf_dirent_t  *orig_entry = NULL;
-	gf_dirent_t  *entry = NULL;
-	call_frame_t *prev = NULL;
-	xlator_t     *next_subvol = NULL;
+        dht_local_t  *local = NULL;
+        gf_dirent_t   entries;
+        gf_dirent_t  *orig_entry = NULL;
+        gf_dirent_t  *entry = NULL;
+        call_frame_t *prev = NULL;
+        xlator_t     *next_subvol = NULL;
         off_t         next_offset = 0;
-	int           count = 0;
+        int           count = 0;
         dht_layout_t *layout = 0;
         dht_conf_t   *conf   = NULL;
         xlator_t     *subvol = 0;
+        int           ret    = 0;
 
-	INIT_LIST_HEAD (&entries.list);
-	prev = cookie;
-	local = frame->local;
-	conf  = this->private;
+        INIT_LIST_HEAD (&entries.list);
+        prev = cookie;
+        local = frame->local;
+        conf  = this->private;
 
-	if (op_ret < 0)
-		goto done;
+        if (op_ret < 0)
+                goto done;
 
         if (!local->layout)
                 local->layout = dht_layout_get (this, local->fd->inode);
 
         layout = local->layout;
 
-	list_for_each_entry (orig_entry, (&orig_entries->list), list) {
+        list_for_each_entry (orig_entry, (&orig_entries->list), list) {
                 next_offset = orig_entry->d_off;
-
-                if (check_is_linkfile (NULL, (&orig_entry->d_stat), NULL)
-                    || (check_is_dir (NULL, (&orig_entry->d_stat), NULL)
-                        && (prev->this != dht_first_up_subvol (this)))) {
+                if (check_is_dir (NULL, (&orig_entry->d_stat), NULL) &&
+                    (prev->this != local->first_up_subvol)) {
+                        continue;
+                }
+                if (check_is_linkfile (NULL, (&orig_entry->d_stat),
+                                       orig_entry->dict,
+                                       conf->link_xattr_name)) {
                         continue;
                 }
 
                 entry = gf_dirent_for_name (orig_entry->d_name);
                 if (!entry) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
+
                         goto unwind;
                 }
 
@@ -2613,25 +3101,41 @@ dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
                                                     orig_entry->d_name);
                         if (!subvol || (subvol != prev->this)) {
                                 /* TODO: Count the number of entries which need
-                                   linkfile to prove its existance in fs */
+                                   linkfile to prove its existence in fs */
                                 layout->search_unhashed++;
                         }
                 }
-                entry->d_stat = orig_entry->d_stat;
 
-                dht_itransform (this, prev->this, orig_entry->d_ino,
-                                &entry->d_ino);
                 dht_itransform (this, prev->this, orig_entry->d_off,
                                 &entry->d_off);
 
-                entry->d_stat.ia_ino = entry->d_ino;
+                entry->d_stat = orig_entry->d_stat;
+                entry->d_ino  = orig_entry->d_ino;
                 entry->d_type = orig_entry->d_type;
                 entry->d_len  = orig_entry->d_len;
 
+                if (orig_entry->dict)
+                        entry->dict = dict_ref (orig_entry->dict);
+
+                /* making sure we set the inode ctx right with layout,
+                   currently possible only for non-directories, so for
+                   directories don't set entry inodes */
+                if (!IA_ISDIR(entry->d_stat.ia_type) && orig_entry->inode) {
+                        ret = dht_layout_preset (this, prev->this,
+                                                 orig_entry->inode);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "failed to link the layout in inode");
+                        entry->inode = inode_ref (orig_entry->inode);
+                } else if (orig_entry->inode) {
+                        dht_inode_ctx_time_update (orig_entry->inode, this,
+                                                   &entry->d_stat, 1);
+                }
+
                 list_add_tail (&entry->list, &entries.list);
                 count++;
-	}
-	op_ret = count;
+        }
+        op_ret = count;
         /* We need to ensure that only the last subvolume's end-of-directory
          * notification is respected so that directory reading does not stop
          * before all subvolumes have been read. That could happen because the
@@ -2643,7 +3147,7 @@ dht_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
                 op_errno = 0;
 
 done:
-	if (count == 0) {
+        if (count == 0) {
                 /* non-zero next_offset means that
                    EOF is not yet hit on the current subvol
                 */
@@ -2653,23 +3157,37 @@ done:
                         next_subvol = prev->this;
                 }
 
-		if (!next_subvol) {
-			goto unwind;
-		}
+                if (!next_subvol) {
+                        goto unwind;
+                }
 
-		STACK_WIND (frame, dht_readdirp_cbk,
-			    next_subvol, next_subvol->fops->readdirp,
-			    local->fd, local->size, next_offset);
-		return 0;
-	}
+		if (conf->readdir_optimize == _gf_true) {
+                        if (next_subvol != local->first_up_subvol) {
+                                ret = dict_set_int32 (local->xattr,
+                                                      GF_READDIR_SKIP_DIRS, 1);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+					         "dict set failed");
+		        } else {
+                                 dict_del (local->xattr,
+                                           GF_READDIR_SKIP_DIRS);
+                        }
+                }
+
+                STACK_WIND (frame, dht_readdirp_cbk,
+                            next_subvol, next_subvol->fops->readdirp,
+                            local->fd, local->size, next_offset,
+                            local->xattr);
+                return 0;
+        }
 
 unwind:
-	if (op_ret < 0)
-		op_ret = 0;
+        if (op_ret < 0)
+                op_ret = 0;
 
-	DHT_STACK_UNWIND (readdirp, frame, op_ret, op_errno, &entries);
+        DHT_STACK_UNWIND (readdirp, frame, op_ret, op_errno, &entries, NULL);
 
-	gf_dirent_free (&entries);
+        gf_dirent_free (&entries);
 
         return 0;
 }
@@ -2678,34 +3196,33 @@ unwind:
 
 int
 dht_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int op_ret, int op_errno, gf_dirent_t *orig_entries)
+                 int op_ret, int op_errno, gf_dirent_t *orig_entries,
+                 dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	gf_dirent_t   entries;
-	gf_dirent_t  *orig_entry = NULL;
-	gf_dirent_t  *entry = NULL;
-	call_frame_t *prev = NULL;
-	xlator_t     *next_subvol = NULL;
+        dht_local_t  *local = NULL;
+        gf_dirent_t   entries;
+        gf_dirent_t  *orig_entry = NULL;
+        gf_dirent_t  *entry = NULL;
+        call_frame_t *prev = NULL;
+        xlator_t     *next_subvol = NULL;
         off_t         next_offset = 0;
-	int           count = 0;
+        int           count = 0;
         dht_layout_t *layout = 0;
-        dht_conf_t   *conf   = NULL;
         xlator_t     *subvol = 0;
 
-	INIT_LIST_HEAD (&entries.list);
-	prev = cookie;
-	local = frame->local;
-	conf  = this->private;
+        INIT_LIST_HEAD (&entries.list);
+        prev = cookie;
+        local = frame->local;
 
-	if (op_ret < 0)
-		goto done;
+        if (op_ret < 0)
+                goto done;
 
         if (!local->layout)
                 local->layout = dht_layout_get (this, local->fd->inode);
 
         layout = local->layout;
 
-	list_for_each_entry (orig_entry, (&orig_entries->list), list) {
+        list_for_each_entry (orig_entry, (&orig_entries->list), list) {
                 next_offset = orig_entry->d_off;
 
                 subvol = dht_layout_search (this, layout, orig_entry->d_name);
@@ -2718,11 +3235,10 @@ dht_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                 goto unwind;
                         }
 
-                        dht_itransform (this, prev->this, orig_entry->d_ino,
-                                        &entry->d_ino);
                         dht_itransform (this, prev->this, orig_entry->d_off,
                                         &entry->d_off);
 
+                        entry->d_ino  = orig_entry->d_ino;
                         entry->d_type = orig_entry->d_type;
                         entry->d_len  = orig_entry->d_len;
 
@@ -2730,7 +3246,7 @@ dht_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         count++;
                 }
         }
-	op_ret = count;
+        op_ret = count;
         /* We need to ensure that only the last subvolume's end-of-directory
          * notification is respected so that directory reading does not stop
          * before all subvolumes have been read. That could happen because the
@@ -2742,7 +3258,7 @@ dht_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 op_errno = 0;
 
 done:
-	if (count == 0) {
+        if (count == 0) {
                 /* non-zero next_offset means that
                    EOF is not yet hit on the current subvol
                 */
@@ -2752,23 +3268,23 @@ done:
                         next_subvol = prev->this;
                 }
 
-		if (!next_subvol) {
-			goto unwind;
-		}
+                if (!next_subvol) {
+                        goto unwind;
+                }
 
-		STACK_WIND (frame, dht_readdir_cbk,
-			    next_subvol, next_subvol->fops->readdir,
-			    local->fd, local->size, next_offset);
-		return 0;
-	}
+                STACK_WIND (frame, dht_readdir_cbk,
+                            next_subvol, next_subvol->fops->readdir,
+                            local->fd, local->size, next_offset, NULL);
+                return 0;
+        }
 
 unwind:
-	if (op_ret < 0)
-		op_ret = 0;
+        if (op_ret < 0)
+                op_ret = 0;
 
-	DHT_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries);
+        DHT_STACK_UNWIND (readdir, frame, op_ret, op_errno, &entries, NULL);
 
-	gf_dirent_free (&entries);
+        gf_dirent_free (&entries);
 
         return 0;
 }
@@ -2776,55 +3292,84 @@ unwind:
 
 int
 dht_do_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t yoff, int whichop)
+                off_t yoff, int whichop, dict_t *dict)
 {
-	dht_local_t  *local  = NULL;
-	dht_conf_t   *conf = NULL;
+        dht_local_t  *local  = NULL;
         int           op_errno = -1;
-	xlator_t     *xvol = NULL;
-	off_t         xoff = 0;
-
+        xlator_t     *xvol = NULL;
+        off_t         xoff = 0;
+        int           ret = 0;
+        dht_conf_t   *conf = NULL;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (this->private, err);
 
-	conf = this->private;
+        conf = this->private;
 
-	local = dht_local_init (frame);
-	if (!local) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		op_errno = ENOMEM;
-		goto err;
-	}
+        local = dht_local_init (frame, NULL, NULL, whichop);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	local->fd = fd_ref (fd);
-	local->size = size;
+        local->fd = fd_ref (fd);
+        local->size = size;
+        local->xattr_req = (dict)? dict_ref (dict) : NULL;
+        local->first_up_subvol = dht_first_up_subvol (this);
 
-	dht_deitransform (this, yoff, &xvol, (uint64_t *)&xoff);
+        dht_deitransform (this, yoff, &xvol, (uint64_t *)&xoff);
+
+        /* TODO: do proper readdir */
+        if (whichop == GF_FOP_READDIRP) {
+                if (dict)
+                        local->xattr = dict_ref (dict);
+                else
+                        local->xattr = dict_new ();
+
+                if (local->xattr) {
+                        ret = dict_set_uint32 (local->xattr,
+                                               conf->link_xattr_name, 256);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "failed to set '%s' key",
+                                        conf->link_xattr_name);
+			if (conf->readdir_optimize == _gf_true) {
+                                if (xvol != local->first_up_subvol) {
+				        ret = dict_set_int32 (local->xattr,
+			                               GF_READDIR_SKIP_DIRS, 1);
+				        if (ret)
+					        gf_log (this->name,
+                                                        GF_LOG_ERROR,
+						        "Dict set failed");
+                                } else {
+                                        dict_del (local->xattr,
+                                                  GF_READDIR_SKIP_DIRS);
+                                }
+			}
+                }
 
-	/* TODO: do proper readdir */
-        if (whichop == GF_FOP_READDIR)
-                STACK_WIND (frame, dht_readdir_cbk, xvol, xvol->fops->readdir,
-                            fd, size, xoff);
-        else
                 STACK_WIND (frame, dht_readdirp_cbk, xvol, xvol->fops->readdirp,
-                            fd, size, xoff);
+                            fd, size, xoff, local->xattr);
+        } else {
+                STACK_WIND (frame, dht_readdir_cbk, xvol, xvol->fops->readdir,
+                            fd, size, xoff, local->xattr);
+        }
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (readdir, frame, -1, op_errno, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-             off_t yoff)
+             off_t yoff, dict_t *xdata)
 {
         int          op = GF_FOP_READDIR;
         dht_conf_t  *conf = NULL;
@@ -2845,15 +3390,15 @@ dht_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
                 op = GF_FOP_READDIRP;
 
 out:
-        dht_do_readdir (frame, this, fd, size, yoff, op);
+        dht_do_readdir (frame, this, fd, size, yoff, op, 0);
         return 0;
 }
 
 int
 dht_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-              off_t yoff)
+              off_t yoff, dict_t *dict)
 {
-        dht_do_readdir (frame, this, fd, size, yoff, GF_FOP_READDIRP);
+        dht_do_readdir (frame, this, fd, size, yoff, GF_FOP_READDIRP, dict);
         return 0;
 }
 
@@ -2861,89 +3406,88 @@ dht_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 
 int
 dht_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int op_ret, int op_errno)
+                  int op_ret, int op_errno, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = 0;
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
 
 
-	local = frame->local;
+        local = frame->local;
 
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1)
-			local->op_errno = op_errno;
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1)
+                        local->op_errno = op_errno;
 
-		if (op_ret == 0)
-			local->op_ret = 0;
-	}
-	UNLOCK (&frame->lock);
+                if (op_ret == 0)
+                        local->op_ret = 0;
+        }
+        UNLOCK (&frame->lock);
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt))
-		DHT_STACK_UNWIND (fsyncdir, frame, local->op_ret, local->op_errno);
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt))
+                DHT_STACK_UNWIND (fsyncdir, frame, local->op_ret,
+                                  local->op_errno, xdata);
 
         return 0;
 }
 
 
 int
-dht_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync)
+dht_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              int datasync, dict_t *xdata)
 {
-	dht_local_t  *local  = NULL;
-	dht_conf_t   *conf = NULL;
+        dht_local_t  *local  = NULL;
+        dht_conf_t   *conf = NULL;
         int           op_errno = -1;
-	int           i = -1;
-
+        int           i = -1;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
         VALIDATE_OR_GOTO (this->private, err);
 
-	conf = this->private;
+        conf = this->private;
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
+        local = dht_local_init (frame, NULL, NULL, GF_FOP_FSYNCDIR);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	local->fd = fd_ref (fd);
-	local->call_cnt = conf->subvolume_cnt;
+        local->fd = fd_ref (fd);
+        local->call_cnt = conf->subvolume_cnt;
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		STACK_WIND (frame, dht_fsyncdir_cbk,
-			    conf->subvolumes[i],
-			    conf->subvolumes[i]->fops->fsyncdir,
-			    fd, datasync);
-	}
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                STACK_WIND (frame, dht_fsyncdir_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->fsyncdir,
+                            fd, datasync, xdata);
+        }
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (fsyncdir, frame, -1, op_errno);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int op_ret, int op_errno,
+                 int op_ret, int op_errno,
                  inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-                 struct iatt *postparent)
+                 struct iatt *postparent, dict_t *xdata)
 {
-	call_frame_t *prev = NULL;
-	int           ret = -1;
+        xlator_t     *prev = NULL;
+        int           ret = -1;
         dht_local_t  *local = NULL;
 
 
-	if (op_ret == -1)
-		goto out;
+        if (op_ret == -1)
+                goto out;
 
         local = frame->local;
         if (!local) {
@@ -2952,38 +3496,39 @@ dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-	prev = cookie;
+        prev = cookie;
 
-	dht_itransform (this, prev->this, stbuf->ia_ino, &stbuf->ia_ino);
         if (local->loc.parent) {
-                preparent->ia_ino = local->loc.parent->ino;
-                postparent->ia_ino = local->loc.parent->ino;
 
-                WIPE (preparent);
-                WIPE (postparent);
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           preparent, 0);
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
         }
 
-        ret = dht_layout_preset (this, prev->this, inode);
-	if (ret < 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"could not set pre-set layout for subvolume %s",
-			prev->this->name);
-		op_ret   = -1;
-		op_errno = EINVAL;
-		goto out;
-	}
+        ret = dht_layout_preset (this, prev, inode);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "could not set pre-set layout for subvolume %s",
+                        prev? prev->name: NULL);
+                op_ret   = -1;
+                op_errno = EINVAL;
+                goto out;
+        }
+        if (local->linked == _gf_true)
+                dht_linkfile_attr_heal (frame, this);
 out:
-        /* 
-         * FIXME: ia_size and st_blocks of preparent and postparent do not have 
+        /*
+         * FIXME: ia_size and st_blocks of preparent and postparent do not have
          * correct values. since, preparent and postparent buffers correspond
          * to a directory these two members should have values equal to sum of
          * corresponding values from each of the subvolume.
          * See dht_iatt_merge for reference.
-         */ 
-
-	DHT_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, stbuf, preparent,
-                          postparent);
-	return 0;
+         */
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
+        DHT_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, stbuf,
+                          preparent, postparent, xdata);
+        return 0;
 }
 
 int
@@ -2991,406 +3536,376 @@ dht_mknod_linkfile_create_cbk (call_frame_t *frame, void *cookie,
                                xlator_t *this,
                                int32_t op_ret, int32_t op_errno,
                                inode_t *inode, struct iatt *stbuf,
-                               struct iatt *preparent, struct iatt *postparent)
+                               struct iatt *preparent, struct iatt *postparent,
+                               dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	xlator_t     *cached_subvol = NULL;
+        dht_local_t  *local = NULL;
+        xlator_t     *cached_subvol = NULL;
 
         if (op_ret == -1)
                 goto err;
 
-	local = frame->local;
-	cached_subvol = local->cached_subvol;
+        local = frame->local;
+        if (!local || !local->cached_subvol) {
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        cached_subvol = local->cached_subvol;
 
-        STACK_WIND (frame, dht_newfile_cbk,
-                    cached_subvol, cached_subvol->fops->mknod,
-                    &local->loc, local->mode, local->rdev,
-                    local->params);
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)cached_subvol,
+                           cached_subvol, cached_subvol->fops->mknod,
+                           &local->loc, local->mode, local->rdev, local->umask,
+                           local->params);
 
         return 0;
- err:
- 	DHT_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL);	
- 	return 0;
+err:
+        DHT_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                          NULL);
+        return 0;
 }
 
 int
 dht_mknod (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dev_t rdev, dict_t *params)
+           loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *params)
 {
-	xlator_t    *subvol = NULL;
-	int          op_errno = -1;
-        int          ret = -1;
+        xlator_t    *subvol = NULL;
+        int          op_errno = -1;
         xlator_t    *avail_subvol = NULL;
-	dht_conf_t  *conf = NULL;
-	dht_local_t *local = NULL;
-
-	VALIDATE_OR_GOTO (frame, err);
-	VALIDATE_OR_GOTO (this, err);
-	VALIDATE_OR_GOTO (loc, err);
+        dht_local_t *local = NULL;
 
-	conf = this->private;
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
 
         dht_get_du_info (frame, this, loc);
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
-
-	subvol = dht_subvol_get_hashed (this, loc);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no subvolume in layout for path=%s",
-			loc->path);
-		op_errno = ENOENT;
-		goto err;
-	}
-
-        ret = loc_dup (loc, &local->loc);
-        if (ret == -1) {
+        local = dht_local_init (frame, loc, NULL, GF_FOP_MKNOD);
+        if (!local) {
                 op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
+                goto err;
+        }
+
+        subvol = dht_subvol_get_hashed (this, loc);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        loc->path);
+                op_errno = ENOENT;
                 goto err;
         }
 
         if (!dht_is_subvol_filled (this, subvol)) {
                 gf_log (this->name, GF_LOG_TRACE,
                         "creating %s on %s", loc->path, subvol->name);
-                
-                STACK_WIND (frame, dht_newfile_cbk,
-                            subvol, subvol->fops->mknod,
-                            loc, mode, rdev, params);
+
+                STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol,
+                                   subvol, subvol->fops->mknod, loc, mode,
+                                   rdev, umask, params);
         } else {
-                avail_subvol = dht_free_disk_available_subvol (this, subvol);
+
+                avail_subvol = dht_free_disk_available_subvol (this, subvol,
+                                                               local);
                 if (avail_subvol != subvol) {
-                        /* Choose the minimum filled volume, and create the 
+                        /* Choose the minimum filled volume, and create the
                            files there */
 
                         local->params = dict_ref (params);
                         local->cached_subvol = avail_subvol;
-                        local->mode = mode; 
+                        local->mode = mode;
                         local->rdev = rdev;
-                        
-                        dht_linkfile_create (frame, 
+                        local->umask = umask;
+                        dht_linkfile_create (frame,
                                              dht_mknod_linkfile_create_cbk,
-                                             avail_subvol, subvol, loc);
+                                             this, avail_subvol, subvol, loc);
                 } else {
                         gf_log (this->name, GF_LOG_TRACE,
                                 "creating %s on %s", loc->path, subvol->name);
-                        
-                        STACK_WIND (frame, dht_newfile_cbk,
-                                    subvol, subvol->fops->mknod,
-                                    loc, mode, rdev, params);
+
+                        STACK_WIND_COOKIE (frame, dht_newfile_cbk,
+                                           (void *)subvol, subvol,
+                                           subvol->fops->mknod, loc, mode,
+                                           rdev, umask, params);
                 }
         }
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
+                          NULL, NULL, NULL, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_symlink (call_frame_t *frame, xlator_t *this,
-	     const char *linkname, loc_t *loc, dict_t *params)
+             const char *linkname, loc_t *loc, mode_t umask, dict_t *params)
 {
-	xlator_t    *subvol = NULL;
-	int          op_errno = -1;
+        xlator_t    *subvol = NULL;
+        int          op_errno = -1;
         dht_local_t *local = NULL;
-        int          ret = -1;
-
-
-	VALIDATE_OR_GOTO (frame, err);
-	VALIDATE_OR_GOTO (this, err);
-	VALIDATE_OR_GOTO (loc, err);
-
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
 
-	subvol = dht_subvol_get_hashed (this, loc);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no subvolume in layout for path=%s",
-			loc->path);
-		op_errno = ENOENT;
-		goto err;
-	}
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
 
-        ret = loc_copy (&local->loc, loc);
-        if (ret == -1) {
-                gf_log (this->name, GF_LOG_TRACE, "Failed to copy loc");
+        local = dht_local_init (frame, loc, NULL, GF_FOP_SYMLINK);
+        if (!local) {
                 op_errno = ENOMEM;
                 goto err;
         }
 
-	gf_log (this->name, GF_LOG_TRACE,
-		"creating %s on %s", loc->path, subvol->name);
+        subvol = dht_subvol_get_hashed (this, loc);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        loc->path);
+                op_errno = ENOENT;
+                goto err;
+        }
 
-	STACK_WIND (frame, dht_newfile_cbk,
-		    subvol, subvol->fops->symlink,
-		    linkname, loc, params);
+        gf_log (this->name, GF_LOG_TRACE,
+                "creating %s on %s", loc->path, subvol->name);
 
-	return 0;
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
+                           subvol->fops->symlink, linkname, loc, umask,
+                           params);
+
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (link, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (link, frame, -1, op_errno,
+                          NULL, NULL, NULL, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
-dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+dht_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+            dict_t *xdata)
 {
-	xlator_t    *cached_subvol = NULL;
-	xlator_t    *hashed_subvol = NULL;
-        int          ret = -1;
-	int          op_errno = -1;
-	dht_local_t *local = NULL;
-
+        xlator_t    *cached_subvol = NULL;
+        xlator_t    *hashed_subvol = NULL;
+        int          op_errno = -1;
+        dht_local_t *local = NULL;
 
-	VALIDATE_OR_GOTO (frame, err);
-	VALIDATE_OR_GOTO (this, err);
-	VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
 
         if (dht_filter_loc_subvol_key (this, loc, &local->loc,
                                        &cached_subvol)) {
-                gf_log (this->name, GF_LOG_NORMAL,
+                gf_log (this->name, GF_LOG_INFO,
                         "unlinking %s on %s (given path %s)",
                         local->loc.path, cached_subvol->name, loc->path);
                 STACK_WIND (frame, dht_unlink_cbk,
                             cached_subvol, cached_subvol->fops->unlink,
-                            &local->loc);
+                            &local->loc, xflag, xdata);
                 goto done;
         }
 
-	cached_subvol = dht_subvol_get_cached (this, loc->inode);
-	if (!cached_subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+        local = dht_local_init (frame, loc, NULL, GF_FOP_UNLINK);
+        if (!local) {
+                op_errno = ENOMEM;
 
-	hashed_subvol = dht_subvol_get_hashed (this, loc);
-	if (!hashed_subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no subvolume in layout for path=%s",
-			loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+                goto err;
+        }
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
+        hashed_subvol = dht_subvol_get_hashed (this, loc);
+        /* Dont fail unlink if hashed_subvol is NULL which can be the result
+         * of layout anomaly */
+        if (!hashed_subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        loc->path);
+        }
 
-	ret = loc_copy (&local->loc, loc);
-	if (ret == -1) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
+        cached_subvol = local->cached_subvol;
+        if (!cached_subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	if (hashed_subvol != cached_subvol) {
-		STACK_WIND (frame, dht_unlink_linkfile_cbk,
-			    hashed_subvol, hashed_subvol->fops->unlink, loc);
+        local->flags = xflag;
+        if (hashed_subvol && hashed_subvol != cached_subvol) {
+                STACK_WIND (frame, dht_unlink_linkfile_cbk,
+                            hashed_subvol, hashed_subvol->fops->unlink, loc,
+                            xflag, xdata);
         } else {
                 STACK_WIND (frame, dht_unlink_cbk,
-                            cached_subvol, cached_subvol->fops->unlink, loc);
+                            cached_subvol, cached_subvol->fops->unlink, loc,
+                            xflag, xdata);
         }
 done:
-	return 0;
+        return 0;
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	      int op_ret, int op_errno,
+              int op_ret, int op_errno,
               inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
-              struct iatt *postparent)
+              struct iatt *postparent, dict_t *xdata)
 {
         call_frame_t *prev = NULL;
-	dht_layout_t *layout = NULL;
-	dht_local_t  *local = NULL;
+        dht_layout_t *layout = NULL;
+        dht_local_t  *local = NULL;
 
         prev = cookie;
-	local = frame->local;
+
+        local = frame->local;
 
         if (op_ret == -1)
                 goto out;
 
-	layout = dht_layout_for_subvol (this, prev->this);
-	if (!layout) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no pre-set layout for subvolume %s",
-			prev->this->name);
-		op_ret   = -1;
-		op_errno = EINVAL;
-		goto out;
-	}
-
-	stbuf->ia_ino = local->loc.inode->ino;
-
-        preparent->ia_ino = local->loc2.parent->ino;
-        postparent->ia_ino = local->loc2.parent->ino;
-
-        WIPE (preparent);
-        WIPE (postparent);
+        layout = dht_layout_for_subvol (this, prev->this);
+        if (!layout) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no pre-set layout for subvolume %s",
+                        prev->this->name);
+                op_ret   = -1;
+                op_errno = EINVAL;
+                goto out;
+        }
 
+        if (local->loc.parent) {
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           preparent, 0);
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
+        }
+        if (local->linked == _gf_true) {
+                local->stbuf = *stbuf;
+                dht_linkfile_attr_heal (frame, this);
+        }
 out:
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
         DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent,
-                          postparent);
+                          postparent, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_link_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int op_ret, int op_errno,
+                       int op_ret, int op_errno,
                        inode_t *inode, struct iatt *stbuf,
-                       struct iatt *preparent, struct iatt *postparent)
+                       struct iatt *preparent, struct iatt *postparent,
+                       dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	xlator_t     *srcvol = NULL;
-
+        dht_local_t  *local = NULL;
+        xlator_t     *srcvol = NULL;
 
-	if (op_ret == -1)
-		goto err;
+        if (op_ret == -1)
+                goto err;
 
-	local = frame->local;
-	srcvol = local->linkfile.srcvol;
+        local = frame->local;
+        srcvol = local->linkfile.srcvol;
 
-	STACK_WIND (frame, dht_link_cbk,
-		    srcvol, srcvol->fops->link,
-		    &local->loc, &local->loc2);
+        STACK_WIND (frame, dht_link_cbk, srcvol, srcvol->fops->link,
+                    &local->loc, &local->loc2, xdata);
 
-	return 0;
+        return 0;
 
 err:
-	DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent,
-                          postparent);
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
+        DHT_STACK_UNWIND (link, frame, op_ret, op_errno, inode, stbuf, preparent,
+                          postparent, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_link (call_frame_t *frame, xlator_t *this,
-	  loc_t *oldloc, loc_t *newloc)
+          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
-	xlator_t    *cached_subvol = NULL;
-	xlator_t    *hashed_subvol = NULL;
-	int          op_errno = -1;
-	int          ret = -1;
-	dht_local_t *local = NULL;
-
-
-	VALIDATE_OR_GOTO (frame, err);
-	VALIDATE_OR_GOTO (this, err);
-	VALIDATE_OR_GOTO (oldloc, err);
-	VALIDATE_OR_GOTO (newloc, err);
-
-	cached_subvol = dht_subvol_get_cached (this, oldloc->inode);
-	if (!cached_subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for path=%s", oldloc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+        xlator_t    *cached_subvol = NULL;
+        xlator_t    *hashed_subvol = NULL;
+        int          op_errno = -1;
+        int          ret = -1;
+        dht_local_t *local = NULL;
 
-	hashed_subvol = dht_subvol_get_hashed (this, newloc);
-	if (!hashed_subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no subvolume in layout for path=%s",
-			newloc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (oldloc, err);
+        VALIDATE_OR_GOTO (newloc, err);
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
+        local = dht_local_init (frame, oldloc, NULL, GF_FOP_LINK);
+        if (!local) {
+                op_errno = ENOMEM;
 
-	ret = loc_copy (&local->loc, oldloc);
-	if (ret == -1) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
+                goto err;
+        }
 
-	ret = loc_copy (&local->loc2, newloc);
-	if (ret == -1) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
+        cached_subvol = local->cached_subvol;
+        if (!cached_subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", oldloc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	if (hashed_subvol != cached_subvol) {
-                memcpy (local->gfid, oldloc->inode->gfid, 16);
-		dht_linkfile_create (frame, dht_link_linkfile_cbk,
-				     cached_subvol, hashed_subvol, newloc);
-	} else {
-		STACK_WIND (frame, dht_link_cbk,
-			    cached_subvol, cached_subvol->fops->link,
-			    oldloc, newloc);
-	}
+        hashed_subvol = dht_subvol_get_hashed (this, newloc);
+        if (!hashed_subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        newloc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	return 0;
+        ret = loc_copy (&local->loc2, newloc);
+        if (ret == -1) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        if (hashed_subvol != cached_subvol) {
+                uuid_copy (local->gfid, oldloc->inode->gfid);
+                dht_linkfile_create (frame, dht_link_linkfile_cbk, this,
+                                     cached_subvol, hashed_subvol, newloc);
+        } else {
+                STACK_WIND (frame, dht_link_cbk,
+                            cached_subvol, cached_subvol->fops->link,
+                            oldloc, newloc, xdata);
+        }
+
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int op_ret, int op_errno,
-		 fd_t *fd, inode_t *inode, struct iatt *stbuf,
-                 struct iatt *preparent, struct iatt *postparent)
+                int op_ret, int op_errno,
+                fd_t *fd, inode_t *inode, struct iatt *stbuf,
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-	call_frame_t *prev = NULL;
-	int           ret = -1;
+        call_frame_t *prev = NULL;
+        int           ret = -1;
         dht_local_t  *local = NULL;
 
-	if (op_ret == -1)
-		goto out;
+        if (op_ret == -1)
+                goto out;
 
         local = frame->local;
         if (!local) {
@@ -3399,107 +3914,99 @@ dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-	prev = cookie;
+        prev = cookie;
 
-	dht_itransform (this, prev->this, stbuf->ia_ino, &stbuf->ia_ino);
         if (local->loc.parent) {
-                preparent->ia_ino = local->loc.parent->ino;
-                postparent->ia_ino = local->loc.parent->ino;
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           preparent, 0);
 
-                WIPE (preparent);
-                WIPE (postparent);
+                dht_inode_ctx_time_update (local->loc.parent, this,
+                                           postparent, 1);
         }
 
         ret = dht_layout_preset (this, prev->this, inode);
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"could not set preset layout for subvol %s",
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "could not set preset layout for subvol %s",
                         prev->this->name);
-		op_ret   = -1;
-		op_errno = EINVAL;
-		goto out;
-	}
-
+                op_ret   = -1;
+                op_errno = EINVAL;
+                goto out;
+        }
+        if (local->linked == _gf_true) {
+                local->stbuf = *stbuf;
+                dht_linkfile_attr_heal (frame, this);
+        }
 out:
-	DHT_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, stbuf, preparent,
-                          postparent);
-	return 0;
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
+        DHT_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, stbuf, preparent,
+                          postparent, NULL);
+        return 0;
 }
 
 
 int
 dht_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-				xlator_t *this,
-				int32_t op_ret, int32_t op_errno,
+                                xlator_t *this,
+                                int32_t op_ret, int32_t op_errno,
                                 inode_t *inode, struct iatt *stbuf,
-                                struct iatt *preparent, struct iatt *postparent)
+                                struct iatt *preparent, struct iatt *postparent,
+                                dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	xlator_t     *cached_subvol = NULL;
+        dht_local_t  *local = NULL;
+        xlator_t     *cached_subvol = NULL;
 
         if (op_ret == -1)
                 goto err;
 
-	local = frame->local;
-	cached_subvol = local->cached_subvol;
+        local = frame->local;
+        cached_subvol = local->cached_subvol;
 
         STACK_WIND (frame, dht_create_cbk,
                     cached_subvol, cached_subvol->fops->create,
                     &local->loc, local->flags, local->mode,
-                    local->fd, local->params);
+                    local->umask, local->fd, local->params);
 
         return 0;
- err:
- 	DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
- 	return 0;
+err:
+        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL, NULL, NULL);
+        return 0;
 }
 
 int
 dht_create (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, int32_t flags, mode_t mode,
-            fd_t *fd, dict_t *params)
+            loc_t *loc, int32_t flags, mode_t mode,
+            mode_t umask, fd_t *fd, dict_t *params)
 {
-	int          op_errno = -1;
-        int          ret = -1;
-	xlator_t    *subvol = NULL;
-	dht_conf_t  *conf = NULL;
+        int          op_errno = -1;
+        xlator_t    *subvol = NULL;
         dht_local_t *local = NULL;
         xlator_t    *avail_subvol = NULL;
 
-	VALIDATE_OR_GOTO (frame, err);
-	VALIDATE_OR_GOTO (this, err);
-	VALIDATE_OR_GOTO (loc, err);
-
-	conf = this->private;
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
 
         dht_get_du_info (frame, this, loc);
 
-	local = dht_local_init (frame);
-	if (!local) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		op_errno = ENOMEM;
-		goto err;
-	}
+        local = dht_local_init (frame, loc, fd, GF_FOP_CREATE);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
         if (dht_filter_loc_subvol_key (this, loc, &local->loc,
                                        &subvol)) {
-                gf_log (this->name, GF_LOG_NORMAL,
+                gf_log (this->name, GF_LOG_INFO,
                         "creating %s on %s (got create on %s)",
                         local->loc.path, subvol->name, loc->path);
                 STACK_WIND (frame, dht_create_cbk,
                             subvol, subvol->fops->create,
-                            &local->loc, flags, mode, fd, params);
+                            &local->loc, flags, mode, umask, fd, params);
                 goto done;
         }
 
-        ret = loc_dup (loc, &local->loc);
-        if (ret == -1) {
-                op_errno = ENOMEM;
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                goto err;
-        }
         subvol = dht_subvol_get_hashed (this, loc);
         if (!subvol) {
                 gf_log (this->name, GF_LOG_DEBUG,
@@ -3514,156 +4021,158 @@ dht_create (call_frame_t *frame, xlator_t *this,
                         "creating %s on %s", loc->path, subvol->name);
                 STACK_WIND (frame, dht_create_cbk,
                             subvol, subvol->fops->create,
-                            loc, flags, mode, fd, params);
+                            loc, flags, mode, umask, fd, params);
                 goto done;
         }
-        /* Choose the minimum filled volume, and create the 
+        /* Choose the minimum filled volume, and create the
            files there */
-        /* TODO */
-        avail_subvol = dht_free_disk_available_subvol (this, subvol);
+        avail_subvol = dht_free_disk_available_subvol (this, subvol, local);
         if (avail_subvol != subvol) {
-                local->fd = fd_ref (fd);
                 local->params = dict_ref (params);
                 local->flags = flags;
                 local->mode = mode;
-
+                local->umask = umask;
                 local->cached_subvol = avail_subvol;
                 local->hashed_subvol = subvol;
                 gf_log (this->name, GF_LOG_TRACE,
                         "creating %s on %s (link at %s)", loc->path,
                         avail_subvol->name, subvol->name);
-                dht_linkfile_create (frame,
-                                     dht_create_linkfile_create_cbk,
-                                     avail_subvol, subvol, loc);
+                dht_linkfile_create (frame, dht_create_linkfile_create_cbk,
+                                     this, avail_subvol, subvol, loc);
                 goto done;
         }
         gf_log (this->name, GF_LOG_TRACE,
                 "creating %s on %s", loc->path, subvol->name);
         STACK_WIND (frame, dht_create_cbk,
                     subvol, subvol->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, params);
 done:
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_mkdir_selfheal_cbk (call_frame_t *frame, void *cookie,
-			xlator_t *this,
-			int32_t op_ret, int32_t op_errno)
+                        xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	dht_local_t   *local = NULL;
-	dht_layout_t  *layout = NULL;
+        dht_local_t   *local = NULL;
+        dht_layout_t  *layout = NULL;
 
+        local = frame->local;
+        layout = local->selfheal.layout;
 
-	local = frame->local;
-	layout = local->selfheal.layout;
-
-	if (op_ret == 0) {
+        if (op_ret == 0) {
                 dht_layout_set (this, local->inode, layout);
-		local->stbuf.ia_ino = local->ia_ino;
                 if (local->loc.parent) {
-                        local->preparent.ia_ino = local->loc.parent->ino;
-                        local->postparent.ia_ino = local->loc.parent->ino;
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->preparent, 0);
 
-                        WIPE (&local->preparent);
-                        WIPE (&local->postparent);
+                        dht_inode_ctx_time_update (local->loc.parent, this,
+                                                   &local->postparent, 1);
                 }
-	}
+        }
 
-	DHT_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
-			  local->inode, &local->stbuf, &local->preparent,
-                          &local->postparent);
+        DHT_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
+                          local->inode, &local->stbuf, &local->preparent,
+                          &local->postparent, NULL);
 
-	return 0;
+        return 0;
 }
 
 int
 dht_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int op_ret, int op_errno, inode_t *inode, struct iatt *stbuf,
-               struct iatt *preparent, struct iatt *postparent)
+               struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = 0;
-	int           ret = -1;
-        int           subvol_filled = 0;
-	call_frame_t *prev = NULL;
-	dht_layout_t *layout = NULL;
-	dht_conf_t   *conf = NULL;
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        int           ret = -1;
+        gf_boolean_t subvol_filled = _gf_false;
+        call_frame_t *prev = NULL;
+        dht_layout_t *layout = NULL;
 
-	conf = this->private;
-	local = frame->local;
-	prev  = cookie;
-	layout = local->layout;
+        local = frame->local;
+        prev  = cookie;
+        layout = local->layout;
 
         subvol_filled = dht_is_subvol_filled (this, prev->this);
 
-	LOCK (&frame->lock);
-	{
+        LOCK (&frame->lock);
+        {
                 if (subvol_filled && (op_ret != -1)) {
                         ret = dht_layout_merge (this, layout, prev->this,
                                                 -1, ENOSPC, NULL);
                 } else {
+			if (op_ret == -1 && op_errno == EEXIST)
+				/* Very likely just a race between mkdir and
+				   self-heal (from lookup of a concurrent mkdir
+				   attempt).
+				   Ignore error for now. layout setting will
+				   anyways fail if this was a different (old)
+				   pre-existing different directory.
+				*/
+				op_ret = 0;
                         ret = dht_layout_merge (this, layout, prev->this,
                                                 op_ret, op_errno, NULL);
                 }
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: failed to merge layouts", local->loc.path);
 
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			goto unlock;
-		}
-		dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        goto unlock;
+                }
+                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
                 dht_iatt_merge (this, &local->preparent, preparent, prev->this);
                 dht_iatt_merge (this, &local->postparent, postparent,
                                 prev->this);
-
-                if (prev->this == dht_first_up_subvol (this)) {
-                        local->ia_ino = local->stbuf.ia_ino;
-                }
-
-	}
+        }
 unlock:
-	UNLOCK (&frame->lock);
+        UNLOCK (&frame->lock);
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt)) {
-		dht_selfheal_new_directory (frame, dht_mkdir_selfheal_cbk,
-					    layout);
-	}
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+                dht_selfheal_new_directory (frame, dht_mkdir_selfheal_cbk,
+                                            layout);
+        }
 
         return 0;
 }
 
 int
-dht_mkdir_hashed_cbk (call_frame_t *frame, void *cookie, 
-		      xlator_t *this, int op_ret, int op_errno,
+dht_mkdir_hashed_cbk (call_frame_t *frame, void *cookie,
+                      xlator_t *this, int op_ret, int op_errno,
                       inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent)
+                      struct iatt *preparent, struct iatt *postparent,
+                      dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	int           ret = -1;
-	call_frame_t *prev = NULL;
-	dht_layout_t *layout = NULL;
-	dht_conf_t   *conf = NULL;
-	int           i = 0;
-	xlator_t     *hashed_subvol = NULL;
+        dht_local_t  *local = NULL;
+        int           ret = -1;
+        call_frame_t *prev = NULL;
+        dht_layout_t *layout = NULL;
+        dht_conf_t   *conf = NULL;
+        int           i = 0;
+        xlator_t     *hashed_subvol = NULL;
 
         VALIDATE_OR_GOTO (this->private, err);
 
-	local = frame->local;
-	prev  = cookie;
-	layout = local->layout;
-	conf = this->private;
-	hashed_subvol = local->hashed_subvol;
+        local = frame->local;
+        prev  = cookie;
+        layout = local->layout;
+        conf = this->private;
+        hashed_subvol = local->hashed_subvol;
 
-        if (uuid_is_null (local->loc.inode->gfid) && !op_ret)
-                memcpy (local->loc.inode->gfid, stbuf->ia_gfid, 16);
+        if (uuid_is_null (local->loc.gfid) && !op_ret)
+                uuid_copy (local->loc.gfid, stbuf->ia_gfid);
 
         if (dht_is_subvol_filled (this, hashed_subvol))
                 ret = dht_layout_merge (this, layout, prev->this,
@@ -3671,49 +4180,55 @@ dht_mkdir_hashed_cbk (call_frame_t *frame, void *cookie,
         else
                 ret = dht_layout_merge (this, layout, prev->this,
                                         op_ret, op_errno, NULL);
-        
-	if (op_ret == -1) {
-		local->op_errno = op_errno;
-		goto err;
-	}
-	local->op_ret = 0;
 
-	dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+        /* TODO: we may have to return from the function
+           if layout merge fails. For now, lets just log an error */
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to merge layouts", local->loc.path);
+
+        if (op_ret == -1) {
+                local->op_errno = op_errno;
+                goto err;
+        }
+        local->op_ret = 0;
+
+        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
         dht_iatt_merge (this, &local->preparent, preparent, prev->this);
         dht_iatt_merge (this, &local->postparent, postparent, prev->this);
 
-	local->ia_ino = local->stbuf.ia_ino;
+        local->call_cnt = conf->subvolume_cnt - 1;
 
-	local->call_cnt = conf->subvolume_cnt - 1;
-	
-	if (local->call_cnt == 0) {
-		dht_selfheal_directory (frame, dht_mkdir_selfheal_cbk,
-					&local->loc, layout);
-	}
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		if (conf->subvolumes[i] == hashed_subvol)
-			continue;
-		STACK_WIND (frame, dht_mkdir_cbk,
-			    conf->subvolumes[i],
-			    conf->subvolumes[i]->fops->mkdir,
-			    &local->loc, local->mode, local->params);
-	}
-	return 0;
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, stbuf->ia_gfid);
+        if (local->call_cnt == 0) {
+                dht_selfheal_directory (frame, dht_mkdir_selfheal_cbk,
+                                        &local->loc, layout);
+        }
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (conf->subvolumes[i] == hashed_subvol)
+                        continue;
+                STACK_WIND (frame, dht_mkdir_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->mkdir, &local->loc,
+                            local->mode, local->umask, local->params);
+        }
+        return 0;
 err:
-	DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL, NULL);
         return 0;
 }
 
 
-int
+ int
 dht_mkdir (call_frame_t *frame, xlator_t *this,
-	   loc_t *loc, mode_t mode, dict_t *params)
+           loc_t *loc, mode_t mode, mode_t umask, dict_t *params)
 {
-	dht_local_t  *local  = NULL;
-	dht_conf_t   *conf = NULL;
+        dht_local_t  *local  = NULL;
+        dht_conf_t   *conf = NULL;
         int           op_errno = -1;
-	int           ret = -1;
-	xlator_t     *hashed_subvol = NULL;
+        xlator_t     *hashed_subvol = NULL;
 
 
         VALIDATE_OR_GOTO (frame, err);
@@ -3723,148 +4238,232 @@ dht_mkdir (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (loc->path, err);
         VALIDATE_OR_GOTO (this->private, err);
 
-	conf = this->private;
+        conf = this->private;
 
         dht_get_du_info (frame, this, loc);
 
-	local = dht_local_init (frame);
-	if (!local) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		op_errno = ENOMEM;
-		goto err;
-	}
-
-	hashed_subvol = dht_subvol_get_hashed (this, loc);
+        local = dht_local_init (frame, loc, NULL, GF_FOP_MKDIR);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	if (hashed_subvol == NULL) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"hashed subvol not found for %s",
+        hashed_subvol = dht_subvol_get_hashed (this, loc);
+        if (hashed_subvol == NULL) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "hashed subvol not found for %s",
                         loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	local->hashed_subvol = hashed_subvol;
-	local->inode = inode_ref (loc->inode);
-	ret = loc_copy (&local->loc, loc);
-	local->mode = mode;
-
-	if (ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		op_errno = ENOMEM;
-		goto err;
-	}
+                op_errno = EINVAL;
+                goto err;
+        }
 
+        local->hashed_subvol = hashed_subvol;
+        local->mode = mode;
+        local->umask = umask;
         local->params = dict_ref (params);
+        local->inode  = inode_ref (loc->inode);
 
-	local->layout = dht_layout_new (this, conf->subvolume_cnt);
-	if (!local->layout) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		op_errno = ENOMEM;
-		goto err;
-	}
+        local->layout = dht_layout_new (this, conf->subvolume_cnt);
+        if (!local->layout) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	STACK_WIND (frame, dht_mkdir_hashed_cbk,
-		    hashed_subvol,
-		    hashed_subvol->fops->mkdir,
-		    loc, mode, params);
+        STACK_WIND (frame, dht_mkdir_hashed_cbk,
+                    hashed_subvol,
+                    hashed_subvol->fops->mkdir,
+                    loc, mode, umask, params);
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_rmdir_selfheal_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int op_ret, int op_errno)
+                        int op_ret, int op_errno, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
+        dht_local_t  *local = NULL;
 
-	local = frame->local;
+        local = frame->local;
+
+        DHT_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
+                          &local->preparent, &local->postparent, NULL);
+
+        return 0;
+}
+
+
+int
+dht_rmdir_hashed_subvol_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int op_ret, int op_errno, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        call_frame_t *prev = NULL;
+
+        local = frame->local;
+        prev  = cookie;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        local->op_ret   = -1;
+                        if (op_errno != ENOENT && op_errno != EACCES) {
+                                local->need_selfheal = 1;
+                        }
+
+
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "rmdir on %s for %s failed (%s)",
+                                prev->this->name, local->loc.path,
+                                strerror (op_errno));
+                        goto unlock;
+                }
+
+                dht_iatt_merge (this, &local->preparent, preparent, prev->this);
+                dht_iatt_merge (this, &local->postparent, postparent,
+                                prev->this);
 
-        if (local->loc.parent) {
-                local->preparent.ia_ino = local->loc.parent->ino;
-                local->postparent.ia_ino = local->loc.parent->ino;
         }
+unlock:
+        UNLOCK (&frame->lock);
+
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+               if (local->need_selfheal) {
+                        local->layout =
+                                dht_layout_get (this, local->loc.inode);
 
-	DHT_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent);
+                        /* TODO: neater interface needed below */
+                        local->stbuf.ia_type = local->loc.inode->ia_type;
 
-	return 0;
+                        uuid_copy (local->gfid, local->loc.inode->gfid);
+                        dht_selfheal_restore (frame, dht_rmdir_selfheal_cbk,
+                                              &local->loc, local->layout);
+               } else {
+
+                        if (local->loc.parent) {
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->preparent,
+                                                           0);
+
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->postparent,
+                                                           1);
+                        }
+
+                        DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
+                                          local->op_errno, &local->preparent,
+                                          &local->postparent, NULL);
+               }
+        }
+
+        return 0;
 }
 
 
 int
 dht_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-	       int op_ret, int op_errno, struct iatt *preparent,
-               struct iatt *postparent)
+               int op_ret, int op_errno, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = 0;
-	call_frame_t *prev = NULL;
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        call_frame_t *prev = NULL;
+        int           done = 0;
 
-	local = frame->local;
-	prev  = cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			local->op_ret   = -1;
-
-			if (op_errno != ENOENT)
-				local->need_selfheal = 1;
-
-			gf_log (this->name, GF_LOG_DEBUG,
-				"rmdir on %s for %s failed (%s)",
-				prev->this->name, local->loc.path,
-				strerror (op_errno));
-			goto unlock;
-		}
+        local = frame->local;
+        prev  = cookie;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        local->op_ret   = -1;
+
+                        if (op_errno != ENOENT && op_errno != EACCES) {
+                                local->need_selfheal = 1;
+                        }
 
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "rmdir on %s for %s failed (%s)",
+                                prev->this->name, local->loc.path,
+                                strerror (op_errno));
+                        goto unlock;
+                }
+
+                /* Track if rmdir succeeded on atleast one subvol*/
+                local->fop_succeeded = 1;
                 dht_iatt_merge (this, &local->preparent, preparent, prev->this);
                 dht_iatt_merge (this, &local->postparent, postparent,
                                 prev->this);
-	}
+        }
 unlock:
-	UNLOCK (&frame->lock);
+        UNLOCK (&frame->lock);
 
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt)) {
-		if (local->need_selfheal) {
+        this_call_cnt = dht_frame_return (frame);
+
+        /* if local->hashed_subvol, we are yet to wind to hashed_subvol. */
+        if (local->hashed_subvol && (this_call_cnt == 1)) {
+                done = 1;
+        } else if (!local->hashed_subvol && !this_call_cnt) {
+                done = 1;
+        }
+
+
+        if (done) {
+                if (local->need_selfheal && local->fop_succeeded) {
                         local->layout =
                                 dht_layout_get (this, local->loc.inode);
 
-			/* TODO: neater interface needed below */
-			local->stbuf.ia_type = local->loc.inode->ia_type;
+                        /* TODO: neater interface needed below */
+                        local->stbuf.ia_type = local->loc.inode->ia_type;
+
+                        uuid_copy (local->gfid, local->loc.inode->gfid);
+                        dht_selfheal_restore (frame, dht_rmdir_selfheal_cbk,
+                                              &local->loc, local->layout);
+                } else if (this_call_cnt) {
+                        /* If non-hashed subvol's have responded, proceed */
+
+                        local->need_selfheal = 0;
+                        STACK_WIND (frame, dht_rmdir_hashed_subvol_cbk,
+                                    local->hashed_subvol,
+                                    local->hashed_subvol->fops->rmdir,
+                                    &local->loc, local->flags, NULL);
+                } else if (!this_call_cnt) {
+                        /* All subvol's have responded, proceed */
 
-			dht_selfheal_restore (frame, dht_rmdir_selfheal_cbk,
-					      &local->loc, local->layout);
-		} else {
                         if (local->loc.parent) {
-                                local->preparent.ia_ino =
-                                        local->loc.parent->ino;
-                                local->postparent.ia_ino =
-                                        local->loc.parent->ino;
 
-                                WIPE (&local->preparent);
-                                WIPE (&local->postparent);
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->preparent,
+                                                           0);
+
+                                dht_inode_ctx_time_update (local->loc.parent,
+                                                           this,
+                                                           &local->postparent,
+                                                           1);
+
                         }
 
-			DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
-					  local->op_errno, &local->preparent,
-                                          &local->postparent);
-		}
-	}
+                        DHT_STACK_UNWIND (rmdir, frame, local->op_ret,
+                                          local->op_errno, &local->preparent,
+                                          &local->postparent, NULL);
+                }
+        }
 
         return 0;
 }
@@ -3873,40 +4472,64 @@ unlock:
 int
 dht_rmdir_do (call_frame_t *frame, xlator_t *this)
 {
-	dht_local_t  *local = NULL;
-	dht_conf_t   *conf = NULL;
-	int           i = 0;
+        dht_local_t  *local = NULL;
+        dht_conf_t   *conf = NULL;
+        int           i = 0;
+        xlator_t     *hashed_subvol = NULL;
 
         VALIDATE_OR_GOTO (this->private, err);
 
-	conf = this->private;
-	local = frame->local;
+        conf = this->private;
+        local = frame->local;
 
-	if (local->op_ret == -1)
-		goto err;
+        if (local->op_ret == -1)
+                goto err;
 
-	local->call_cnt = conf->subvolume_cnt;
+        local->call_cnt = conf->subvolume_cnt;
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		STACK_WIND (frame, dht_rmdir_cbk,
-			    conf->subvolumes[i],
-			    conf->subvolumes[i]->fops->rmdir,
-			    &local->loc, local->flags);
-	}
+        /* first remove from non-hashed_subvol */
+        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
 
-	return 0;
+        if (!hashed_subvol) {
+                gf_log (this->name, GF_LOG_WARNING, "failed to get hashed "
+                        "subvol for %s",local->loc.path);
+        } else {
+                local->hashed_subvol = hashed_subvol;
+        }
+
+        /* When DHT has only 1 child */
+        if (conf->subvolume_cnt == 1) {
+                STACK_WIND (frame, dht_rmdir_hashed_subvol_cbk,
+                            conf->subvolumes[0],
+                            conf->subvolumes[0]->fops->rmdir,
+                            &local->loc, local->flags, NULL);
+                return 0;
+        }
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (hashed_subvol &&
+                    (hashed_subvol == conf->subvolumes[i]))
+                        continue;
+
+                STACK_WIND (frame, dht_rmdir_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->rmdir,
+                            &local->loc, local->flags, NULL);
+        }
+
+        return 0;
 
 err:
-	DHT_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
-                          &local->preparent, &local->postparent);
-	return 0;
+        DHT_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
+                          &local->preparent, &local->postparent, NULL);
+        return 0;
 }
 
 
 int
 dht_rmdir_linkfile_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                int op_ret, int op_errno, struct iatt *preparent,
-                               struct iatt *postparent)
+                               struct iatt *postparent, dict_t *xdata)
 {
         dht_local_t    *local = NULL;
         call_frame_t   *prev = NULL;
@@ -3954,6 +4577,7 @@ dht_rmdir_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         call_frame_t   *main_frame = NULL;
         dht_local_t    *main_local = NULL;
         int             this_call_cnt = 0;
+        dht_conf_t     *conf = this->private;
 
         local = frame->local;
         prev  = cookie;
@@ -3965,7 +4589,7 @@ dht_rmdir_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (op_ret != 0)
                 goto err;
 
-        if (check_is_linkfile (inode, stbuf, xattr) == 0) {
+        if (!check_is_linkfile (inode, stbuf, xattr, conf->link_xattr_name)) {
                 main_local->op_ret  = -1;
                 main_local->op_errno = ENOTEMPTY;
 
@@ -3976,7 +4600,74 @@ dht_rmdir_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         STACK_WIND (frame, dht_rmdir_linkfile_unlink_cbk,
-                    src, src->fops->unlink, &local->loc);
+                    src, src->fops->unlink, &local->loc, 0, NULL);
+        return 0;
+err:
+
+        this_call_cnt = dht_frame_return (main_frame);
+        if (is_last_call (this_call_cnt))
+                dht_rmdir_do (main_frame, this);
+
+        DHT_STACK_DESTROY (frame);
+        return 0;
+}
+
+
+int
+dht_rmdir_cached_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                             int op_ret, int op_errno, inode_t *inode,
+                             struct iatt *stbuf, dict_t *xattr,
+                             struct iatt *parent)
+{
+        dht_local_t    *local         = NULL;
+        xlator_t       *src           = NULL;
+        call_frame_t   *main_frame    = NULL;
+        dht_local_t    *main_local    = NULL;
+        int             this_call_cnt = 0;
+        dht_conf_t     *conf          = this->private;
+        dict_t         *xattrs        = NULL;
+        int             ret           = 0;
+
+        local = frame->local;
+        src   = local->hashed_subvol;
+
+        main_frame = local->main_frame;
+        main_local = main_frame->local;
+
+        if (op_ret == 0) {
+                main_local->op_ret  = -1;
+                main_local->op_errno = ENOTEMPTY;
+
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s found on cached subvol %s",
+                        local->loc.path, src->name);
+                goto err;
+        } else if (op_errno != ENOENT) {
+                main_local->op_ret  = -1;
+                main_local->op_errno = op_errno;
+                goto err;
+        }
+
+        xattrs = dict_new ();
+        if (!xattrs) {
+                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
+                goto err;
+        }
+
+        ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to set linkto key"
+                        " in dict");
+                if (xattrs)
+                        dict_unref (xattrs);
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_rmdir_lookup_cbk,
+                    src, src->fops->lookup, &local->loc, xattrs);
+        if (xattrs)
+                dict_unref (xattrs);
+
         return 0;
 err:
 
@@ -3993,12 +4684,15 @@ int
 dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
                            gf_dirent_t *entries, xlator_t *src)
 {
-        int                 ret = 0;
-        int                 build_ret = 0;
-        gf_dirent_t        *trav = NULL;
+        int                 ret          = 0;
+        int                 build_ret    = 0;
+        gf_dirent_t        *trav         = NULL;
         call_frame_t       *lookup_frame = NULL;
         dht_local_t        *lookup_local = NULL;
-        dht_local_t        *local = NULL;
+        dht_local_t        *local        = NULL;
+        dict_t             *xattrs       = NULL;
+        dht_conf_t         *conf         = this->private;
+        xlator_t           *subvol       = NULL;
 
         local = frame->local;
 
@@ -4007,7 +4701,8 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
                         continue;
                 if (strcmp (trav->d_name, "..") == 0)
                         continue;
-                if (check_is_linkfile (NULL, (&trav->d_stat), NULL) == 1) {
+                if (check_is_linkfile (NULL, (&trav->d_stat), trav->dict,
+                                              conf->link_xattr_name)) {
                         ret++;
                         continue;
                 }
@@ -4019,6 +4714,21 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
                 return 0;
         }
 
+        xattrs = dict_new ();
+        if (!xattrs) {
+                gf_log (this->name, GF_LOG_ERROR, "dict_new failed");
+                return -1;
+        }
+
+        ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to set linkto key"
+                        " in dict");
+                if (xattrs)
+                        dict_unref (xattrs);
+                return -1;
+        }
+
         list_for_each_entry (trav, &entries->list, list) {
                 if (strcmp (trav->d_name, ".") == 0)
                         continue;
@@ -4030,29 +4740,27 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
 
                 lookup_frame = copy_frame (frame);
                 if (!lookup_frame) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of Memory");
                         /* out of memory, let the rmdir fail
                            (as non-empty, unfortunately) */
                         goto err;
                 }
 
-                lookup_local = GF_CALLOC (sizeof (*local), 1,
-                                          gf_dht_mt_dht_local_t);
+                lookup_local = mem_get0 (this->local_pool);
                 if (!lookup_local) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of Memory");
                         goto err;
                 }
 
                 lookup_frame->local = lookup_local;
                 lookup_local->main_frame = frame;
+                lookup_local->hashed_subvol = src;
 
                 build_ret = dht_build_child_loc (this, &lookup_local->loc,
                                                  &local->loc, trav->d_name);
                 if (build_ret != 0)
                         goto err;
 
+                uuid_copy (lookup_local->loc.gfid, trav->d_stat.ia_gfid);
+
                 gf_log (this->name, GF_LOG_TRACE,
                         "looking up %s on %s",
                         lookup_local->loc.path, src->name);
@@ -4063,14 +4771,31 @@ dht_rmdir_is_subvol_empty (call_frame_t *frame, xlator_t *this,
                 }
                 UNLOCK (&frame->lock);
 
-                STACK_WIND (lookup_frame, dht_rmdir_lookup_cbk,
-                            src, src->fops->lookup,
-                            &lookup_local->loc, NULL);
+                subvol = dht_linkfile_subvol (this, NULL, &trav->d_stat,
+                                              trav->dict);
+                if (!subvol) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "linkfile not having link subvolume. path=%s",
+                                lookup_local->loc.path);
+                        STACK_WIND (lookup_frame, dht_rmdir_lookup_cbk,
+                                    src, src->fops->lookup,
+                                    &lookup_local->loc, xattrs);
+                } else {
+                        STACK_WIND (lookup_frame, dht_rmdir_cached_lookup_cbk,
+                                    subvol, subvol->fops->lookup,
+                                    &lookup_local->loc, xattrs);
+                }
                 ret++;
         }
 
+        if (xattrs)
+                dict_unref (xattrs);
+
         return ret;
 err:
+        if (xattrs)
+                dict_unref (xattrs);
+
         DHT_STACK_DESTROY (lookup_frame);
         return 0;
 }
@@ -4078,19 +4803,20 @@ err:
 
 int
 dht_rmdir_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int op_ret, int op_errno, gf_dirent_t *entries)
+                        int op_ret, int op_errno, gf_dirent_t *entries,
+                        dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = -1;
-	call_frame_t *prev = NULL;
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = -1;
+        call_frame_t *prev = NULL;
         xlator_t     *src = NULL;
         int           ret = 0;
 
-	local = frame->local;
-	prev  = cookie;
+        local = frame->local;
+        prev  = cookie;
         src   = prev->this;
 
-	if (op_ret > 2) {
+        if (op_ret > 2) {
                 ret = dht_rmdir_is_subvol_empty (frame, this, entries, src);
 
                 switch (ret) {
@@ -4108,731 +4834,507 @@ dht_rmdir_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                 prev->this->name, local->loc.path, ret);
                         break;
                 }
-	}
+        }
 
-	this_call_cnt = dht_frame_return (frame);
+        this_call_cnt = dht_frame_return (frame);
 
-	if (is_last_call (this_call_cnt)) {
-		dht_rmdir_do (frame, this);
-	}
+        if (is_last_call (this_call_cnt)) {
+                dht_rmdir_do (frame, this);
+        }
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_rmdir_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int op_ret, int op_errno, fd_t *fd)
-{
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = -1;
-	call_frame_t *prev = NULL;
-
-
-	local = frame->local;
-	prev  = cookie;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"opendir on %s for %s failed (%s)",
-			prev->this->name, local->loc.path,
-			strerror (op_errno));
-		goto err;
-	}
-
-	STACK_WIND (frame, dht_rmdir_readdirp_cbk,
-		    prev->this, prev->this->fops->readdirp,
-		    local->fd, 4096, 0);
-
-	return 0;
-
-err:
-	this_call_cnt = dht_frame_return (frame);
-
-	if (is_last_call (this_call_cnt)) {
-		dht_rmdir_do (frame, this);
-	}
-
-	return 0;
-}
-
-
-int
-dht_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
-{
-	dht_local_t  *local  = NULL;
-	dht_conf_t   *conf = NULL;
-        int           op_errno = -1;
-	int           i = -1;
-	int           ret = -1;
-
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-        VALIDATE_OR_GOTO (this->private, err);
-
-	conf = this->private;
-
-	local = dht_local_init (frame);
-	if (!local) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		op_errno = ENOMEM;
-		goto err;
-	}
-
-	local->call_cnt = conf->subvolume_cnt;
-	local->op_ret   = 0;
-
-	ret = loc_copy (&local->loc, loc);
-	if (ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		op_errno = ENOMEM;
-		goto err;
-	}
-
-        local->flags = flags;
-
-	local->fd = fd_create (local->loc.inode, frame->root->pid);
-	if (!local->fd) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		op_errno = ENOMEM;
-		goto err;
-	}
-
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		STACK_WIND (frame, dht_rmdir_opendir_cbk,
-			    conf->subvolumes[i],
-			    conf->subvolumes[i]->fops->opendir,
-			    loc, local->fd);
-	}
-
-	return 0;
-
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (rmdir, frame, -1, op_errno,
-                          NULL, NULL);
-
-	return 0;
-}
-
-
-int
-dht_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		 int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-	DHT_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict);
-	return 0;
-}
-
-
-int
-dht_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-	     gf_xattrop_flags_t flags, dict_t *dict)
+                       int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
-	dht_local_t  *local = NULL;
-
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	subvol = dht_subvol_get_cached (this, loc->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
-
-	local->inode = inode_ref (loc->inode);
-	local->call_cnt = 1;
-
-	STACK_WIND (frame,
-		    dht_xattrop_cbk,
-		    subvol, subvol->fops->xattrop,
-		    loc, flags, dict);
-
-	return 0;
-
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL);
+        dht_local_t  *local         = NULL;
+        int           this_call_cnt = -1;
+        call_frame_t *prev          = NULL;
+        dict_t       *dict          = NULL;
+        int           ret           = 0;
+        dht_conf_t   *conf          = this->private;
+        int           i             = 0;
 
-	return 0;
-}
+        local = frame->local;
+        prev  = cookie;
 
+        this_call_cnt = dht_frame_return (frame);
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "opendir on %s for %s failed (%s)",
+                        prev->this->name, local->loc.path,
+                        strerror (op_errno));
+                if (op_errno != ENOENT) {
+                        local->op_ret = -1;
+                        local->op_errno = op_errno;
+                }
+                goto err;
+        }
 
-int
-dht_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-	DHT_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict);
-	return 0;
-}
+        if (!is_last_call (this_call_cnt))
+                return 0;
 
+        if (local->op_ret == -1)
+                goto err;
 
-int
-dht_fxattrop (call_frame_t *frame, xlator_t *this,
-	      fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict)
-{
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
+        dict = dict_new ();
+        if (!dict) {
+                local->op_ret = -1;
+                local->op_errno = ENOMEM;
+                goto err;
+        }
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+        ret = dict_set_uint32 (dict, conf->link_xattr_name, 256);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to set '%s' key",
+                        local->loc.path, conf->link_xattr_name);
 
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+        local->call_cnt = conf->subvolume_cnt;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                STACK_WIND (frame, dht_rmdir_readdirp_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->readdirp,
+                            local->fd, 4096, 0, dict);
+        }
 
-	STACK_WIND (frame,
-		    dht_fxattrop_cbk,
-		    subvol, subvol->fops->fxattrop,
-		    fd, flags, dict);
+        if (dict)
+                dict_unref (dict);
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL);
+        if (is_last_call (this_call_cnt)) {
+                dht_rmdir_do (frame, this);
+        }
 
-	return 0;
+        return 0;
 }
 
 
 int
-dht_inodelk_cbk (call_frame_t *frame, void *cookie,
-		 xlator_t *this, int32_t op_ret, int32_t op_errno)
-
-{
-	DHT_STACK_UNWIND (inodelk, frame, op_ret, op_errno);
-	return 0;
-}
-
-
-int32_t
-dht_inodelk (call_frame_t *frame, xlator_t *this,
-	     const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *lock)
+dht_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+           dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
+        dht_local_t  *local  = NULL;
+        dht_conf_t   *conf = NULL;
         int           op_errno = -1;
-	dht_local_t  *local = NULL;
-
+        int           i = -1;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
         VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (this->private, err);
 
-	subvol = dht_subvol_get_cached (this, loc->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
-
-	local->inode = inode_ref (loc->inode);
-	local->call_cnt = 1;
-
-	STACK_WIND (frame,
-		    dht_inodelk_cbk,
-		    subvol, subvol->fops->inodelk,
-		    volume, loc, cmd, lock);
-
-	return 0;
-
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (inodelk, frame, -1, op_errno);
-
-	return 0;
-}
-
-
-int
-dht_finodelk_cbk (call_frame_t *frame, void *cookie,
-		  xlator_t *this, int32_t op_ret, int32_t op_errno)
-
-{
-	DHT_STACK_UNWIND (finodelk, frame, op_ret, op_errno);
-	return 0;
-}
+        conf = this->private;
 
+        local = dht_local_init (frame, loc, NULL, GF_FOP_RMDIR);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-int
-dht_finodelk (call_frame_t *frame, xlator_t *this,
-	      const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *lock)
-{
-	xlator_t     *subvol = NULL;
-        int           op_errno = -1;
+        local->call_cnt = conf->subvolume_cnt;
+        local->op_ret   = 0;
+        local->fop_succeeded = 0;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+        local->flags = flags;
 
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+        local->fd = fd_create (local->loc.inode, frame->root->pid);
+        if (!local->fd) {
 
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	STACK_WIND (frame,
-		    dht_finodelk_cbk,
-		    subvol, subvol->fops->finodelk,
-		    volume, fd, cmd, lock);
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                STACK_WIND (frame, dht_rmdir_opendir_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->opendir,
+                            loc, local->fd, NULL);
+        }
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (finodelk, frame, -1, op_errno);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (rmdir, frame, -1, op_errno,
+                          NULL, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
-
 int
 dht_entrylk_cbk (call_frame_t *frame, void *cookie,
-		 xlator_t *this, int32_t op_ret, int32_t op_errno)
+                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-	DHT_STACK_UNWIND (entrylk, frame, op_ret, op_errno);
-	return 0;
+        DHT_STACK_UNWIND (entrylk, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 dht_entrylk (call_frame_t *frame, xlator_t *this,
-	     const char *volume, loc_t *loc, const char *basename,
-	     entrylk_cmd cmd, entrylk_type type)
+             const char *volume, loc_t *loc, const char *basename,
+             entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
+        xlator_t     *subvol = NULL;
         int           op_errno = -1;
-	dht_local_t  *local = NULL;
+        dht_local_t  *local = NULL;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (loc, err);
         VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
 
-	subvol = dht_subvol_get_cached (this, loc->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+        local = dht_local_init (frame, loc, NULL, GF_FOP_ENTRYLK);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	local->inode = inode_ref (loc->inode);
-	local->call_cnt = 1;
+        local->call_cnt = 1;
 
-	STACK_WIND (frame, dht_entrylk_cbk,
-		    subvol, subvol->fops->entrylk,
-		    volume, loc, basename, cmd, type);
+        STACK_WIND (frame, dht_entrylk_cbk,
+                    subvol, subvol->fops->entrylk,
+                    volume, loc, basename, cmd, type, xdata);
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (entrylk, frame, -1, op_errno);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (entrylk, frame, -1, op_errno, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_fentrylk_cbk (call_frame_t *frame, void *cookie,
-		  xlator_t *this, int32_t op_ret, int32_t op_errno)
+                  xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
 
 {
-	DHT_STACK_UNWIND (fentrylk, frame, op_ret, op_errno);
-	return 0;
+        DHT_STACK_UNWIND (fentrylk, frame, op_ret, op_errno, NULL);
+        return 0;
 }
 
 
 int
 dht_fentrylk (call_frame_t *frame, xlator_t *this,
-	      const char *volume, fd_t *fd, const char *basename,
-	      entrylk_cmd cmd, entrylk_type type)
+              const char *volume, fd_t *fd, const char *basename,
+              entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
-	xlator_t     *subvol = NULL;
+        xlator_t     *subvol = NULL;
         int           op_errno = -1;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
 
-	subvol = dht_subvol_get_cached (this, fd->inode);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+        subvol = dht_subvol_get_cached (this, fd->inode);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	STACK_WIND (frame, dht_fentrylk_cbk,
-		    subvol, subvol->fops->fentrylk,
-		    volume, fd, basename, cmd, type);
+        STACK_WIND (frame, dht_fentrylk_cbk,
+                    subvol, subvol->fops->fentrylk,
+                    volume, fd, basename, cmd, type, xdata);
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (fentrylk, frame, -1, op_errno);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fentrylk, frame, -1, op_errno, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
-dht_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, struct iatt *statpre,
-                 struct iatt *statpost)
+dht_forget (xlator_t *this, inode_t *inode)
 {
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = 0;
-	call_frame_t *prev = NULL;
-
+        uint64_t        ctx_int = 0;
+        dht_inode_ctx_t *ctx    = NULL;
+        dht_layout_t *layout = NULL;
 
-	local = frame->local;
-	prev = cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			gf_log (this->name, GF_LOG_DEBUG,
-				"subvolume %s returned -1 (%s)",
-				prev->this->name, strerror (op_errno));
-			goto unlock;
-		}
+        inode_ctx_del (inode, this, &ctx_int);
 
-		dht_iatt_merge (this, &local->prebuf, statpre, prev->this);
-                dht_iatt_merge (this, &local->stbuf, statpost, prev->this);
-		
-		if (local->inode) {
-			local->prebuf.ia_ino = local->inode->ino;
-                        local->stbuf.ia_ino = local->inode->ino;
-                }
+        if (!ctx_int)
+                return 0;
 
-		local->op_ret = 0;
-	}
-unlock:
-	UNLOCK (&frame->lock);
+        ctx = (dht_inode_ctx_t *) (long) ctx_int;
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt))
-		DHT_STACK_UNWIND (setattr, frame, local->op_ret, local->op_errno,
-				  &local->prebuf, &local->stbuf);
+        layout = ctx->layout;
+        ctx->layout = NULL;
+        dht_layout_unref (this, layout);
+        GF_FREE (ctx);
 
         return 0;
 }
 
 
 int
-dht_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             struct iatt *stbuf, int32_t valid)
+dht_notify (xlator_t *this, int event, void *data, ...)
 {
-	dht_layout_t *layout = NULL;
-	dht_local_t  *local  = NULL;
-        int           op_errno = -1;
-	int           i = -1;
+        xlator_t                *subvol = NULL;
+        int                      cnt    = -1;
+        int                      i      = -1;
+        dht_conf_t              *conf   = NULL;
+        int                      ret    = -1;
+        int                      propagate = 0;
+
+        int                      had_heard_from_all = 0;
+        int                      have_heard_from_all = 0;
+        struct timeval           time = {0,};
+        gf_defrag_info_t        *defrag = NULL;
+        dict_t                  *dict   = NULL;
+        gf_defrag_type           cmd    = 0;
+        dict_t                  *output = NULL;
+        va_list                  ap;
 
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (loc, err);
-        VALIDATE_OR_GOTO (loc->inode, err);
-        VALIDATE_OR_GOTO (loc->path, err);
-
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_DEBUG,
-			"memory allocation failed :(");
-		goto err;
-	}
-
-	local->layout = layout = dht_layout_get (this, loc->inode);
-	if (!layout) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no layout for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	if (!layout_is_sane (layout)) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"layout is not sane for path=%s", loc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	local->inode = inode_ref (loc->inode);
-	local->call_cnt = layout->cnt;
-
-	for (i = 0; i < layout->cnt; i++) {
-		STACK_WIND (frame, dht_setattr_cbk,
-			    layout->list[i].xlator,
-			    layout->list[i].xlator->fops->setattr,
-			    loc, stbuf, valid);
-	}
-
-	return 0;
-
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL);
-
-	return 0;
-}
+        conf = this->private;
+        if (!conf)
+                return ret;
 
+        /* had all subvolumes reported status once till now? */
+        had_heard_from_all = 1;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (!conf->last_event[i]) {
+                        had_heard_from_all = 0;
+                }
+        }
 
-int
-dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
-              int32_t valid)
-{
-	dht_layout_t *layout = NULL;
-	dht_local_t  *local  = NULL;
-        int           op_errno = -1;
-	int           i = -1;
+        switch (event) {
+        case GF_EVENT_CHILD_UP:
+                subvol = data;
 
+                conf->gen++;
 
-        VALIDATE_OR_GOTO (frame, err);
-        VALIDATE_OR_GOTO (this, err);
-        VALIDATE_OR_GOTO (fd, err);
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        if (subvol == conf->subvolumes[i]) {
+                                cnt = i;
+                                break;
+                        }
+                }
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
+                if (cnt == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "got GF_EVENT_CHILD_UP bad subvolume %s",
+                                subvol->name);
+                        break;
+                }
 
-	local->layout = layout = dht_layout_get (this, fd->inode);
-	if (!layout) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no layout for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+                gettimeofday (&time, NULL);
+                LOCK (&conf->subvolume_lock);
+                {
+                        conf->subvolume_status[cnt] = 1;
+                        conf->last_event[cnt] = event;
+                        conf->subvol_up_time[cnt] = time.tv_sec;
+                }
+                UNLOCK (&conf->subvolume_lock);
 
-	if (!layout_is_sane (layout)) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"layout is not sane for fd=%p", fd);
-		op_errno = EINVAL;
-		goto err;
-	}
+                /* one of the node came back up, do a stat update */
+                dht_get_du_info_for_subvol (this, cnt);
 
-	local->inode = inode_ref (fd->inode);
-	local->call_cnt = layout->cnt;
+                break;
 
-	for (i = 0; i < layout->cnt; i++) {
-		STACK_WIND (frame, dht_setattr_cbk,
-			    layout->list[i].xlator,
-			    layout->list[i].xlator->fops->fsetattr,
-			    fd, stbuf, valid);
-	}
+        case GF_EVENT_CHILD_MODIFIED:
+                subvol = data;
 
-	return 0;
+                conf->gen++;
+                propagate = 1;
 
-err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL);
+                break;
 
-	return 0;
-}
+        case GF_EVENT_CHILD_DOWN:
+                subvol = data;
 
+                if (conf->assert_no_child_down) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Received CHILD_DOWN. Exiting");
+                        if (conf->defrag) {
+                                gf_defrag_stop (conf->defrag,
+                                                GF_DEFRAG_STATUS_FAILED, NULL);
+                        } else {
+                                kill (getpid(), SIGTERM);
+                        }
+                }
 
-int
-dht_forget (xlator_t *this, inode_t *inode)
-{
-	uint64_t      tmp_layout = 0;
-	dht_layout_t *layout = NULL;
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        if (subvol == conf->subvolumes[i]) {
+                                cnt = i;
+                                break;
+                        }
+                }
 
-	inode_ctx_get (inode, this, &tmp_layout);
+                if (cnt == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "got GF_EVENT_CHILD_DOWN bad subvolume %s",
+                                subvol->name);
+                        break;
+                }
 
-	if (!tmp_layout)
-		return 0;
+                LOCK (&conf->subvolume_lock);
+                {
+                        conf->subvolume_status[cnt] = 0;
+                        conf->last_event[cnt] = event;
+                        conf->subvol_up_time[cnt] = 0;
+                }
+                UNLOCK (&conf->subvolume_lock);
 
-	layout = (dht_layout_t *)(long)tmp_layout;
-        dht_layout_unref (this, layout);
+                break;
 
-	return 0;
-}
+        case GF_EVENT_CHILD_CONNECTING:
+                subvol = data;
 
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        if (subvol == conf->subvolumes[i]) {
+                                cnt = i;
+                                break;
+                        }
+                }
 
+                if (cnt == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "got GF_EVENT_CHILD_CONNECTING bad subvolume %s",
+                                subvol->name);
+                        break;
+                }
 
-int
-dht_init_subvolumes (xlator_t *this, dht_conf_t *conf)
-{
-        xlator_list_t *subvols = NULL;
-        int            cnt = 0;
+                LOCK (&conf->subvolume_lock);
+                {
+                        conf->last_event[cnt] = event;
+                }
+                UNLOCK (&conf->subvolume_lock);
 
-        if (!conf)
-                return -1;
+                break;
+        case GF_EVENT_VOLUME_DEFRAG:
+        {
+                if (!conf->defrag) {
+                        return ret;
+                }
+                defrag = conf->defrag;
 
-        for (subvols = this->children; subvols; subvols = subvols->next)
-                cnt++;
+                dict = data;
+                va_start (ap, data);
+                output = va_arg (ap, dict_t*);
 
-        conf->subvolumes = GF_CALLOC (cnt, sizeof (xlator_t *),
-                                      gf_dht_mt_xlator_t);
-        if (!conf->subvolumes) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                return -1;
+                ret = dict_get_int32 (dict, "rebalance-command",
+                                      (int32_t*)&cmd);
+                if (ret)
+                        return ret;
+                LOCK (&defrag->lock);
+                {
+                        if (defrag->is_exiting)
+                                goto unlock;
+                        if (cmd == GF_DEFRAG_CMD_STATUS)
+                                gf_defrag_status_get (defrag, output);
+                        else if (cmd == GF_DEFRAG_CMD_STOP)
+                                gf_defrag_stop (defrag,
+                                                GF_DEFRAG_STATUS_STOPPED, output);
+                }
+unlock:
+                UNLOCK (&defrag->lock);
+                return 0;
+                break;
         }
-        conf->subvolume_cnt = cnt;
-
-        cnt = 0;
-        for (subvols = this->children; subvols; subvols = subvols->next)
-                conf->subvolumes[cnt++] = subvols->xlator;
 
-	conf->subvolume_status = GF_CALLOC (cnt, sizeof (char),
-                                            gf_dht_mt_char);
-	if (!conf->subvolume_status) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		return -1;
-	}
-
-        return 0;
-}
+        default:
+                propagate = 1;
+                break;
+        }
 
 
-int
-dht_notify (xlator_t *this, int event, void *data, ...)
-{
-	xlator_t   *subvol = NULL;
-	int         cnt    = -1;
-	int         i      = -1;
-	dht_conf_t *conf   = NULL;
-	int         ret    = -1;
-
-        /* Notify is on by default for all events except CHILD_UP. Do not
-         * notify parent for CHILD_UP till all distribute children have come up.
-         */
-        int         notify = 1;
-
-	conf = this->private;
-        if (!conf)
-                return ret;
+        /* have all subvolumes reported status once by now? */
+        have_heard_from_all = 1;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (!conf->last_event[i])
+                        have_heard_from_all = 0;
+        }
 
-	switch (event) {
-	case GF_EVENT_CHILD_UP:
-		subvol = data;
+        /* if all subvols have reported status, no need to hide anything
+           or wait for anything else. Just propagate blindly */
+        if (have_heard_from_all) {
+                propagate = 1;
 
-		conf->gen++;
+        }
 
-		for (i = 0; i < conf->subvolume_cnt; i++) {
-			if (subvol == conf->subvolumes[i]) {
-				cnt = i;
-				break;
-			}
-		}
 
-		if (cnt == -1) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"got GF_EVENT_CHILD_UP bad subvolume %s",
-				subvol->name);
-			break;
-		}
+        if (!had_heard_from_all && have_heard_from_all) {
+                /* This is the first event which completes aggregation
+                   of events from all subvolumes. If at least one subvol
+                   had come up, propagate CHILD_UP, but only this time
+                */
+                event = GF_EVENT_CHILD_DOWN;
 
-		LOCK (&conf->subvolume_lock);
-		{
-                        if (!conf->subvolume_status[cnt]) {
-                                conf->subvolume_status[cnt] = 1;
-                                ++conf->upsubvols;
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        if (conf->last_event[i] == GF_EVENT_CHILD_UP) {
+                                event = GF_EVENT_CHILD_UP;
+                                break;
                         }
 
-                        if (conf->upsubvols != conf->subvolume_cnt)
-                                notify = 0;
-		}
-		UNLOCK (&conf->subvolume_lock);
-
-                /* one of the node came back up, do a stat update */
-                dht_get_du_info_for_subvol (this, cnt);
-
-		break;
+                        if (conf->last_event[i] == GF_EVENT_CHILD_CONNECTING) {
+                                event = GF_EVENT_CHILD_CONNECTING;
+                                /* continue to check other events for CHILD_UP */
+                        }
+                }
 
-	case GF_EVENT_CHILD_DOWN:
-		subvol = data;
+                /* rebalance is started with assert_no_child_down. So we do
+                 * not need to handle CHILD_DOWN event here.
+                 */
+                if (conf->defrag) {
+                        ret = gf_thread_create (&conf->defrag->th, NULL,
+						gf_defrag_start, this);
+                        if (ret) {
+                                conf->defrag = NULL;
+                                GF_FREE (conf->defrag);
+                                kill (getpid(), SIGTERM);
+                        }
+                }
+        }
 
-		for (i = 0; i < conf->subvolume_cnt; i++) {
-			if (subvol == conf->subvolumes[i]) {
-				cnt = i;
-				break;
-			}
-		}
+        ret = 0;
+        if (propagate)
+                ret = default_notify (this, event, data);
 
-		if (cnt == -1) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"got GF_EVENT_CHILD_DOWN bad subvolume %s",
-				subvol->name);
-			break;
-		}
+        return ret;
+}
 
-		LOCK (&conf->subvolume_lock);
-		{
-                        if (conf->subvolume_status[cnt]) {
-                                conf->subvolume_status[cnt] = 0;
-                                --conf->upsubvols;
-                        }
-		}
-		UNLOCK (&conf->subvolume_lock);
+int
+dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this, dht_layout_t **layout)
+{
+        dht_inode_ctx_t         *ctx            = NULL;
+        int                      ret            = -1;
 
-		break;
-	}
+        ret = dht_inode_ctx_get (inode, this, &ctx);
 
-        if (notify)
-                ret = default_notify (this, event, data);
+        if (!ret && ctx) {
+                if (ctx->layout) {
+                        if (layout)
+                                *layout = ctx->layout;
+                        ret = 0;
+                } else {
+                        ret = -1;
+                }
+        }
 
-	return ret;
+        return ret;
 }
-
diff --git a/xlators/cluster/dht/src/dht-common.h b/xlators/cluster/dht/src/dht-common.h
index 5c486e4a5..2ece28a61 100644
--- a/xlators/cluster/dht/src/dht-common.h
+++ b/xlators/cluster/dht/src/dht-common.h
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -22,284 +13,775 @@
 #include "config.h"
 #endif
 
+#include <regex.h>
+#include <signal.h>
+
 #include "dht-mem-types.h"
+#include "libxlator.h"
+#include "syncop.h"
 
 #ifndef _DHT_H
 #define _DHT_H
 
-#define GF_XATTR_FIX_LAYOUT_KEY   "trusted.distribute.fix.layout"
+#define GF_XATTR_FIX_LAYOUT_KEY     "distribute.fix.layout"
 #define GF_DHT_LOOKUP_UNHASHED_ON   1
 #define GF_DHT_LOOKUP_UNHASHED_AUTO 2
+#define DHT_PATHINFO_HEADER         "DISTRIBUTE:"
 
 #include <fnmatch.h>
 
 typedef int (*dht_selfheal_dir_cbk_t) (call_frame_t *frame, void *cookie,
-				       xlator_t *this,
-				       int32_t op_ret, int32_t op_errno);
+                                       xlator_t     *this,
+                                       int32_t       op_ret, int32_t op_errno,
+                                       dict_t *xdata);
+typedef int (*dht_defrag_cbk_fn_t) (xlator_t        *this, call_frame_t *frame,
+                                    int              ret);
 
 
 struct dht_layout {
-        int               cnt;
-	int               preset;
-        int               gen;
-	int               type;
-        int               ref;   /* use with dht_conf_t->layout_lock */
-        int               search_unhashed;
+        int                spread_cnt;  /* layout spread count per directory,
+                                           is controlled by 'setxattr()' with
+                                           special key */
+        int                cnt;
+        int                preset;
+        int                gen;
+        int                type;
+        int                ref; /* use with dht_conf_t->layout_lock */
+        gf_boolean_t       search_unhashed;
         struct {
-		int       err;   /* 0 = normal
-				   -1 = dir exists and no xattr
-				   >0 = dir lookup failed with errno
-				 */
-                uint32_t  start;
-                uint32_t  stop;
-                xlator_t *xlator;
-        } list[0];
+                int        err;   /* 0 = normal
+                                     -1 = dir exists and no xattr
+                                     >0 = dir lookup failed with errno
+                                  */
+                uint32_t   start;
+                uint32_t   stop;
+                xlator_t  *xlator;
+        } list[];
+};
+typedef struct dht_layout  dht_layout_t;
+
+struct dht_stat_time {
+        uint32_t        atime;
+        uint32_t        atime_nsec;
+        uint32_t        ctime;
+        uint32_t        ctime_nsec;
+        uint32_t        mtime;
+        uint32_t        mtime_nsec;
+};
+
+typedef struct dht_stat_time dht_stat_time_t;
+
+struct dht_inode_ctx {
+        dht_layout_t    *layout;
+        dht_stat_time_t  time;
 };
-typedef struct dht_layout dht_layout_t;
+
+typedef struct dht_inode_ctx dht_inode_ctx_t;
 
 
 typedef enum {
-	DHT_HASH_TYPE_DM,
+        DHT_HASH_TYPE_DM,
+        DHT_HASH_TYPE_DM_USER,
 } dht_hashfn_type_t;
 
+/* rebalance related */
+struct dht_rebalance_ {
+        xlator_t            *from_subvol;
+        xlator_t            *target_node;
+        off_t                offset;
+        size_t               size;
+        int32_t              flags;
+        int                  count;
+        struct iobref       *iobref;
+        struct iovec        *vector;
+        struct iatt          stbuf;
+        dht_defrag_cbk_fn_t  target_op_fn;
+        dict_t              *xdata;
+};
 
 struct dht_local {
-	int                      call_cnt;
-	loc_t                    loc;
-	loc_t                    loc2;
-	int                      op_ret;
-	int                      op_errno;
-	int                      layout_mismatch;
+        int                      call_cnt;
+        loc_t                    loc;
+        loc_t                    loc2;
+        int                      op_ret;
+        int                      op_errno;
+        int                      layout_mismatch;
         /* Use stbuf as the postbuf, when we require both
          * pre and post attrs */
-	struct iatt              stbuf;
+        struct iatt              stbuf;
         struct iatt              prebuf;
         struct iatt              preoldparent;
         struct iatt              postoldparent;
         struct iatt              preparent;
         struct iatt              postparent;
-	struct statvfs           statvfs;
-	fd_t                    *fd;
-	inode_t                 *inode;
+        struct statvfs           statvfs;
+        fd_t                    *fd;
+        inode_t                 *inode;
         dict_t                  *params;
-	dict_t                  *xattr;
-	dict_t                  *xattr_req;
-	dht_layout_t            *layout;
-	size_t                   size;
-	ino_t                    ia_ino;
-	xlator_t                *src_hashed, *src_cached;
-	xlator_t                *dst_hashed, *dst_cached;
-	xlator_t                *cached_subvol;
-	xlator_t                *hashed_subvol;
-	char                     need_selfheal;
+        dict_t                  *xattr;
+        dict_t                  *xattr_req;
+        dht_layout_t            *layout;
+        size_t                   size;
+        ino_t                    ia_ino;
+        xlator_t                *src_hashed, *src_cached;
+        xlator_t                *dst_hashed, *dst_cached;
+        xlator_t                *cached_subvol;
+        xlator_t                *hashed_subvol;
+        char                     need_selfheal;
         int                      file_count;
         int                      dir_count;
         call_frame_t            *main_frame;
-	struct {
-		fop_mknod_cbk_t  linkfile_cbk;
-		struct iatt      stbuf;
-		loc_t            loc;
-		inode_t         *inode;
-		dict_t          *xattr;
-		xlator_t        *srcvol;
-	} linkfile;
-	struct {
-		uint32_t         hole_cnt;
-		uint32_t         overlaps_cnt;
-		uint32_t         missing;
-		uint32_t         down;
-		uint32_t         misc;
-		dht_selfheal_dir_cbk_t   dir_cbk;
-		dht_layout_t    *layout;
-	} selfheal;
+        int                      fop_succeeded;
+        struct {
+                fop_mknod_cbk_t  linkfile_cbk;
+                struct iatt      stbuf;
+                loc_t            loc;
+                inode_t         *inode;
+                dict_t          *xattr;
+                xlator_t        *srcvol;
+        } linkfile;
+        struct {
+                uint32_t         hole_cnt;
+                uint32_t         overlaps_cnt;
+                uint32_t         down;
+                uint32_t         misc;
+                dht_selfheal_dir_cbk_t   dir_cbk;
+                dht_layout_t    *layout;
+        } selfheal;
         uint32_t                 uid;
         uint32_t                 gid;
 
-	/* needed by nufa */
-	int32_t flags;
-	mode_t  mode;
-	dev_t   rdev;
+        /* needed by nufa */
+        int32_t flags;
+        mode_t  mode;
+        dev_t   rdev;
+        mode_t  umask;
 
         /* need for file-info */
-        char   *pathinfo;
+        char   *xattr_val;
         char   *key;
 
+        /* which xattr request? */
+        char xsel[256];
+        int32_t alloc_len;
+
         char   *newpath;
 
         /* gfid related */
         uuid_t  gfid;
+
+        /*Marker Related*/
+        struct marker_str    marker;
+
+        /* flag used to make sure we need to return estale in
+           {lookup,revalidate}_cbk */
+        char return_estale;
+        char need_lookup_everywhere;
+
+        glusterfs_fop_t      fop;
+
+        gf_boolean_t     linked;
+        xlator_t        *link_subvol;
+
+        struct dht_rebalance_ rebalance;
+        xlator_t        *first_up_subvol;
+
 };
 typedef struct dht_local dht_local_t;
 
 /* du - disk-usage */
 struct dht_du {
         double   avail_percent;
+	double   avail_inodes;
         uint64_t avail_space;
         uint32_t log;
 };
 typedef struct dht_du dht_du_t;
 
+enum gf_defrag_type {
+        GF_DEFRAG_CMD_START = 1,
+        GF_DEFRAG_CMD_STOP = 1 + 1,
+        GF_DEFRAG_CMD_STATUS = 1 + 2,
+        GF_DEFRAG_CMD_START_LAYOUT_FIX = 1 + 3,
+        GF_DEFRAG_CMD_START_FORCE = 1 + 4,
+};
+typedef enum gf_defrag_type gf_defrag_type;
+
+enum gf_defrag_status_t {
+        GF_DEFRAG_STATUS_NOT_STARTED,
+        GF_DEFRAG_STATUS_STARTED,
+        GF_DEFRAG_STATUS_STOPPED,
+        GF_DEFRAG_STATUS_COMPLETE,
+        GF_DEFRAG_STATUS_FAILED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_STARTED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_STOPPED,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_COMPLETE,
+        GF_DEFRAG_STATUS_LAYOUT_FIX_FAILED,
+};
+typedef enum gf_defrag_status_t gf_defrag_status_t;
+
+typedef struct gf_defrag_pattern_list gf_defrag_pattern_list_t;
+
+struct gf_defrag_pattern_list {
+        char                       path_pattern[256];
+        uint64_t                   size;
+        gf_defrag_pattern_list_t  *next;
+};
+
+struct gf_defrag_info_ {
+        uint64_t                     total_files;
+        uint64_t                     total_data;
+        uint64_t                     num_files_lookedup;
+        uint64_t                     total_failures;
+        uint64_t                     skipped;
+        gf_lock_t                    lock;
+        int                          cmd;
+        pthread_t                    th;
+        gf_defrag_status_t           defrag_status;
+        struct rpc_clnt             *rpc;
+        uint32_t                     connected;
+        uint32_t                     is_exiting;
+        pid_t                        pid;
+        inode_t                     *root_inode;
+        uuid_t                       node_uuid;
+        struct timeval               start_time;
+        gf_boolean_t                 stats;
+        gf_defrag_pattern_list_t    *defrag_pattern;
+};
+
+typedef struct gf_defrag_info_ gf_defrag_info_t;
+
 struct dht_conf {
-	gf_lock_t      subvolume_lock;
+        gf_lock_t      subvolume_lock;
         int            subvolume_cnt;
         xlator_t     **subvolumes;
-	char          *subvolume_status;
-	dht_layout_t **file_layouts;
-	dht_layout_t **dir_layouts;
-	dht_layout_t  *default_dir_layout;
-	gf_boolean_t   search_unhashed;
-	int            gen;
+        char          *subvolume_status;
+        int           *last_event;
+        dht_layout_t **file_layouts;
+        dht_layout_t **dir_layouts;
+        gf_boolean_t   search_unhashed;
+        int            gen;
         dht_du_t      *du_stats;
-        uint64_t       min_free_disk;
+        double         min_free_disk;
+        double         min_free_inodes;
         char           disk_unit;
         int32_t        refresh_interval;
         gf_boolean_t   unhashed_sticky_bit;
-	struct timeval last_stat_fetch;
+        struct timeval last_stat_fetch;
         gf_lock_t      layout_lock;
         void          *private;     /* Can be used by wrapper xlators over
                                        dht */
         gf_boolean_t   use_readdirp;
-        int             upsubvols;      /* Count the number of online children*/
+        char           vol_uuid[UUID_SIZE + 1];
+        gf_boolean_t   assert_no_child_down;
+        time_t        *subvol_up_time;
+
+        /* This is the count used as the distribute layout for a directory */
+        /* Will be a global flag to control the layout spread count */
+        uint32_t       dir_spread_cnt;
+
+        /* to keep track of nodes which are decomissioned */
+        xlator_t     **decommissioned_bricks;
+        int            decommission_in_progress;
+        int            decommission_subvols_cnt;
+
+        /* defrag related */
+        gf_defrag_info_t *defrag;
+
+        /* Request to filter directory entries in readdir request */
+
+        gf_boolean_t    readdir_optimize;
+
+        /* Support regex-based name reinterpretation. */
+        regex_t         rsync_regex;
+        gf_boolean_t    rsync_regex_valid;
+        regex_t         extra_regex;
+        gf_boolean_t    extra_regex_valid;
+
+        /* Support variable xattr names. */
+        char            *xattr_name;
+        char            *link_xattr_name;
+        char            *wild_xattr_name;
 };
 typedef struct dht_conf dht_conf_t;
 
 
 struct dht_disk_layout {
-	uint32_t           cnt;
-	uint32_t           type;
-	struct {
-		uint32_t   start;
-		uint32_t   stop;
-	} list[1];
+        uint32_t           cnt;
+        uint32_t           type;
+        struct {
+                uint32_t   start;
+                uint32_t   stop;
+        } list[1];
 };
 typedef struct dht_disk_layout dht_disk_layout_t;
 
-#define WIPE(statp) do { typeof(*statp) z = {0,}; if (statp) *statp = z; } while (0)
+typedef enum {
+        GF_DHT_MIGRATE_DATA,
+        GF_DHT_MIGRATE_DATA_EVEN_IF_LINK_EXISTS,
+        GF_DHT_MIGRATE_HARDLINK,
+        GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS
+} gf_dht_migrate_data_type_t;
 
 #define ENTRY_MISSING(op_ret, op_errno) (op_ret == -1 && op_errno == ENOENT)
 
-#define is_fs_root(loc) (strcmp (loc->path, "/") == 0)
-
-#define is_revalidate(loc) (inode_ctx_get (loc->inode, this, NULL) == 0)
+#define is_revalidate(loc) (dht_inode_ctx_layout_get (loc->inode, this, NULL) == 0)
 
 #define is_last_call(cnt) (cnt == 0)
 
-#define DHT_LINKFILE_MODE (S_ISVTX)
-#define check_is_linkfile(i,s,x) (                                      \
-                ((st_mode_from_ia (s->ia_prot, s->ia_type) & ~S_IFMT)   \
-                 == DHT_LINKFILE_MODE) &&                               \
-                (s->ia_size == 0))
+#define DHT_MIGRATION_IN_PROGRESS 1
+#define DHT_MIGRATION_COMPLETED   2
+
+#define check_is_linkfile(i,s,x,n) (IS_DHT_LINKFILE_MODE (s) && dict_get (x, n))
+
+#define IS_DHT_MIGRATION_PHASE2(buf)  (                                 \
+                IA_ISREG ((buf)->ia_type) &&                            \
+                ((st_mode_from_ia ((buf)->ia_prot, (buf)->ia_type) &    \
+                  ~S_IFMT) == DHT_LINKFILE_MODE))
+
+#define IS_DHT_MIGRATION_PHASE1(buf)  (                                 \
+                IA_ISREG ((buf)->ia_type) &&                            \
+                ((buf)->ia_prot.sticky == 1) &&                         \
+                ((buf)->ia_prot.sgid == 1))
+
+#define DHT_STRIP_PHASE1_FLAGS(buf)  do {                       \
+                if ((buf) && IS_DHT_MIGRATION_PHASE1(buf)) {    \
+                        (buf)->ia_prot.sticky = 0;              \
+                        (buf)->ia_prot.sgid = 0;                \
+                }                                               \
+        } while (0)
+
+#define dht_inode_missing(op_errno) (op_errno == ENOENT || op_errno == ESTALE)
 
 #define check_is_dir(i,s,x) (IA_ISDIR(s->ia_type))
 
 #define layout_is_sane(layout) ((layout) && (layout->cnt > 0))
 
 #define DHT_STACK_UNWIND(fop, frame, params ...) do {           \
-		dht_local_t *__local = NULL;                    \
-                xlator_t *__xl = NULL;                          \
+                dht_local_t *__local = NULL;                    \
+                xlator_t    *__xl    = NULL;                    \
                 if (frame) {                                    \
-                        __xl = frame->this;                     \
-                        __local = frame->local;                 \
+                        __xl         = frame->this;             \
+                        __local      = frame->local;            \
                         frame->local = NULL;                    \
                 }                                               \
-		STACK_UNWIND_STRICT (fop, frame, params);       \
-		dht_local_wipe (__xl, __local);                 \
-	} while (0)
-
-#define DHT_STACK_DESTROY(frame) do {		       \
-		dht_local_t *__local = NULL;           \
-                xlator_t *__xl = NULL;                 \
-                __xl = frame->this;                    \
-		__local = frame->local;                \
-		frame->local = NULL;		       \
-		STACK_DESTROY (frame->root);	       \
-		dht_local_wipe (__xl, __local);	       \
-	} while (0)
-
-dht_layout_t *dht_layout_new (xlator_t *this, int cnt);
-dht_layout_t *dht_layout_get (xlator_t *this, inode_t *inode);
-dht_layout_t *dht_layout_for_subvol (xlator_t *this, xlator_t *subvol);
-xlator_t *dht_layout_search (xlator_t *this, dht_layout_t *layout,
-			     const char *name);
-int dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout);
-int dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
-			  uint32_t *holes_p, uint32_t *overlaps_p,
-			  uint32_t *missing_p, uint32_t *down_p,
-			  uint32_t *misc_p);
-int dht_layout_dir_mismatch (xlator_t *this, dht_layout_t *layout,
-			     xlator_t *subvol, loc_t *loc, dict_t *xattr);
+                STACK_UNWIND_STRICT (fop, frame, params);       \
+                dht_local_wipe (__xl, __local);                 \
+        } while (0)
+
+#define DHT_STACK_DESTROY(frame) do {                   \
+                dht_local_t *__local = NULL;            \
+                xlator_t    *__xl    = NULL;            \
+                __xl                 = frame->this;     \
+                __local              = frame->local;    \
+                frame->local         = NULL;            \
+                STACK_DESTROY (frame->root);            \
+                dht_local_wipe (__xl, __local);         \
+        } while (0)
+
+#define DHT_UPDATE_TIME(ctx_sec, ctx_nsec, new_sec, new_nsec, inode, post) do {\
+                int32_t sec = 0;                                        \
+                sec = new_sec;                                          \
+                LOCK (&inode->lock);                                    \
+                {                                                       \
+                        new_sec = max(new_sec, ctx_sec);                \
+                        if (sec < new_sec)                              \
+                                new_nsec = ctx_nsec;                    \
+                        if (sec == new_sec)                             \
+                                new_nsec = max (new_nsec, ctx_nsec);    \
+                        if (post) {                                     \
+                                ctx_sec = new_sec;                      \
+                                ctx_nsec = new_nsec;                    \
+                        }                                               \
+                }                                                       \
+                UNLOCK (&inode->lock);                                  \
+        } while (0)
+
+#define is_greater_time(a, an, b, bn) (((a) < (b)) || (((a) == (b)) && ((an) < (bn))))
+dht_layout_t                            *dht_layout_new (xlator_t *this, int cnt);
+dht_layout_t                            *dht_layout_get (xlator_t *this, inode_t *inode);
+dht_layout_t                            *dht_layout_for_subvol (xlator_t *this, xlator_t *subvol);
+xlator_t *dht_layout_search (xlator_t   *this, dht_layout_t *layout,
+                             const char *name);
+int                                      dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout);
+int dht_layout_anomalies (xlator_t      *this, loc_t *loc, dht_layout_t *layout,
+                          uint32_t      *holes_p, uint32_t *overlaps_p,
+                          uint32_t      *missing_p, uint32_t *down_p,
+                          uint32_t      *misc_p, uint32_t *no_space_p);
+int dht_layout_dir_mismatch (xlator_t   *this, dht_layout_t *layout,
+                             xlator_t   *subvol, loc_t *loc, dict_t *xattr);
 
 xlator_t *dht_linkfile_subvol (xlator_t *this, inode_t *inode,
-			       struct iatt *buf, dict_t *xattr);
-int dht_linkfile_unlink (call_frame_t *frame, xlator_t *this,
-			 xlator_t *subvol, loc_t *loc);
+                               struct iatt *buf, dict_t *xattr);
+int dht_linkfile_unlink (call_frame_t      *frame, xlator_t *this,
+                         xlator_t          *subvol, loc_t *loc);
 
 int dht_layouts_init (xlator_t *this, dht_conf_t *conf);
 int dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
-		      int op_ret, int op_errno, dict_t *xattr);
+                      int       op_ret, int op_errno, dict_t *xattr);
 
 int dht_disk_layout_extract (xlator_t *this, dht_layout_t *layout,
-			     int pos, int32_t **disk_layout_p);
-int dht_disk_layout_merge (xlator_t *this, dht_layout_t *layout,
-			   int pos, void *disk_layout_raw);
+                             int       pos, int32_t **disk_layout_p);
+int dht_disk_layout_merge (xlator_t   *this, dht_layout_t *layout,
+                           int         pos, void *disk_layout_raw, int disk_layout_len);
 
 
 int dht_frame_return (call_frame_t *frame);
 
-int dht_itransform (xlator_t *this, xlator_t *subvol, uint64_t x, uint64_t *y);
+int                             dht_itransform (xlator_t *this, xlator_t *subvol, uint64_t x, uint64_t *y);
 int dht_deitransform (xlator_t *this, uint64_t y, xlator_t **subvol,
-		      uint64_t *x);
+                      uint64_t *x);
 
 void dht_local_wipe (xlator_t *this, dht_local_t *local);
-dht_local_t *dht_local_init (call_frame_t *frame);
-int dht_iatt_merge (xlator_t *this, struct iatt *to, struct iatt *from,
-		    xlator_t *subvol);
+dht_local_t *dht_local_init (call_frame_t    *frame, loc_t *loc, fd_t *fd,
+                             glusterfs_fop_t  fop);
+int dht_iatt_merge (xlator_t                 *this, struct iatt *to, struct iatt *from,
+                    xlator_t                 *subvol);
 
 xlator_t *dht_subvol_get_hashed (xlator_t *this, loc_t *loc);
 xlator_t *dht_subvol_get_cached (xlator_t *this, inode_t *inode);
 xlator_t *dht_subvol_next (xlator_t *this, xlator_t *prev);
-int dht_subvol_cnt (xlator_t *this, xlator_t *subvol);
+xlator_t *dht_subvol_next_available (xlator_t *this, xlator_t *prev);
+int       dht_subvol_cnt (xlator_t *this, xlator_t *subvol);
 
-int dht_hash_compute (int type, const char *name, uint32_t *hash_p);
+int dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p);
 
-int dht_linkfile_create (call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
-			 xlator_t *tovol, xlator_t *fromvol, loc_t *loc);
-int dht_lookup_directory (call_frame_t *frame, xlator_t *this, loc_t *loc);
-int dht_lookup_everywhere (call_frame_t *frame, xlator_t *this, loc_t *loc);
+int dht_linkfile_create (call_frame_t    *frame, fop_mknod_cbk_t linkfile_cbk,
+                         xlator_t        *this, xlator_t *tovol,
+                         xlator_t        *fromvol, loc_t *loc);
+int                                       dht_lookup_directory (call_frame_t *frame, xlator_t *this, loc_t *loc);
+int                                       dht_lookup_everywhere (call_frame_t *frame, xlator_t *this, loc_t *loc);
 int
-dht_selfheal_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
-			loc_t *loc, dht_layout_t *layout);
+dht_selfheal_directory (call_frame_t     *frame, dht_selfheal_dir_cbk_t cbk,
+                        loc_t            *loc, dht_layout_t *layout);
 int
 dht_selfheal_new_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
-			    dht_layout_t *layout);
+                            dht_layout_t *layout);
 int
-dht_selfheal_restore (call_frame_t *frame, dht_selfheal_dir_cbk_t cbk,
-		      loc_t *loc, dht_layout_t *layout);
+dht_selfheal_restore (call_frame_t       *frame, dht_selfheal_dir_cbk_t cbk,
+                      loc_t              *loc, dht_layout_t *layout);
 int
 dht_layout_sort_volname (dht_layout_t *layout);
 
-int dht_rename (call_frame_t *frame, xlator_t *this,
-		loc_t *oldloc, loc_t *newloc);
-
 int dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc);
 
-int dht_is_subvol_filled (xlator_t *this, xlator_t *subvol);
-xlator_t *dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol);
-int dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx);
+gf_boolean_t dht_is_subvol_filled (xlator_t *this, xlator_t *subvol);
+xlator_t *dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol,
+                                          dht_local_t *layout);
+int       dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx);
 
 int dht_layout_preset (xlator_t *this, xlator_t *subvol, inode_t *inode);
-int dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout);
-void dht_layout_unref (xlator_t *this, dht_layout_t *layout);
+int           dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout);;
+void          dht_layout_unref (xlator_t *this, dht_layout_t *layout);
 dht_layout_t *dht_layout_ref (xlator_t *this, dht_layout_t *layout);
-xlator_t *dht_first_up_subvol (xlator_t *this);
-xlator_t *dht_last_up_subvol (xlator_t *this);
-int dht_frame_su_do (call_frame_t *frame);
-int dht_frame_su_undo (call_frame_t *frame);
+xlator_t     *dht_first_up_subvol (xlator_t *this);
+xlator_t     *dht_last_up_subvol (xlator_t *this);
 
 int dht_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name);
 
-int dht_filter_loc_subvol_key (xlator_t *this, loc_t *loc, loc_t *new_loc,
+int dht_filter_loc_subvol_key (xlator_t  *this, loc_t *loc, loc_t *new_loc,
                                xlator_t **subvol);
 
+int                                     dht_rename_cleanup (call_frame_t *frame);
+int dht_rename_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t           op_ret, int32_t op_errno,
+                          inode_t          *inode, struct iatt *stbuf,
+                          struct iatt      *preparent, struct iatt *postparent,
+                          dict_t *xdata);
+
+int dht_fix_directory_layout (call_frame_t *frame,
+                              dht_selfheal_dir_cbk_t  dir_cbk,
+                              dht_layout_t           *layout);
+
+int dht_init_subvolumes (xlator_t *this, dht_conf_t *conf);
+
+/* migration/rebalance */
+int dht_start_rebalance_task (xlator_t *this, call_frame_t *frame);
+
+int dht_rebalance_in_progress_check (xlator_t *this, call_frame_t *frame);
+int dht_rebalance_complete_check (xlator_t *this, call_frame_t *frame);
+
+
+/* FOPS */
+int32_t dht_lookup (call_frame_t *frame,
+                    xlator_t *this,
+                    loc_t    *loc,
+                    dict_t   *xattr_req);
+
+int32_t dht_stat (call_frame_t *frame,
+                  xlator_t *this,
+                  loc_t    *loc, dict_t *xdata);
+
+int32_t dht_fstat (call_frame_t *frame,
+                   xlator_t *this,
+                   fd_t     *fd, dict_t *xdata);
+
+int32_t dht_truncate (call_frame_t *frame,
+                      xlator_t *this,
+                      loc_t    *loc,
+                      off_t     offset, dict_t *xdata);
+
+int32_t dht_ftruncate (call_frame_t *frame,
+                       xlator_t *this,
+                       fd_t     *fd,
+                       off_t     offset, dict_t *xdata);
+
+int32_t dht_access (call_frame_t *frame,
+                    xlator_t *this,
+                    loc_t    *loc,
+                    int32_t   mask, dict_t *xdata);
+
+int32_t dht_readlink (call_frame_t *frame,
+                      xlator_t *this,
+                      loc_t    *loc,
+                      size_t    size, dict_t *xdata);
+
+int32_t dht_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata);
+
+int32_t dht_mkdir (call_frame_t *frame, xlator_t *this,
+                   loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
+
+int32_t dht_unlink (call_frame_t *frame,
+                    xlator_t *this,
+                    loc_t    *loc, int xflag, dict_t *xdata);
+
+int32_t dht_rmdir (call_frame_t *frame, xlator_t *this,
+                   loc_t    *loc, int flags, dict_t *xdata);
+
+int32_t dht_symlink (call_frame_t   *frame, xlator_t *this,
+                     const char *linkpath, loc_t *loc, mode_t umask,
+                     dict_t *xdata);
+
+int32_t dht_rename (call_frame_t *frame,
+                    xlator_t *this,
+                    loc_t    *oldloc,
+                    loc_t    *newloc, dict_t *xdata);
+
+int32_t dht_link (call_frame_t *frame,
+                  xlator_t *this,
+                  loc_t    *oldloc,
+                  loc_t    *newloc, dict_t *xdata);
+
+int32_t dht_create (call_frame_t *frame, xlator_t *this,
+                    loc_t    *loc, int32_t flags, mode_t mode,
+                    mode_t umask, fd_t     *fd, dict_t *params);
+
+int32_t dht_open (call_frame_t *frame,
+                  xlator_t *this,
+                  loc_t    *loc,
+                  int32_t   flags, fd_t *fd, dict_t *xdata);
+
+int32_t dht_readv (call_frame_t *frame,
+                   xlator_t *this,
+                   fd_t     *fd,
+                   size_t    size,
+                   off_t     offset, uint32_t flags, dict_t *xdata);
+
+int32_t dht_writev (call_frame_t      *frame,
+                    xlator_t      *this,
+                    fd_t          *fd,
+                    struct iovec  *vector,
+                    int32_t        count,
+                    off_t          offset,
+                    uint32_t       flags,
+                    struct iobref *iobref, dict_t *xdata);
+
+int32_t dht_flush (call_frame_t *frame,
+                   xlator_t *this,
+                   fd_t     *fd, dict_t *xdata);
+
+int32_t dht_fsync (call_frame_t *frame,
+                   xlator_t *this,
+                   fd_t     *fd,
+                   int32_t   datasync, dict_t *xdata);
+
+int32_t dht_opendir (call_frame_t *frame,
+                     xlator_t *this,
+                     loc_t    *loc, fd_t *fd, dict_t *xdata);
+
+int32_t dht_fsyncdir (call_frame_t *frame,
+                      xlator_t *this,
+                      fd_t     *fd,
+                      int32_t   datasync, dict_t *xdata);
+
+int32_t dht_statfs (call_frame_t *frame,
+                    xlator_t *this,
+                    loc_t    *loc, dict_t *xdata);
+
+int32_t dht_setxattr (call_frame_t *frame,
+                      xlator_t *this,
+                      loc_t    *loc,
+                      dict_t   *dict,
+                      int32_t   flags, dict_t *xdata);
+
+int32_t dht_getxattr (call_frame_t   *frame,
+                      xlator_t   *this,
+                      loc_t      *loc,
+                      const char *name, dict_t *xdata);
+
+int32_t dht_fsetxattr (call_frame_t *frame,
+                       xlator_t *this,
+                       fd_t     *fd,
+                       dict_t   *dict,
+                       int32_t   flags, dict_t *xdata);
+
+int32_t dht_fgetxattr (call_frame_t   *frame,
+                       xlator_t   *this,
+                       fd_t       *fd,
+                       const char *name, dict_t *xdata);
+
+int32_t dht_removexattr (call_frame_t   *frame,
+                         xlator_t   *this,
+                         loc_t      *loc,
+                         const char *name, dict_t *xdata);
+int32_t dht_fremovexattr (call_frame_t   *frame,
+                          xlator_t   *this,
+                          fd_t      *fd,
+                          const char *name, dict_t *xdata);
+
+int32_t dht_lk (call_frame_t        *frame,
+                xlator_t        *this,
+                fd_t            *fd,
+                int32_t          cmd,
+                struct gf_flock *flock, dict_t *xdata);
+
+int32_t dht_inodelk (call_frame_t *frame, xlator_t *this,
+                     const char      *volume, loc_t *loc, int32_t cmd,
+                     struct gf_flock *flock, dict_t *xdata);
+
+int32_t dht_finodelk (call_frame_t        *frame, xlator_t *this,
+                      const char      *volume, fd_t *fd, int32_t cmd,
+                      struct gf_flock *flock, dict_t *xdata);
+
+int32_t dht_entrylk (call_frame_t    *frame, xlator_t *this,
+                     const char  *volume, loc_t *loc, const char *basename,
+                     entrylk_cmd  cmd, entrylk_type type, dict_t *xdata);
+
+int32_t dht_fentrylk (call_frame_t    *frame, xlator_t *this,
+                      const char  *volume, fd_t *fd, const char *basename,
+                      entrylk_cmd  cmd, entrylk_type type, dict_t *xdata);
+
+int32_t dht_readdir (call_frame_t  *frame,
+                     xlator_t *this,
+                     fd_t     *fd,
+                     size_t    size, off_t off, dict_t *xdata);
+
+int32_t dht_readdirp (call_frame_t *frame,
+                      xlator_t *this,
+                      fd_t     *fd,
+                      size_t    size, off_t off, dict_t *dict);
+
+int32_t dht_xattrop (call_frame_t           *frame,
+                     xlator_t           *this,
+                     loc_t              *loc,
+                     gf_xattrop_flags_t  flags,
+                     dict_t             *dict, dict_t *xdata);
+
+int32_t dht_fxattrop (call_frame_t *frame,
+                      xlator_t           *this,
+                      fd_t               *fd,
+                      gf_xattrop_flags_t  flags,
+                      dict_t             *dict, dict_t *xdata);
+
+int32_t dht_forget (xlator_t *this, inode_t *inode);
+int32_t dht_setattr (call_frame_t  *frame, xlator_t *this, loc_t *loc,
+                     struct iatt   *stbuf, int32_t valid, dict_t *xdata);
+int32_t dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      struct iatt  *stbuf, int32_t valid, dict_t *xdata);
+int32_t dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd,
+		      int32_t mode, off_t offset, size_t len, dict_t *xdata);
+int32_t dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd,
+		    off_t offset, size_t len, dict_t *xdata);
+int32_t dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd,
+                    off_t offset, off_t len, dict_t *xdata);
+
+int32_t dht_init (xlator_t *this);
+void    dht_fini (xlator_t *this);
+int     dht_reconfigure (xlator_t *this, dict_t *options);
+int32_t dht_notify (xlator_t *this, int32_t event, void *data, ...);
+
+/* definitions for nufa/switch */
+int dht_revalidate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, dict_t *xattr,
+                        struct iatt *postparent);
+int dht_lookup_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int op_ret, int op_errno, inode_t *inode,
+                        struct iatt *stbuf, dict_t *xattr,
+                        struct iatt *postparent);
+int dht_lookup_linkfile_cbk (call_frame_t *frame, void *cookie,
+                             xlator_t *this, int op_ret, int op_errno,
+                             inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+                             struct iatt *postparent);
+int dht_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno,
+                    inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+                    struct iatt *postparent);
+int dht_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int op_ret, int op_errno,
+                    fd_t *fd, inode_t *inode, struct iatt *stbuf,
+                    struct iatt *preparent, struct iatt *postparent,
+                    dict_t *xdata);
+int dht_newfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno,
+                     inode_t *inode, struct iatt *stbuf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata);
+
+int
+gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict);
+
+int
+gf_defrag_stop (gf_defrag_info_t *defrag, gf_defrag_status_t status,
+                dict_t *output);
+
+void*
+gf_defrag_start (void *this);
+
+int32_t
+gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
+                           struct iatt *stbuf);
+int
+dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
+                 int flag);
+int
+dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this,
+                          dht_layout_t **layout_int);
+int
+dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
+                          dht_layout_t* layout_int);
+int
+dht_inode_ctx_time_update (inode_t *inode, xlator_t *this, struct iatt *stat,
+                           int32_t update_ctx);
+void dht_inode_ctx_time_set (inode_t *inode, xlator_t *this, struct iatt *stat);
+
+int dht_inode_ctx_get (inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx);
+int dht_inode_ctx_set (inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx);
+int
+dht_dir_attr_heal (void *data);
+int
+dht_dir_attr_heal_done (int ret, call_frame_t *sync_frame, void *data);
+int
+dht_dir_has_layout (dict_t *xattr, char *name);
+gf_boolean_t
+dht_is_subvol_in_layout (dht_layout_t *layout, xlator_t *xlator);
+xlator_t *
+dht_subvol_with_free_space_inodes (xlator_t *this, xlator_t *subvol,
+                                   dht_layout_t *layout);
+xlator_t *
+dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol,
+                                  dht_layout_t *layout);
+int
+dht_linkfile_attr_heal (call_frame_t *frame, xlator_t *this);
+
+void
+dht_layout_dump (dht_layout_t  *layout, const char *prefix);
+int32_t
+dht_priv_dump (xlator_t *this);
+int32_t
+dht_inodectx_dump (xlator_t *this, inode_t *inode);
+
+int
+dht_inode_ctx_get1 (xlator_t *this, inode_t *inode, xlator_t **subvol);
 
-#endif /* _DHT_H */
+#endif/* _DHT_H */
diff --git a/xlators/cluster/dht/src/dht-diskusage.c b/xlators/cluster/dht/src/dht-diskusage.c
index fbc108575..fe3955ecb 100644
--- a/xlators/cluster/dht/src/dht-diskusage.c
+++ b/xlators/cluster/dht/src/dht-diskusage.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -33,50 +24,70 @@
 #include <sys/time.h>
 
 
-int 
+int
 dht_du_info_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int op_ret, int op_errno, struct statvfs *statvfs)
+		 int op_ret, int op_errno, struct statvfs *statvfs,
+                 dict_t *xdata)
 {
 	dht_conf_t    *conf         = NULL;
-        call_frame_t  *prev          = NULL;
+	call_frame_t  *prev          = NULL;
 	int            this_call_cnt = 0;
-        int            i = 0;
-        double         percent = 0;
-        uint64_t       bytes = 0;
-
-        conf = this->private;
-        prev = cookie;
-
-        if (op_ret == -1) 
-                goto out;
+	int            i = 0;
+	double         percent = 0;
+	double         percent_inodes = 0;
+	uint64_t       bytes = 0;
 
-        if (statvfs && statvfs->f_blocks) {
-                percent = (statvfs->f_bfree * 100) / statvfs->f_blocks;
-                bytes = (statvfs->f_bfree * statvfs->f_bsize);
-        }
-        
-        LOCK (&conf->subvolume_lock);
-        {
-                for (i = 0; i < conf->subvolume_cnt; i++)
-                        if (prev->this == conf->subvolumes[i]) {
-                                conf->du_stats[i].avail_percent = percent;
-                                conf->du_stats[i].avail_space   = bytes;
-                                gf_log (this->name, GF_LOG_DEBUG,
-                                        "on subvolume '%s': avail_percent is: "
-                                        "%.2f and avail_space is: %"PRIu64"",
-                                        prev->this->name, 
-                                        conf->du_stats[i].avail_percent,
-                                        conf->du_stats[i].avail_space);
-                        }
-        }
-        UNLOCK (&conf->subvolume_lock);
-
- out:
+	conf = this->private;
+	prev = cookie;
+
+	if (op_ret == -1) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"failed to get disk info from %s", prev->this->name);
+		goto out;
+	}
+
+	if (statvfs && statvfs->f_blocks) {
+		percent = (statvfs->f_bavail * 100) / statvfs->f_blocks;
+		bytes = (statvfs->f_bavail * statvfs->f_frsize);
+	}
+
+	if (statvfs && statvfs->f_files) {
+		percent_inodes = (statvfs->f_ffree * 100) / statvfs->f_files;
+	} else {
+		/* set percent inodes to 100 for dynamically allocated inode filesystems
+		   this logic holds good so that, distribute has nothing to worry about
+		   total inodes rather let the 'create()' to be scheduled on the hashed
+		   subvol regardless of the total inodes. since we have no awareness on
+		   loosing inodes this logic fits well
+		*/
+		percent_inodes = 100;
+	}
+
+	LOCK (&conf->subvolume_lock);
+	{
+		for (i = 0; i < conf->subvolume_cnt; i++)
+			if (prev->this == conf->subvolumes[i]) {
+				conf->du_stats[i].avail_percent = percent;
+				conf->du_stats[i].avail_space   = bytes;
+				conf->du_stats[i].avail_inodes  = percent_inodes;
+				gf_log (this->name, GF_LOG_DEBUG,
+					"on subvolume '%s': avail_percent is: "
+					"%.2f and avail_space is: %"PRIu64" "
+					"and avail_inodes is: %.2f",
+					prev->this->name,
+					conf->du_stats[i].avail_percent,
+					conf->du_stats[i].avail_space,
+					conf->du_stats[i].avail_inodes);
+			}
+	}
+	UNLOCK (&conf->subvolume_lock);
+
+out:
 	this_call_cnt = dht_frame_return (frame);
 	if (is_last_call (this_call_cnt))
 		DHT_STACK_DESTROY (frame);
 
-        return 0;
+	return 0;
 }
 
 int
@@ -85,179 +96,319 @@ dht_get_du_info_for_subvol (xlator_t *this, int subvol_idx)
 	dht_conf_t    *conf         = NULL;
 	call_frame_t  *statfs_frame = NULL;
 	dht_local_t   *statfs_local = NULL;
-        call_pool_t   *pool         = NULL;
+	call_pool_t   *pool         = NULL;
+	loc_t          tmp_loc      = {0,};
 
 	conf = this->private;
-        pool = this->ctx->pool;
-
-        statfs_frame = create_frame (this, pool);
-        if (!statfs_frame) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                goto err;
-        }
-        
-        statfs_local = dht_local_init (statfs_frame);
-        if (!statfs_local) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                goto err;
-        }
-        
-        loc_t tmp_loc = { .inode = NULL,
-                          .path = "/",
-        };
-        
-        statfs_local->call_cnt = 1;
-        STACK_WIND (statfs_frame, dht_du_info_cbk,
-                    conf->subvolumes[subvol_idx],
-                    conf->subvolumes[subvol_idx]->fops->statfs,
-                    &tmp_loc);
-
-        return 0;
- err:
+	pool = this->ctx->pool;
+
+	statfs_frame = create_frame (this, pool);
+	if (!statfs_frame) {
+		goto err;
+	}
+
+	/* local->fop value is not used in this case */
+	statfs_local = dht_local_init (statfs_frame, NULL, NULL,
+				       GF_FOP_MAXVALUE);
+	if (!statfs_local) {
+		goto err;
+	}
+
+        /* make it root gfid, should be enough to get the proper info back */
+        tmp_loc.gfid[15] = 1;
+
+	statfs_local->call_cnt = 1;
+	STACK_WIND (statfs_frame, dht_du_info_cbk,
+		    conf->subvolumes[subvol_idx],
+		    conf->subvolumes[subvol_idx]->fops->statfs,
+		    &tmp_loc, NULL);
+
+	return 0;
+err:
 	if (statfs_frame)
 		DHT_STACK_DESTROY (statfs_frame);
-        
-        return -1;
+
+	return -1;
 }
 
 int
 dht_get_du_info (call_frame_t *frame, xlator_t *this, loc_t *loc)
 {
-        int            i = 0;
+	int            i            = 0;
 	dht_conf_t    *conf         = NULL;
 	call_frame_t  *statfs_frame = NULL;
 	dht_local_t   *statfs_local = NULL;
-        struct timeval tv = {0,};
+	struct timeval tv           = {0,};
+        loc_t          tmp_loc      = {0,};
 
 	conf  = this->private;
 
 	gettimeofday (&tv, NULL);
-	if (tv.tv_sec > (conf->refresh_interval 
-			 + conf->last_stat_fetch.tv_sec)) {
 
-                statfs_frame = copy_frame (frame);
-                if (!statfs_frame) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
-                        goto err;
-                }
+        /* make it root gfid, should be enough to get the proper
+           info back */
+        tmp_loc.gfid[15] = 1;
 
-                statfs_local = dht_local_init (statfs_frame);
-                if (!statfs_local) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
-                        goto err;
-                }
-
-                loc_copy (&statfs_local->loc, loc);
-                loc_t tmp_loc = { .inode = NULL,
-                              .path = "/",
-                };
-                
-                statfs_local->call_cnt = conf->subvolume_cnt;
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        STACK_WIND (statfs_frame, dht_du_info_cbk,
-                                    conf->subvolumes[i],
-                                    conf->subvolumes[i]->fops->statfs,
-                                    &tmp_loc);
-                }
+	if (tv.tv_sec > (conf->refresh_interval
+			 + conf->last_stat_fetch.tv_sec)) {
 
-                conf->last_stat_fetch.tv_sec = tv.tv_sec;
-        }
-        return 0;
+		statfs_frame = copy_frame (frame);
+		if (!statfs_frame) {
+			goto err;
+		}
+
+		/* In this case, 'local->fop' is not used */
+		statfs_local = dht_local_init (statfs_frame, loc, NULL,
+					       GF_FOP_MAXVALUE);
+		if (!statfs_local) {
+			goto err;
+		}
+
+		statfs_local->call_cnt = conf->subvolume_cnt;
+		for (i = 0; i < conf->subvolume_cnt; i++) {
+			STACK_WIND (statfs_frame, dht_du_info_cbk,
+				    conf->subvolumes[i],
+				    conf->subvolumes[i]->fops->statfs,
+				    &tmp_loc, NULL);
+		}
+
+		conf->last_stat_fetch.tv_sec = tv.tv_sec;
+	}
+	return 0;
 err:
 	if (statfs_frame)
 		DHT_STACK_DESTROY (statfs_frame);
 
-        return -1;
+	return -1;
 }
 
 
-int
+gf_boolean_t
 dht_is_subvol_filled (xlator_t *this, xlator_t *subvol)
 {
-        int         i = 0;
-        int         subvol_filled = 0;
+	int         i = 0;
 	dht_conf_t *conf = NULL;
+	gf_boolean_t subvol_filled_inodes = _gf_false;
+	gf_boolean_t subvol_filled_space = _gf_false;
+	gf_boolean_t is_subvol_filled = _gf_false;
 
-        conf = this->private;
+	conf = this->private;
 
-        /* Check for values above specified percent or free disk */
-        LOCK (&conf->subvolume_lock);
-        {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (subvol == conf->subvolumes[i]) {
-                                if (conf->disk_unit == 'p') {
-                                        if (conf->du_stats[i].avail_percent <
-                                            conf->min_free_disk) {
-                                                subvol_filled = 1;
-                                                break;
-                                        }
-                                } else {
-                                        if (conf->du_stats[i].avail_space <
-                                            conf->min_free_disk) {
-                                                subvol_filled = 1;
-                                                break;
-                                        }
-                                }
-                        }
+	/* Check for values above specified percent or free disk */
+	LOCK (&conf->subvolume_lock);
+	{
+		for (i = 0; i < conf->subvolume_cnt; i++) {
+			if (subvol == conf->subvolumes[i]) {
+				if (conf->disk_unit == 'p') {
+					if (conf->du_stats[i].avail_percent <
+					    conf->min_free_disk) {
+						subvol_filled_space = _gf_true;
+						break;
+					}
+
+				} else {
+					if (conf->du_stats[i].avail_space <
+					    conf->min_free_disk) {
+						subvol_filled_space = _gf_true;
+						break;
+					}
+				}
+				if (conf->du_stats[i].avail_inodes <
+				    conf->min_free_inodes) {
+					subvol_filled_inodes = _gf_true;
+					break;
+				}
+			}
+		}
+	}
+	UNLOCK (&conf->subvolume_lock);
+
+	if (subvol_filled_space && conf->subvolume_status[i]) {
+		if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
+			gf_log (this->name, GF_LOG_WARNING,
+				"disk space on subvolume '%s' is getting "
+				"full (%.2f %%), consider adding more nodes",
+				subvol->name,
+				(100 - conf->du_stats[i].avail_percent));
+		}
+	}
+
+	if (subvol_filled_inodes && conf->subvolume_status[i]) {
+		if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
+			gf_log (this->name, GF_LOG_CRITICAL,
+				"inodes on subvolume '%s' are at "
+				"(%.2f %%), consider adding more nodes",
+				subvol->name,
+				(100 - conf->du_stats[i].avail_inodes));
+		}
+	}
+
+	is_subvol_filled = (subvol_filled_space || subvol_filled_inodes);
+
+	return is_subvol_filled;
+}
+
+
+/*Get the best subvolume to create the file in*/
+xlator_t *
+dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol,
+                                dht_local_t *local)
+{
+	xlator_t   *avail_subvol = NULL;
+	dht_conf_t *conf = NULL;
+        dht_layout_t *layout = NULL;
+        loc_t      *loc = NULL;
+
+	conf = this->private;
+        if (!local)
+                goto out;
+        loc = &local->loc;
+        if (!local->layout) {
+                layout = dht_layout_get (this, loc->parent);
+
+                if (!layout) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "layout missing path=%s parent=%s",
+                                loc->path, uuid_utoa (loc->parent->gfid));
+                        goto out;
                 }
+        } else {
+                layout = dht_layout_ref (this, local->layout);
         }
-        UNLOCK (&conf->subvolume_lock);
-
-        if (subvol_filled && conf->subvolume_status[i]) {
-                if (!(conf->du_stats[i].log++ % (GF_UNIVERSAL_ANSWER * 10))) {
-                        gf_log (this->name, GF_LOG_WARNING,
-                                "disk space on subvolume '%s' is getting "
-                                "full (%.2f %%), consider adding more nodes", 
-                                subvol->name, 
-                                (100 - conf->du_stats[i].avail_percent));
+
+        LOCK (&conf->subvolume_lock);
+	{
+                avail_subvol = dht_subvol_with_free_space_inodes(this, subvol,
+                                                                 layout);
+                if(!avail_subvol)
+                {
+                        avail_subvol = dht_subvol_maxspace_nonzeroinode(this,
+                                                                        subvol,
+                                                                        layout);
                 }
-        }
 
-        return subvol_filled;
+	}
+	UNLOCK (&conf->subvolume_lock);
+out:
+	if (!avail_subvol) {
+		gf_log (this->name,
+                        GF_LOG_DEBUG,
+			"no subvolume has enough free space and/or inodes\
+                         to create");
+                avail_subvol = subvol;
+	}
+
+        if (layout)
+                dht_layout_unref (this, layout);
+	return avail_subvol;
 }
 
+static inline
+int32_t dht_subvol_has_err (xlator_t *this, dht_layout_t *layout)
+{
+        int ret = -1;
+        int i   = 0;
+
+        if (!this || !layout)
+                goto out;
+
+        /* check if subvol has layout errors, before selecting it */
+        for (i = 0; i < layout->cnt; i++) {
+                if (!strcmp (layout->list[i].xlator->name, this->name) &&
+                     (layout->list[i].err != 0)) {
+                        ret = -1;
+                        goto out;
+                }
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+/*Get subvolume which has both space and inodes more than the min criteria*/
 xlator_t *
-dht_free_disk_available_subvol (xlator_t *this, xlator_t *subvol) 
+dht_subvol_with_free_space_inodes(xlator_t *this, xlator_t *subvol,
+                                  dht_layout_t *layout)
 {
-        int         i = 0;
-        double      max= 0;
-        xlator_t   *avail_subvol = NULL;
-	dht_conf_t *conf = NULL;
+        int i = 0;
+        double max = 0;
+        double max_inodes = 0;
+        int    ignore_subvol = 0;
+
+        xlator_t *avail_subvol = NULL;
+        dht_conf_t *conf = NULL;
 
         conf = this->private;
 
-        LOCK (&conf->subvolume_lock);
-        {
-                for (i = 0; i < conf->subvolume_cnt; i++) {
-                        if (conf->disk_unit == 'p') {
-                                if (conf->du_stats[i].avail_percent > max) {
-                                        max = conf->du_stats[i].avail_percent;
-                                        avail_subvol = conf->subvolumes[i];
-                                }
-                        } else {
-                                if (conf->du_stats[i].avail_space > max) {
-                                        max = conf->du_stats[i].avail_space;
-                                        avail_subvol = conf->subvolumes[i];
-                                }
+        for(i=0; i < conf->subvolume_cnt; i++) {
+                /* check if subvol has layout errors, before selecting it */
+                ignore_subvol = dht_subvol_has_err (conf->subvolumes[i],
+                                                    layout);
+                if (ignore_subvol)
+                        continue;
+
+                if ((conf->disk_unit == 'p') &&
+                    (conf->du_stats[i].avail_percent > conf->min_free_disk) &&
+                    (conf->du_stats[i].avail_inodes  > conf->min_free_inodes)) {
+                        if ((conf->du_stats[i].avail_inodes > max_inodes) ||
+                            (conf->du_stats[i].avail_percent > max)) {
+                                max = conf->du_stats[i].avail_percent;
+                                max_inodes = conf->du_stats[i].avail_inodes;
+                                avail_subvol = conf->subvolumes[i];
                         }
                 }
-        }
-        UNLOCK (&conf->subvolume_lock);
 
-        if (!avail_subvol) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "no subvolume has enough free space to create");
+                if ((conf->disk_unit != 'p') &&
+                    (conf->du_stats[i].avail_space > conf->min_free_disk) &&
+                    (conf->du_stats[i].avail_inodes  > conf->min_free_inodes)) {
+                        if ((conf->du_stats[i].avail_inodes > max_inodes) ||
+                            (conf->du_stats[i].avail_space > max)) {
+                                max = conf->du_stats[i].avail_space;
+                                max_inodes = conf->du_stats[i].avail_inodes;
+                                avail_subvol = conf->subvolumes[i];
+                        }
+                }
         }
 
-        if (max < conf->min_free_disk)
-                avail_subvol = subvol;
+        return avail_subvol;
+}
+
+
+/* Get subvol which has atleast one inode and maximum space */
+xlator_t *
+dht_subvol_maxspace_nonzeroinode (xlator_t *this, xlator_t *subvol,
+                                  dht_layout_t *layout)
+{
+        int         i = 0;
+        double      max = 0;
+        int         ignore_subvol = 0;
+
+        xlator_t   *avail_subvol = NULL;
+        dht_conf_t *conf = NULL;
 
-        if (!avail_subvol)
-                avail_subvol = subvol;                
+        conf = this->private;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                /* check if subvol has layout errors, before selecting it */
+                ignore_subvol = dht_subvol_has_err (conf->subvolumes[i],
+                                                    layout);
+                if (ignore_subvol)
+                        continue;
+
+                if (conf->disk_unit == 'p') {
+                        if ((conf->du_stats[i].avail_percent > max)
+                            && (conf->du_stats[i].avail_inodes > 0 )) {
+                                max = conf->du_stats[i].avail_percent;
+                                avail_subvol = conf->subvolumes[i];
+                        }
+               } else {
+                         if ((conf->du_stats[i].avail_space > max)
+                            && (conf->du_stats[i].avail_inodes > 0)) {
+                                 max = conf->du_stats[i].avail_space;
+                                 avail_subvol = conf->subvolumes[i];
+                         }
+               }
+        }
 
         return avail_subvol;
 }
diff --git a/xlators/cluster/dht/src/dht-hashfn.c b/xlators/cluster/dht/src/dht-hashfn.c
index 09394a814..656cf23a0 100644
--- a/xlators/cluster/dht/src/dht-hashfn.c
+++ b/xlators/cluster/dht/src/dht-hashfn.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -32,50 +23,89 @@
 int
 dht_hash_compute_internal (int type, const char *name, uint32_t *hash_p)
 {
-	int      ret = 0;
-	uint32_t hash = 0;
-
-	switch (type) {
-	case DHT_HASH_TYPE_DM:
-		hash = gf_dm_hashfn (name, strlen (name));
-		break;
-	default:
-		ret = -1;
-		break;
-	}
-
-	if (ret == 0) {
-		*hash_p = hash;
-	}
-
-	return ret;
+        int      ret = 0;
+        uint32_t hash = 0;
+
+        switch (type) {
+        case DHT_HASH_TYPE_DM:
+        case DHT_HASH_TYPE_DM_USER:
+                hash = gf_dm_hashfn (name, strlen (name));
+                break;
+        default:
+                ret = -1;
+                break;
+        }
+
+        if (ret == 0) {
+                *hash_p = hash;
+        }
+
+        return ret;
 }
 
 
-#define MAKE_RSYNC_FRIENDLY_NAME(rsync_frndly_name, name) do {          \
-                rsync_frndly_name = (char *) name;			\
-                if (name[0] == '.') {                                   \
-                        char *dot   = 0;                                \
-                        int namelen = 0;                                \
-                                                                        \
-                        dot = strrchr (name, '.');                      \
-                        if (dot && dot > (name + 1) && *(dot + 1)) {    \
-                                namelen = (dot - name);                 \
-                                rsync_frndly_name = alloca (namelen);   \
-                                strncpy (rsync_frndly_name, name + 1,   \
-                                         namelen);                      \
-                                rsync_frndly_name[namelen - 1] = 0;     \
-                        }                                               \
-                }                                                       \
-        } while (0);
-
+static inline
+gf_boolean_t
+dht_munge_name (const char *original, char *modified, size_t len, regex_t *re)
+{
+        regmatch_t      matches[2];
+        size_t          new_len;
+
+        if (regexec(re,original,2,matches,0) != REG_NOMATCH) {
+                if (matches[1].rm_so != -1) {
+                        new_len = matches[1].rm_eo - matches[1].rm_so;
+                        /* Equal would fail due to the NUL at the end. */
+                        if (new_len < len) {
+                                memcpy (modified,original+matches[1].rm_so,
+                                        new_len);
+                                modified[new_len] = '\0';
+                                return _gf_true;
+                        }
+                }
+        }
+
+        /* This is guaranteed safe because of how the dest was allocated. */
+        strcpy(modified,original);
+        return _gf_false;
+}
 
 int
-dht_hash_compute (int type, const char *name, uint32_t *hash_p)
+dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p)
 {
-	char     *rsync_friendly_name = NULL;
-
-	MAKE_RSYNC_FRIENDLY_NAME (rsync_friendly_name, name);
-
-	return dht_hash_compute_internal (type, rsync_friendly_name, hash_p);
+        char            *rsync_friendly_name    = NULL;
+        dht_conf_t      *priv                   = this->private;
+        size_t           len                    = 0;
+        gf_boolean_t     munged                 = _gf_false;
+
+        /*
+         * It wouldn't be safe to use alloca in an inline function that doesn't
+         * actually get inlined, and it wouldn't be efficient to do a real
+         * allocation, so we use alloca here (if needed) and pass that to the
+         * inline.
+         */
+
+        if (priv->extra_regex_valid) {
+                len = strlen(name) + 1;
+                rsync_friendly_name = alloca(len);
+                munged = dht_munge_name (name, rsync_friendly_name, len,
+                                         &priv->extra_regex);
+        }
+
+        if (!munged && priv->rsync_regex_valid) {
+                len = strlen(name) + 1;
+                rsync_friendly_name = alloca(len);
+                gf_log (this->name, GF_LOG_TRACE, "trying regex for %s", name);
+                munged = dht_munge_name (name, rsync_friendly_name, len,
+                                         &priv->rsync_regex);
+                if (munged) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "munged down to %s", rsync_friendly_name);
+                }
+        }
+
+        if (!munged) {
+                rsync_friendly_name = (char *)name;
+        }
+
+        return dht_hash_compute_internal (type, rsync_friendly_name, hash_p);
 }
diff --git a/xlators/cluster/dht/src/dht-helper.c b/xlators/cluster/dht/src/dht-helper.c
index 9e21713f7..f1dc5072f 100644
--- a/xlators/cluster/dht/src/dht-helper.c
+++ b/xlators/cluster/dht/src/dht-helper.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -27,56 +18,133 @@
 #include "xlator.h"
 #include "dht-common.h"
 
+static inline int
+dht_inode_ctx_set1 (xlator_t *this, inode_t *inode, xlator_t *subvol)
+{
+        uint64_t tmp_subvol = 0;
+
+        tmp_subvol = (long)subvol;
+        return inode_ctx_set1 (inode, this, &tmp_subvol);
+}
+
+int
+dht_inode_ctx_get1 (xlator_t *this, inode_t *inode, xlator_t **subvol)
+{
+        int ret = -1;
+        uint64_t tmp_subvol = 0;
+
+        ret =  inode_ctx_get1 (inode, this, &tmp_subvol);
+        if (tmp_subvol && subvol)
+                *subvol = (xlator_t *)tmp_subvol;
+
+        return ret;
+}
+
 
 int
 dht_frame_return (call_frame_t *frame)
 {
-	dht_local_t *local = NULL;
-	int          this_call_cnt = -1;
+        dht_local_t *local = NULL;
+        int          this_call_cnt = -1;
+
+        if (!frame)
+                return -1;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                this_call_cnt = --local->call_cnt;
+        }
+        UNLOCK (&frame->lock);
 
-	if (!frame)
-		return -1;
+        return this_call_cnt;
+}
 
-	local = frame->local;
 
-	LOCK (&frame->lock);
-	{
-		this_call_cnt = --local->call_cnt;
+static uint64_t
+dht_bits_for (uint64_t num)
+{
+	uint64_t bits = 0, ctrl = 1;
+
+	while (ctrl < num) {
+		ctrl *= 2;
+		bits ++;
 	}
-	UNLOCK (&frame->lock);
 
-	return this_call_cnt;
+	return bits;
 }
 
+/*
+ * A slightly "updated" version of the algorithm described in the commit log
+ * is used here.
+ *
+ * The only enhancement is that:
+ *
+ * - The number of bits used by the backend filesystem for HUGE d_off which
+ *   is described as 63, and
+ * - The number of bits used by the d_off presented by the transformation
+ *   upwards which is described as 64, are both made "configurable."
+ */
+
+
+#define BACKEND_D_OFF_BITS 63
+#define PRESENT_D_OFF_BITS 63
+
+#define ONE 1ULL
+#define MASK (~0ULL)
+#define PRESENT_MASK (MASK >> (64 - PRESENT_D_OFF_BITS))
+#define BACKEND_MASK (MASK >> (64 - BACKEND_D_OFF_BITS))
+
+#define TOP_BIT (ONE << (PRESENT_D_OFF_BITS - 1))
+#define SHIFT_BITS (max (0, (BACKEND_D_OFF_BITS - PRESENT_D_OFF_BITS + 1)))
 
 int
 dht_itransform (xlator_t *this, xlator_t *subvol, uint64_t x, uint64_t *y_p)
 {
-	dht_conf_t *conf = NULL;
-	int         cnt = 0;
-	int         max = 0;
-	uint64_t    y = 0;
+        dht_conf_t *conf = NULL;
+        int         cnt = 0;
+        int         max = 0;
+        uint64_t    y = 0;
+        uint64_t    hi_mask = 0;
+        uint64_t    off_mask = 0;
+        int         max_bits = 0;
+
+        if (x == ((uint64_t) -1)) {
+                y = (uint64_t) -1;
+                goto out;
+        }
 
+        conf = this->private;
+        if (!conf)
+                goto out;
 
-	if (x == ((uint64_t) -1)) {
-		y = (uint64_t) -1;
+        max = conf->subvolume_cnt;
+        cnt = dht_subvol_cnt (this, subvol);
+
+	if (max == 1) {
+		y = x;
 		goto out;
 	}
 
-	conf = this->private;
-        if (!conf)
-                goto out;
+        max_bits = dht_bits_for (max);
 
-	max = conf->subvolume_cnt;
-	cnt = dht_subvol_cnt (this, subvol);
+        hi_mask = ~(PRESENT_MASK >> (max_bits + 1));
 
-	y = ((x * max) + cnt);
+        if (x & hi_mask) {
+                /* HUGE d_off */
+                off_mask = MASK << max_bits;
+                y = TOP_BIT | ((x >> SHIFT_BITS) & off_mask) | cnt;
+        } else {
+                /* small d_off */
+                y = ((x * max) + cnt);
+        }
 
 out:
-	if (y_p)
-		*y_p = y;
+        if (y_p)
+                *y_p = y;
 
-	return 0;
+        return 0;
 }
 
 int
@@ -90,7 +158,7 @@ dht_filter_loc_subvol_key (xlator_t *this, loc_t *loc, loc_t *new_loc,
         int            ret       = 0; /* not found */
 
         /* Why do other tasks if first required 'char' itself is not there */
-        if (loc->name && !strchr (loc->name, '@'))
+        if (!new_loc || !loc || !loc->name || !strchr (loc->name, '@'))
                 goto out;
 
         trav = this->children;
@@ -118,7 +186,6 @@ dht_filter_loc_subvol_key (xlator_t *this, loc_t *loc, loc_t *new_loc,
                                 new_loc->path   = ((new_path) ? new_path:
                                                    gf_strdup (loc->path));
                                 new_loc->name   = new_name;
-                                new_loc->ino    = loc->ino;
                                 new_loc->inode  = inode_ref (loc->inode);
                                 new_loc->parent = inode_ref (loc->parent);
                         }
@@ -131,158 +198,200 @@ dht_filter_loc_subvol_key (xlator_t *this, loc_t *loc, loc_t *new_loc,
 out:
         if (!ret) {
                 /* !success */
-                if (new_path)
-                        GF_FREE (new_path);
-                if (new_name)
-                        GF_FREE (new_name);
+                GF_FREE (new_path);
+                GF_FREE (new_name);
         }
         return ret;
 }
 
 int
 dht_deitransform (xlator_t *this, uint64_t y, xlator_t **subvol_p,
-		  uint64_t *x_p)
+                  uint64_t *x_p)
 {
-	dht_conf_t *conf = NULL;
-	int         cnt = 0;
-	int         max = 0;
-	uint64_t    x = 0;
-	xlator_t   *subvol = 0;
+        dht_conf_t *conf = NULL;
+        int         cnt = 0;
+        int         max = 0;
+        uint64_t    x = 0;
+        xlator_t   *subvol = 0;
+        int         max_bits = 0;
+        uint64_t    off_mask = 0;
+        uint64_t    host_mask = 0;
 
         if (!this->private)
-                goto out;
+                return -1;
 
-	conf = this->private;
-	max = conf->subvolume_cnt;
+        conf = this->private;
+        max = conf->subvolume_cnt;
 
-	cnt = y % max;
-	x   = y / max;
+	if (max == 1) {
+		x = y;
+		cnt = 0;
+		goto out;
+	}
 
-	subvol = conf->subvolumes[cnt];
+        if (y & TOP_BIT) {
+                /* HUGE d_off */
+                max_bits = dht_bits_for (max);
+                off_mask = (MASK << max_bits);
+                host_mask = ~(off_mask);
 
-	if (subvol_p)
-		*subvol_p = subvol;
+                x = ((y & ~TOP_BIT) & off_mask) << SHIFT_BITS;
 
-	if (x_p)
-		*x_p = x;
+                cnt = y & host_mask;
+	} else {
+                /* small d_off */
+                cnt = y % max;
+                x = y / max;
+        }
 
 out:
-	return 0;
+        subvol = conf->subvolumes[cnt];
+
+        if (subvol_p)
+                *subvol_p = subvol;
+
+        if (x_p)
+                *x_p = x;
+
+        return 0;
 }
 
 
 void
 dht_local_wipe (xlator_t *this, dht_local_t *local)
 {
-	if (!local)
-		return;
+        if (!local)
+                return;
 
-	loc_wipe (&local->loc);
-	loc_wipe (&local->loc2);
+        loc_wipe (&local->loc);
+        loc_wipe (&local->loc2);
 
-	if (local->xattr)
-		dict_unref (local->xattr);
+        if (local->xattr)
+                dict_unref (local->xattr);
 
-	if (local->inode)
-		inode_unref (local->inode);
+        if (local->inode)
+                inode_unref (local->inode);
 
-	if (local->layout) {
-		dht_layout_unref (this, local->layout);
+        if (local->layout) {
+                dht_layout_unref (this, local->layout);
                 local->layout = NULL;
         }
 
-	loc_wipe (&local->linkfile.loc);
+        loc_wipe (&local->linkfile.loc);
 
-	if (local->linkfile.xattr)
-		dict_unref (local->linkfile.xattr);
+        if (local->linkfile.xattr)
+                dict_unref (local->linkfile.xattr);
 
-	if (local->linkfile.inode)
-		inode_unref (local->linkfile.inode);
+        if (local->linkfile.inode)
+                inode_unref (local->linkfile.inode);
 
-	if (local->fd) {
-		fd_unref (local->fd);
-		local->fd = NULL;
-	}
+        if (local->fd) {
+                fd_unref (local->fd);
+                local->fd = NULL;
+        }
 
         if (local->params) {
                 dict_unref (local->params);
                 local->params = NULL;
         }
 
-	if (local->xattr_req)
-		dict_unref (local->xattr_req);
+        if (local->xattr_req)
+                dict_unref (local->xattr_req);
 
         if (local->selfheal.layout) {
                 dht_layout_unref (this, local->selfheal.layout);
                 local->selfheal.layout = NULL;
         }
 
-        if (local->newpath) {
-                GF_FREE (local->newpath);
-        }
+        GF_FREE (local->newpath);
+
+        GF_FREE (local->key);
+
+        GF_FREE (local->rebalance.vector);
 
-	GF_FREE (local);
+        if (local->rebalance.iobref)
+                iobref_unref (local->rebalance.iobref);
+
+        mem_put (local);
 }
 
 
 dht_local_t *
-dht_local_init (call_frame_t *frame)
+dht_local_init (call_frame_t *frame, loc_t *loc, fd_t *fd, glusterfs_fop_t fop)
 {
-	dht_local_t *local = NULL;
-
-	/* TODO: use mem-pool */
-	local = GF_CALLOC (1, sizeof (*local),
-                           gf_dht_mt_dht_local_t);
+        dht_local_t *local = NULL;
+        inode_t     *inode = NULL;
+        int          ret   = 0;
 
-	if (!local)
-		return NULL;
+        local = mem_get0 (THIS->local_pool);
+        if (!local)
+                goto out;
 
-	local->op_ret = -1;
-	local->op_errno = EUCLEAN;
+        if (loc) {
+                ret = loc_copy (&local->loc, loc);
+                if (ret)
+                        goto out;
 
-	frame->local = local;
+                inode = loc->inode;
+        }
 
-	return local;
-}
+        if (fd) {
+                local->fd = fd_ref (fd);
+                if (!inode)
+                        inode = fd->inode;
+        }
 
+        local->op_ret   = -1;
+        local->op_errno = EUCLEAN;
+        local->fop      = fop;
 
-char *
-basestr (const char *str)
-{
-        char *basestr = NULL;
+        if (inode) {
+                local->layout   = dht_layout_get (frame->this, inode);
+                local->cached_subvol = dht_subvol_get_cached (frame->this,
+                                                              inode);
+        }
 
-        basestr = strrchr (str, '/');
-        if (basestr)
-                basestr ++;
+        frame->local = local;
 
-        return basestr;
+out:
+        if (ret) {
+                if (local)
+                        mem_put (local);
+                local = NULL;
+        }
+        return local;
 }
 
-
 xlator_t *
 dht_first_up_subvol (xlator_t *this)
 {
-	dht_conf_t *conf = NULL;
-	xlator_t   *child = NULL;
-	int         i = 0;
+        dht_conf_t *conf = NULL;
+        xlator_t   *child = NULL;
+        int         i = 0;
+        time_t      time = 0;
 
-	conf = this->private;
+        conf = this->private;
         if (!conf)
                 goto out;
 
-	LOCK (&conf->subvolume_lock);
-	{
-		for (i = 0; i < conf->subvolume_cnt; i++) {
-			if (conf->subvolume_status[i]) {
-				child = conf->subvolumes[i];
-				break;
-			}
-		}
-	}
-	UNLOCK (&conf->subvolume_lock);
+        LOCK (&conf->subvolume_lock);
+        {
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        if (conf->subvol_up_time[i]) {
+                                if (!time) {
+                                        time = conf->subvol_up_time[i];
+                                        child = conf->subvolumes[i];
+                                } else if (time > conf->subvol_up_time[i]) {
+                                        time  = conf->subvol_up_time[i];
+                                        child = conf->subvolumes[i];
+                                }
+                        }
+                }
+        }
+        UNLOCK (&conf->subvolume_lock);
 
 out:
-	return child;
+        return child;
 }
 
 xlator_t *
@@ -317,17 +426,23 @@ dht_subvol_get_hashed (xlator_t *this, loc_t *loc)
         dht_layout_t *layout = NULL;
         xlator_t     *subvol = NULL;
 
-        if (is_fs_root (loc)) {
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, loc, out);
+
+        if (__is_root_gfid (loc->gfid)) {
                 subvol = dht_first_up_subvol (this);
                 goto out;
         }
 
+        GF_VALIDATE_OR_GOTO (this->name, loc->parent, out);
+        GF_VALIDATE_OR_GOTO (this->name, loc->name, out);
+
         layout = dht_layout_get (this, loc->parent);
 
         if (!layout) {
                 gf_log (this->name, GF_LOG_DEBUG,
-                        "layout missing path=%s parent=%"PRId64,
-                        loc->path, loc->parent->ino);
+                        "layout missing path=%s parent=%s",
+                        loc->path, uuid_utoa (loc->parent->gfid));
                 goto out;
         }
 
@@ -355,6 +470,8 @@ dht_subvol_get_cached (xlator_t *this, inode_t *inode)
         dht_layout_t *layout = NULL;
         xlator_t     *subvol = NULL;
 
+        GF_VALIDATE_OR_GOTO (this->name, this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
 
         layout = dht_layout_get (this, inode);
 
@@ -362,7 +479,7 @@ dht_subvol_get_cached (xlator_t *this, inode_t *inode)
                 goto out;
         }
 
-	subvol = layout->list[0].xlator;
+        subvol = layout->list[0].xlator;
 
 out:
         if (layout) {
@@ -376,150 +493,714 @@ out:
 xlator_t *
 dht_subvol_next (xlator_t *this, xlator_t *prev)
 {
-	dht_conf_t *conf = NULL;
-	int         i = 0;
-	xlator_t   *next = NULL;
+        dht_conf_t *conf = NULL;
+        int         i = 0;
+        xlator_t   *next = NULL;
 
-	conf = this->private;
+        conf = this->private;
         if (!conf)
                 goto out;
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		if (conf->subvolumes[i] == prev) {
-			if ((i + 1) < conf->subvolume_cnt)
-				next = conf->subvolumes[i + 1];
-			break;
-		}
-	}
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (conf->subvolumes[i] == prev) {
+                        if ((i + 1) < conf->subvolume_cnt)
+                                next = conf->subvolumes[i + 1];
+                        break;
+                }
+        }
 
 out:
-	return next;
+        return next;
 }
 
+/* This func wraps around, if prev is actually the last subvol.
+ */
+xlator_t *
+dht_subvol_next_available (xlator_t *this, xlator_t *prev)
+{
+        dht_conf_t *conf = NULL;
+        int         i = 0;
+        xlator_t   *next = NULL;
 
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (conf->subvolumes[i] == prev) {
+                        /* if prev is last in conf->subvolumes, then wrap
+                         * around.
+                         */
+                        if ((i + 1) < conf->subvolume_cnt) {
+                                next = conf->subvolumes[i + 1];
+                        } else {
+                                next = conf->subvolumes[0];
+                        }
+                        break;
+                }
+        }
+
+out:
+        return next;
+}
 int
 dht_subvol_cnt (xlator_t *this, xlator_t *subvol)
 {
-	int i = 0;
-	int ret = -1;
-	dht_conf_t *conf = NULL;
+        int i = 0;
+        int ret = -1;
+        dht_conf_t *conf = NULL;
 
-	conf = this->private;
+        conf = this->private;
         if (!conf)
                 goto out;
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		if (subvol == conf->subvolumes[i]) {
-			ret = i;
-			break;
-		}
-	}
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (subvol == conf->subvolumes[i]) {
+                        ret = i;
+                        break;
+                }
+        }
 
 out:
-	return ret;
+        return ret;
 }
 
 
-#define set_if_greater(a, b) do {		\
-		if ((a) < (b))			\
-			(a) = (b);		\
-	} while (0)
+#define set_if_greater(a, b) do {               \
+                if ((a) < (b))                  \
+                        (a) = (b);              \
+        } while (0)
+
+
+#define set_if_greater_time(a, an, b, bn) do {                          \
+                if (((a) < (b)) || (((a) == (b)) && ((an) < (bn)))){    \
+                        (a) = (b);                                      \
+                        (an) = (bn);                                    \
+                }                                                       \
+        } while (0)                                                     \
+
 
 int
 dht_iatt_merge (xlator_t *this, struct iatt *to,
-		struct iatt *from, xlator_t *subvol)
+                struct iatt *from, xlator_t *subvol)
 {
         if (!from || !to)
                 return 0;
 
-	to->ia_dev      = from->ia_dev;
+        to->ia_dev      = from->ia_dev;
 
         uuid_copy (to->ia_gfid, from->ia_gfid);
 
-	dht_itransform (this, subvol, from->ia_ino, &to->ia_ino);
+        to->ia_ino      = from->ia_ino;
+        to->ia_prot     = from->ia_prot;
+        to->ia_type     = from->ia_type;
+        to->ia_nlink    = from->ia_nlink;
+        to->ia_rdev     = from->ia_rdev;
+        to->ia_size    += from->ia_size;
+        to->ia_blksize  = from->ia_blksize;
+        to->ia_blocks  += from->ia_blocks;
+
+        set_if_greater (to->ia_uid, from->ia_uid);
+        set_if_greater (to->ia_gid, from->ia_gid);
+
+        set_if_greater_time(to->ia_atime, to->ia_atime_nsec,
+                            from->ia_atime, from->ia_atime_nsec);
+        set_if_greater_time (to->ia_mtime, to->ia_mtime_nsec,
+                             from->ia_mtime, from->ia_mtime_nsec);
+        set_if_greater_time (to->ia_ctime, to->ia_ctime_nsec,
+                             from->ia_ctime, from->ia_ctime_nsec);
 
-	to->ia_prot     = from->ia_prot;
-	to->ia_type     = from->ia_type;
-	to->ia_nlink    = from->ia_nlink;
-	to->ia_rdev     = from->ia_rdev;
-	to->ia_size    += from->ia_size;
-	to->ia_blksize  = from->ia_blksize;
-	to->ia_blocks  += from->ia_blocks;
+        return 0;
+}
 
-	set_if_greater (to->ia_uid, from->ia_uid);
-	set_if_greater (to->ia_gid, from->ia_gid);
+int
+dht_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name)
+{
+        if (!child) {
+                goto err;
+        }
 
-	set_if_greater (to->ia_atime, from->ia_atime);
-	set_if_greater (to->ia_mtime, from->ia_mtime);
-	set_if_greater (to->ia_ctime, from->ia_ctime);
+        if (strcmp (parent->path, "/") == 0)
+                gf_asprintf ((char **)&child->path, "/%s", name);
+        else
+                gf_asprintf ((char **)&child->path, "%s/%s", parent->path, name);
+
+        if (!child->path) {
+                goto err;
+        }
 
-	return 0;
+        child->name = strrchr (child->path, '/');
+        if (child->name)
+                child->name++;
+
+        child->parent = inode_ref (parent->inode);
+        child->inode = inode_new (parent->inode->table);
+
+        if (!child->inode) {
+                goto err;
+        }
+
+        return 0;
+err:
+        loc_wipe (child);
+        return -1;
 }
 
+
+
 int
-dht_frame_su_do (call_frame_t *frame)
+dht_init_subvolumes (xlator_t *this, dht_conf_t *conf)
 {
-        dht_local_t     *local = NULL;
+        xlator_list_t *subvols = NULL;
+        int            cnt = 0;
 
-        local = frame->local;
+        if (!conf)
+                return -1;
+
+        for (subvols = this->children; subvols; subvols = subvols->next)
+                cnt++;
 
-        local->uid = frame->root->uid;
-        local->gid = frame->root->gid;
+        conf->subvolumes = GF_CALLOC (cnt, sizeof (xlator_t *),
+                                      gf_dht_mt_xlator_t);
+        if (!conf->subvolumes) {
+                return -1;
+        }
+        conf->subvolume_cnt = cnt;
 
-        frame->root->uid = 0;
-        frame->root->gid = 0;
+        cnt = 0;
+        for (subvols = this->children; subvols; subvols = subvols->next)
+                conf->subvolumes[cnt++] = subvols->xlator;
+
+        conf->subvolume_status = GF_CALLOC (cnt, sizeof (char),
+                                            gf_dht_mt_char);
+        if (!conf->subvolume_status) {
+                return -1;
+        }
+
+        conf->last_event = GF_CALLOC (cnt, sizeof (int),
+                                      gf_dht_mt_char);
+        if (!conf->last_event) {
+                return -1;
+        }
+
+        conf->subvol_up_time = GF_CALLOC (cnt, sizeof (time_t),
+                                          gf_dht_mt_subvol_time);
+        if (!conf->subvol_up_time) {
+                return -1;
+        }
+
+        conf->du_stats = GF_CALLOC (conf->subvolume_cnt, sizeof (dht_du_t),
+                                    gf_dht_mt_dht_du_t);
+        if (!conf->du_stats) {
+                return -1;
+        }
+
+        conf->decommissioned_bricks = GF_CALLOC (cnt, sizeof (xlator_t *),
+                                                 gf_dht_mt_xlator_t);
+        if (!conf->decommissioned_bricks) {
+                return -1;
+        }
 
         return 0;
 }
 
 
-int
-dht_frame_su_undo (call_frame_t *frame)
+
+
+static int
+dht_migration_complete_check_done (int op_ret, call_frame_t *frame, void *data)
 {
-        dht_local_t     *local = NULL;
+        dht_local_t *local = NULL;
 
         local = frame->local;
 
-        frame->root->uid = local->uid;
-        frame->root->gid = local->gid;
+        local->rebalance.target_op_fn (THIS, frame, op_ret);
 
         return 0;
 }
 
 
 int
-dht_build_child_loc (xlator_t *this, loc_t *child, loc_t *parent, char *name)
+dht_migration_complete_check_task (void *data)
 {
-        if (!child) {
-                goto err;
+        int           ret      = -1;
+        xlator_t     *src_node = NULL;
+        xlator_t     *dst_node = NULL;
+        dht_local_t  *local    = NULL;
+        dict_t       *dict     = NULL;
+        dht_layout_t *layout   = NULL;
+        struct iatt   stbuf    = {0,};
+        xlator_t     *this     = NULL;
+        call_frame_t *frame    = NULL;
+        loc_t         tmp_loc  = {0,};
+        char         *path     = NULL;
+        dht_conf_t   *conf     = NULL;
+        inode_t      *inode    = NULL;
+        fd_t         *iter_fd  = NULL;
+        uint64_t      tmp_subvol = 0;
+        int           open_failed = 0;
+
+        this  = THIS;
+        frame = data;
+        local = frame->local;
+        conf = this->private;
+
+        src_node = local->cached_subvol;
+
+        if (!local->loc.inode && !local->fd) {
+                local->op_errno = EINVAL;
+                goto out;
         }
 
-        if (strcmp (parent->path, "/") == 0)
-                gf_asprintf ((char **)&child->path, "/%s", name);
-        else
-                gf_asprintf ((char **)&child->path, "%s/%s", parent->path, name);
+        inode = (!local->fd) ? local->loc.inode : local->fd->inode;
 
-        if (!child->path) {
+        /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
+         * as root:root. If a fd is already open, access check wont be done*/
+
+        if (!local->loc.inode) {
+                ret = syncop_fgetxattr (src_node, local->fd, &dict,
+                                        conf->link_xattr_name);
+        } else {
+                SYNCTASK_SETID (0, 0);
+                ret = syncop_getxattr (src_node, &local->loc, &dict,
+                                       conf->link_xattr_name);
+                SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+        }
+
+        if (!ret)
+                dst_node = dht_linkfile_subvol (this, NULL, NULL, dict);
+
+        if (ret) {
+                if (!dht_inode_missing(-ret) || (!local->loc.inode)) {
+                        local->op_errno = -ret;
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "%s: failed to get the 'linkto' xattr %s",
+                                local->loc.path, strerror (-ret));
+                        ret = -1;
+                        goto out;
+                }
+
+                /* Need to do lookup on hashed subvol, then get the file */
+                ret = syncop_lookup (this, &local->loc, NULL, &stbuf, NULL,
+                                     NULL);
+                if (ret) {
+                        local->op_errno = -ret;
+                        ret = -1;
+                        goto out;
+                }
+
+                dst_node = dht_subvol_get_cached (this, local->loc.inode);
+        }
+
+        if (!dst_node) {
                 gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                goto err;
+                        "%s: failed to get the destination node",
+                        local->loc.path);
+                ret = -1;
+                local->op_errno = EINVAL;
+                goto out;
         }
 
-        child->name = strrchr (child->path, '/');
-        if (child->name)
-                child->name++;
+        /* lookup on dst */
+        if (local->loc.inode) {
+                ret = syncop_lookup (dst_node, &local->loc, NULL, &stbuf, NULL,
+                                     NULL);
+
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "%s: failed to lookup the file on %s",
+                                local->loc.path, dst_node->name);
+                        local->op_errno = -ret;
+                        ret = -1;
+                        goto out;
+                }
 
-        child->parent = inode_ref (parent->inode);
-        child->inode = inode_new (parent->inode->table);
+                if (uuid_compare (stbuf.ia_gfid, local->loc.inode->gfid)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "%s: gfid different on the target file on %s",
+                                local->loc.path, dst_node->name);
+                        ret = -1;
+                        local->op_errno = EIO;
+                        goto out;
+                }
+        }
 
-        if (!child->inode) {
+        /* update inode ctx (the layout) */
+        dht_layout_unref (this, local->layout);
+
+        ret = dht_layout_preset (this, dst_node, inode);
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "%s: could not set preset layout for subvol %s",
+                        local->loc.path, dst_node->name);
+                ret   = -1;
+                local->op_errno = EINVAL;
+                goto out;
+        }
+
+        layout = dht_layout_for_subvol (this, dst_node);
+        if (!layout) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "%s: no pre-set layout for subvolume %s",
+                        local->loc.path, dst_node ? dst_node->name : "<nil>");
+                ret = -1;
+                local->op_errno = EINVAL;
+                goto out;
+        }
+
+        ret = dht_layout_set (this, inode, layout);
+        if (ret) {
                 gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                goto err;
+                        "%s: failed to set the new layout",
+                        local->loc.path);
+                local->op_errno = EINVAL;
+                goto out;
+        }
+
+        local->cached_subvol = dst_node;
+        ret = 0;
+
+        /* once we detect the migration complete, the inode-ctx2 is no more
+           required.. delete the ctx and also, it means, open() already
+           done on all the fd of inode */
+        ret = inode_ctx_reset1 (inode, this, &tmp_subvol);
+        if (tmp_subvol)
+                goto out;
+
+        if (list_empty (&inode->fd_list))
+                goto out;
+
+        /* perform open as root:root. There is window between linkfile
+         * creation(root:root) and setattr with the correct uid/gid
+         */
+        SYNCTASK_SETID(0, 0);
+
+        /* perform 'open()' on all the fd's present on the inode */
+        tmp_loc.inode = inode;
+        inode_path (inode, NULL, &path);
+        if (path)
+                tmp_loc.path = path;
+        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
+                if (fd_is_anonymous (iter_fd))
+                        continue;
+
+                /* flags for open are stripped down to allow following the
+                 * new location of the file, otherwise we can get EEXIST or
+                 * truncate the file again as rebalance is moving the data */
+                ret = syncop_open (dst_node, &tmp_loc,
+                                   (iter_fd->flags &
+                                   ~(O_CREAT | O_EXCL | O_TRUNC)), iter_fd);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to open "
+                                "the fd (%p, flags=0%o) on file %s @ %s",
+                                iter_fd, iter_fd->flags, path, dst_node->name);
+                        open_failed = 1;
+                        local->op_errno = -ret;
+                        ret = -1;
+                }
         }
+        GF_FREE (path);
+
+        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+
+        if (open_failed) {
+                ret = -1;
+                goto out;
+        }
+        ret = 0;
+out:
+
+        return ret;
+}
+
+int
+dht_rebalance_complete_check (xlator_t *this, call_frame_t *frame)
+{
+        int         ret     = -1;
+
+        ret = synctask_new (this->ctx->env, dht_migration_complete_check_task,
+                            dht_migration_complete_check_done,
+                            frame, frame);
+        return ret;
+}
+
+/* During 'in-progress' state, both nodes should have the file */
+static int
+dht_inprogress_check_done (int op_ret, call_frame_t *sync_frame, void *data)
+{
+        dht_local_t *local = NULL;
+
+        local = sync_frame->local;
+
+        local->rebalance.target_op_fn (THIS, sync_frame, op_ret);
 
         return 0;
-err:
-        loc_wipe (child);
-        return -1;
+}
+
+static int
+dht_rebalance_inprogress_task (void *data)
+{
+        int           ret      = -1;
+        xlator_t     *src_node = NULL;
+        xlator_t     *dst_node = NULL;
+        dht_local_t  *local    = NULL;
+        dict_t       *dict     = NULL;
+        call_frame_t *frame    = NULL;
+        xlator_t     *this     = NULL;
+        char         *path     = NULL;
+        struct iatt   stbuf    = {0,};
+        loc_t         tmp_loc  = {0,};
+        dht_conf_t   *conf     = NULL;
+        inode_t      *inode    = NULL;
+        fd_t         *iter_fd  = NULL;
+        int           open_failed = 0;
+
+        this  = THIS;
+        frame = data;
+        local = frame->local;
+        conf = this->private;
+
+        src_node = local->cached_subvol;
+
+        if (!local->loc.inode && !local->fd)
+                goto out;
+
+        inode = (!local->fd) ? local->loc.inode : local->fd->inode;
+
+        /* getxattr on cached_subvol for 'linkto' value. Do path based getxattr
+         * as root:root. If a fd is already open, access check wont be done*/
+        if (local->loc.inode) {
+                SYNCTASK_SETID (0, 0);
+                ret = syncop_getxattr (src_node, &local->loc, &dict,
+                                       conf->link_xattr_name);
+                SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+        } else {
+                ret = syncop_fgetxattr (src_node, local->fd, &dict,
+                                        conf->link_xattr_name);
+        }
+
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to get the 'linkto' xattr %s",
+                        local->loc.path, strerror (-ret));
+                ret = -1;
+                goto out;
+        }
+
+        dst_node = dht_linkfile_subvol (this, NULL, NULL, dict);
+        if (!dst_node) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to get the 'linkto' xattr from dict",
+                        local->loc.path);
+                ret = -1;
+                goto out;
+        }
+
+        local->rebalance.target_node = dst_node;
+
+        if (local->loc.inode) {
+                /* lookup on dst */
+                ret = syncop_lookup (dst_node, &local->loc, NULL,
+                                     &stbuf, NULL, NULL);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "%s: failed to lookup the file on %s",
+                                local->loc.path, dst_node->name);
+                        ret = -1;
+                        goto out;
+                }
+
+                if (uuid_compare (stbuf.ia_gfid, local->loc.inode->gfid)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "%s: gfid different on the target file on %s",
+                                local->loc.path, dst_node->name);
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = 0;
+
+        if (list_empty (&inode->fd_list))
+                goto done;
+
+        /* perform open as root:root. There is window between linkfile
+         * creation(root:root) and setattr with the correct uid/gid
+         */
+        SYNCTASK_SETID (0, 0);
+
+        tmp_loc.inode = inode;
+        inode_path (inode, NULL, &path);
+        if (path)
+                tmp_loc.path = path;
+
+        list_for_each_entry (iter_fd, &inode->fd_list, inode_list) {
+                if (fd_is_anonymous (iter_fd))
+                        continue;
+
+                /* flags for open are stripped down to allow following the
+                 * new location of the file, otherwise we can get EEXIST or
+                 * truncate the file again as rebalance is moving the data */
+                ret = syncop_open (dst_node, &tmp_loc,
+                                   (iter_fd->flags &
+                                   ~(O_CREAT | O_EXCL | O_TRUNC)), iter_fd);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to send open "
+                                "the fd (%p, flags=0%o) on file %s @ %s",
+                                iter_fd, iter_fd->flags, path, dst_node->name);
+                        ret = -1;
+                        open_failed = 1;
+                }
+        }
+        GF_FREE (path);
+
+        SYNCTASK_SETID (frame->root->uid, frame->root->gid);
+
+        if (open_failed) {
+                ret = -1;
+                goto out;
+        }
+
+done:
+        ret = dht_inode_ctx_set1 (this, inode, dst_node);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to set inode-ctx target file at %s",
+                        local->loc.path, dst_node->name);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+dht_rebalance_in_progress_check (xlator_t *this, call_frame_t *frame)
+{
+
+        int         ret     = -1;
+
+        ret = synctask_new (this->ctx->env, dht_rebalance_inprogress_task,
+                            dht_inprogress_check_done,
+                            frame, frame);
+        return ret;
+}
+
+int
+dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
+                          dht_layout_t *layout_int)
+{
+        dht_inode_ctx_t         *ctx            = NULL;
+        int                      ret            = -1;
+
+        ret = dht_inode_ctx_get (inode, this, &ctx);
+        if (!ret && ctx) {
+                ctx->layout = layout_int;
+        } else {
+                ctx = GF_CALLOC (1, sizeof (*ctx), gf_dht_mt_inode_ctx_t);
+                if (!ctx)
+                        return ret;
+                ctx->layout = layout_int;
+        }
+
+        ret = dht_inode_ctx_set (inode, this, ctx);
+
+        return ret;
+}
+
+
+void
+dht_inode_ctx_time_set (inode_t *inode, xlator_t *this, struct iatt *stat)
+{
+        dht_inode_ctx_t         *ctx            = NULL;
+        dht_stat_time_t         *time           = 0;
+        int                      ret            = -1;
+
+        ret = dht_inode_ctx_get (inode, this, &ctx);
+
+        if (ret)
+		return;
+
+        time = &ctx->time;
+
+	time->mtime      = stat->ia_mtime;
+	time->mtime_nsec = stat->ia_mtime_nsec;
+
+	time->ctime      = stat->ia_ctime;
+	time->ctime_nsec = stat->ia_ctime_nsec;
+
+	time->atime      = stat->ia_atime;
+	time->atime_nsec = stat->ia_atime_nsec;
+
+	return;
+}
+
+
+int
+dht_inode_ctx_time_update (inode_t *inode, xlator_t *this, struct iatt *stat,
+                           int32_t post)
+{
+        dht_inode_ctx_t         *ctx            = NULL;
+        dht_stat_time_t         *time           = 0;
+        int                      ret            = -1;
+
+        GF_VALIDATE_OR_GOTO (this->name, stat, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+
+        ret = dht_inode_ctx_get (inode, this, &ctx);
+
+        if (ret) {
+                ctx = GF_CALLOC (1, sizeof (*ctx), gf_dht_mt_inode_ctx_t);
+                if (!ctx)
+                        return -1;
+        }
+
+        time = &ctx->time;
+
+        DHT_UPDATE_TIME(time->mtime, time->mtime_nsec,
+                        stat->ia_mtime, stat->ia_mtime_nsec, inode, post);
+        DHT_UPDATE_TIME(time->ctime, time->ctime_nsec,
+                        stat->ia_ctime, stat->ia_ctime_nsec, inode, post);
+        DHT_UPDATE_TIME(time->atime, time->atime_nsec,
+                        stat->ia_atime, stat->ia_atime_nsec, inode, post);
+
+        ret = dht_inode_ctx_set (inode, this, ctx);
+out:
+        return 0;
+}
+
+int
+dht_inode_ctx_get (inode_t *inode, xlator_t *this, dht_inode_ctx_t **ctx)
+{
+        int             ret     = -1;
+        uint64_t        ctx_int = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+
+        ret = inode_ctx_get (inode, this, &ctx_int);
+
+        if (ret)
+                return ret;
+
+        if (ctx)
+                *ctx = (dht_inode_ctx_t *) ctx_int;
+out:
+        return ret;
+}
+
+int dht_inode_ctx_set (inode_t *inode, xlator_t *this, dht_inode_ctx_t *ctx)
+{
+        int             ret = -1;
+        uint64_t        ctx_int = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, inode, out);
+        GF_VALIDATE_OR_GOTO (this->name, ctx, out);
+
+        ctx_int = (long)ctx;
+        ret = inode_ctx_set (inode, this, &ctx_int);
+out:
+        return ret;
 }
diff --git a/xlators/cluster/dht/src/dht-inode-read.c b/xlators/cluster/dht/src/dht-inode-read.c
new file mode 100644
index 000000000..e8a9a7196
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-inode-read.c
@@ -0,0 +1,1139 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "dht-common.h"
+
+int dht_access2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_readv2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_attr2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_open2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_flush2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_lk2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_fsync2 (xlator_t *this, call_frame_t *frame, int ret);
+
+int
+dht_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+              int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           ret = 0;
+
+        local = frame->local;
+        prev = cookie;
+
+        local->op_errno = op_errno;
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+                goto out;
+        }
+
+        if (!op_ret || (local->call_cnt != 1))
+                goto out;
+
+        /* rebalance would have happened */
+        local->rebalance.target_op_fn = dht_open2;
+        ret = dht_rebalance_complete_check (this, frame);
+        if (!ret)
+                return 0;
+
+out:
+        DHT_STACK_UNWIND (open, frame, op_ret, op_errno, local->fd, xdata);
+
+        return 0;
+}
+
+int
+dht_open2 (xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t *local  = NULL;
+        xlator_t    *subvol = NULL;
+        int          op_errno = EINVAL;
+
+        local = frame->local;
+        if (!local)
+                goto out;
+
+        op_errno = ENOENT;
+        if (op_ret)
+                goto out;
+
+        local->call_cnt = 2;
+        subvol = local->cached_subvol;
+
+        STACK_WIND (frame, dht_open_cbk, subvol, subvol->fops->open,
+                    &local->loc, local->rebalance.flags, local->fd,
+                    NULL);
+        return 0;
+
+out:
+        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
+
+int
+dht_open (call_frame_t *frame, xlator_t *this,
+          loc_t *loc, int flags, fd_t *fd, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, loc, fd, GF_FOP_OPEN);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        local->rebalance.flags = flags;
+        local->call_cnt = 1;
+
+        STACK_WIND (frame, dht_open_cbk, subvol, subvol->fops->open,
+                    loc, flags, fd, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
+
+        return 0;
+}
+
+int
+dht_file_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
+{
+        xlator_t     *subvol = 0;
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           ret = -1;
+        inode_t      *inode = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+                local->op_errno = op_errno;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+                goto out;
+        }
+
+        if (local->call_cnt != 1)
+                goto out;
+
+        local->op_errno = op_errno;
+        /* Check if the rebalance phase2 is true */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
+                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+                ret = dht_inode_ctx_get1 (this, inode, &subvol);
+                if (!subvol) {
+                        /* Phase 2 of migration */
+                        local->rebalance.target_op_fn = dht_attr2;
+                        ret = dht_rebalance_complete_check (this, frame);
+                        if (!ret)
+                                return 0;
+                } else {
+                        /* value is already set in fd_ctx, that means no need
+                           to check for whether its complete or not. */
+                        dht_attr2 (this, frame, 0);
+                        return 0;
+                }
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
+        DHT_STACK_UNWIND (stat, frame, op_ret, op_errno, stbuf, xdata);
+err:
+        return 0;
+}
+
+int
+dht_attr2 (xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t *local  = NULL;
+        xlator_t    *subvol = NULL;
+        int          op_errno = EINVAL;
+
+        local = frame->local;
+        if (!local)
+                goto out;
+
+        op_errno = local->op_errno;
+        if (op_ret == -1)
+                goto out;
+
+        subvol = local->cached_subvol;
+        local->call_cnt = 2;
+
+        if (local->fop == GF_FOP_FSTAT) {
+                STACK_WIND (frame, dht_file_attr_cbk, subvol,
+                            subvol->fops->fstat, local->fd, NULL);
+        } else {
+                STACK_WIND (frame, dht_file_attr_cbk, subvol,
+                            subvol->fops->stat, &local->loc, NULL);
+        }
+        return 0;
+
+out:
+        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
+
+int
+dht_attr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+              int op_ret, int op_errno, struct iatt *stbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        call_frame_t *prev = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "subvolume %s returned -1 (%s)",
+                                prev->this->name, strerror (op_errno));
+
+                        goto unlock;
+                }
+
+                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+
+                local->op_ret = 0;
+        }
+unlock:
+        UNLOCK (&frame->lock);
+out:
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+                DHT_STACK_UNWIND (stat, frame, local->op_ret, local->op_errno,
+                                  &local->stbuf, xdata);
+        }
+err:
+        return 0;
+}
+
+int
+dht_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+        dht_layout_t *layout = NULL;
+        int           i = 0;
+        int           call_cnt = 0;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+
+
+        local = dht_local_init (frame, loc, NULL, GF_FOP_STAT);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        layout = local->layout;
+        if (!layout) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no layout for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        if (IA_ISREG (loc->inode->ia_type)) {
+                local->call_cnt = 1;
+
+                subvol = local->cached_subvol;
+
+                STACK_WIND (frame, dht_file_attr_cbk, subvol,
+                            subvol->fops->stat, loc, xdata);
+
+                return 0;
+        }
+
+        local->call_cnt = call_cnt = layout->cnt;
+
+        for (i = 0; i < call_cnt; i++) {
+                subvol = layout->list[i].xlator;
+
+                STACK_WIND (frame, dht_attr_cbk,
+                            subvol, subvol->fops->stat,
+                            loc, xdata);
+        }
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+dht_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+        dht_layout_t *layout = NULL;
+        int           i = 0;
+        int           call_cnt = 0;
+
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FSTAT);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        layout = local->layout;
+        if (!layout) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no layout for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        if (IA_ISREG (fd->inode->ia_type)) {
+                local->call_cnt = 1;
+
+                subvol = local->cached_subvol;
+
+                STACK_WIND (frame, dht_file_attr_cbk, subvol,
+                            subvol->fops->fstat, fd, xdata);
+
+                return 0;
+        }
+
+        local->call_cnt = call_cnt = layout->cnt;
+
+        for (i = 0; i < call_cnt; i++) {
+                subvol = layout->list[i].xlator;
+                STACK_WIND (frame, dht_attr_cbk,
+                            subvol, subvol->fops->fstat,
+                            fd, xdata);
+        }
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
+
+        return 0;
+}
+
+int
+dht_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int op_ret, int op_errno,
+               struct iovec *vector, int count, struct iatt *stbuf,
+               struct iobref *iobref, dict_t *xdata)
+{
+        dht_local_t *local      = NULL;
+        int          ret        = 0;
+        inode_t     *inode      = NULL;
+        xlator_t    *subvol = 0;
+
+        local = frame->local;
+        if (!local) {
+                op_ret = -1;
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        /* This is already second try, no need for re-check */
+        if (local->call_cnt != 1)
+                goto out;
+
+        if ((op_ret == -1) && !dht_inode_missing(op_errno))
+                goto out;
+
+        local->op_errno = op_errno;
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (stbuf)) {
+                /* File would be migrated to other node */
+                ret = dht_inode_ctx_get1 (this, inode, &subvol);
+                if (!subvol) {
+                        local->rebalance.target_op_fn = dht_readv2;
+                        ret = dht_rebalance_complete_check (this, frame);
+                        if (!ret)
+                                return 0;
+                } else {
+                        /* value is already set in fd_ctx, that means no need
+                           to check for whether its complete or not. */
+                        dht_readv2 (this, frame, 0);
+                        return 0;
+                }
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
+        DHT_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count, stbuf,
+                          iobref, xdata);
+
+        return 0;
+}
+
+int
+dht_readv2 (xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t *local  = NULL;
+        xlator_t    *subvol = NULL;
+        int          op_errno = EINVAL;
+
+        local = frame->local;
+        if (!local)
+                goto out;
+
+        op_errno = local->op_errno;
+        if (op_ret == -1)
+                goto out;
+
+        local->call_cnt = 2;
+        subvol = local->cached_subvol;
+
+        STACK_WIND (frame, dht_readv_cbk, subvol, subvol->fops->readv,
+                    local->fd, local->rebalance.size, local->rebalance.offset,
+                    local->rebalance.flags, NULL);
+
+        return 0;
+
+out:
+        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
+        return 0;
+}
+
+int
+dht_readv (call_frame_t *frame, xlator_t *this,
+           fd_t *fd, size_t size, off_t off, uint32_t flags, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_READ);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        local->rebalance.offset = off;
+        local->rebalance.size   = size;
+        local->rebalance.flags  = flags;
+        local->call_cnt = 1;
+
+        STACK_WIND (frame, dht_readv_cbk,
+                    subvol, subvol->fops->readv,
+                    fd, size, off, flags, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+dht_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, dict_t *xdata)
+{
+        int          ret = -1;
+        dht_local_t *local = NULL;
+        xlator_t    *subvol = NULL;
+        call_frame_t *prev = NULL;
+
+        local = frame->local;
+        prev = cookie;
+
+        if (!prev || !prev->this)
+                goto out;
+        if (local->call_cnt != 1)
+                goto out;
+        if ((op_ret == -1) && (op_errno == ENOTCONN) &&
+            IA_ISDIR(local->loc.inode->ia_type)) {
+
+                subvol = dht_subvol_next_available (this, prev->this);
+                if (!subvol)
+                        goto out;
+
+                /* check if we are done with visiting every node */
+                if (subvol == local->cached_subvol) {
+                        goto out;
+                }
+
+                STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
+                            &local->loc, local->rebalance.flags, NULL);
+                return 0;
+        }
+        if ((op_ret == -1) && dht_inode_missing(op_errno)) {
+                /* File would be migrated to other node */
+                local->op_errno = op_errno;
+                local->rebalance.target_op_fn = dht_access2;
+                ret = dht_rebalance_complete_check (frame->this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int
+dht_access2 (xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t *local  = NULL;
+        xlator_t    *subvol = NULL;
+        int          op_errno = EINVAL;
+
+        local = frame->local;
+        if (!local)
+                goto out;
+
+        op_errno = local->op_errno;
+        if (op_ret == -1)
+                goto out;
+
+        local->call_cnt = 2;
+        subvol = local->cached_subvol;
+
+        STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
+                    &local->loc, local->rebalance.flags, NULL);
+
+        return 0;
+
+out:
+        DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
+        return 0;
+}
+
+
+int
+dht_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+            dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+
+        local = dht_local_init (frame, loc, NULL, GF_FOP_ACCESS);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->rebalance.flags = mask;
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_access_cbk, subvol, subvol->fops->access,
+                    loc, mask, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (access, frame, -1, op_errno, NULL);
+
+        return 0;
+}
+
+
+int
+dht_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int op_ret, int op_errno, dict_t *xdata)
+{
+        dht_local_t  *local  = NULL;
+        inode_t      *inode  = NULL;
+        xlator_t     *subvol = 0;
+
+        local = frame->local;
+
+        local->op_errno = op_errno;
+
+        if (local->call_cnt != 1)
+                goto out;
+
+        /* If context is set, then send flush() it to the destination */
+        dht_inode_ctx_get1 (this, inode, &subvol);
+        if (subvol) {
+                dht_flush2 (this, frame, 0);
+                return 0;
+        }
+
+out:
+        DHT_STACK_UNWIND (flush, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int
+dht_flush2 (xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+
+        local = frame->local;
+
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+        STACK_WIND (frame, dht_flush_cbk,
+                    subvol, subvol->fops->flush, local->fd, NULL);
+
+        return 0;
+}
+
+
+int
+dht_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FLUSH);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        local->call_cnt = 1;
+
+        STACK_WIND (frame, dht_flush_cbk,
+                    subvol, subvol->fops->flush, fd, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
+
+        return 0;
+}
+
+
+int
+dht_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this, int op_ret,
+               int op_errno, struct iatt *prebuf, struct iatt *postbuf,
+               dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           ret = -1;
+        inode_t      *inode = NULL;
+        xlator_t     *subvol = 0;
+
+        local = frame->local;
+        prev = cookie;
+
+        local->op_errno = op_errno;
+        if (op_ret == -1 && !dht_inode_missing(op_errno)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+
+        local->op_errno = op_errno;
+        dht_inode_ctx_get1 (this, inode, &subvol);
+        if (!subvol) {
+                local->rebalance.target_op_fn = dht_fsync2;
+
+                /* Check if the rebalance phase1 is true */
+                if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                        dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                        dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+
+                        ret = dht_rebalance_in_progress_check (this, frame);
+                }
+
+                /* Check if the rebalance phase2 is true */
+                if (IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                        ret = dht_rebalance_complete_check (this, frame);
+                }
+                if (!ret)
+                        return 0;
+        } else {
+                dht_fsync2 (this, frame, 0);
+                return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (fsync, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+
+        return 0;
+}
+
+int
+dht_fsync2 (xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+
+        local = frame->local;
+
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+        STACK_WIND (frame, dht_fsync_cbk, subvol, subvol->fops->fsync,
+                    local->fd, local->rebalance.flags, NULL);
+
+        return 0;
+}
+
+int
+dht_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int datasync,
+           dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FSYNC);
+        if (!local) {
+                op_errno = ENOMEM;
+
+                goto err;
+        }
+
+        local->call_cnt = 1;
+        local->rebalance.flags = datasync;
+
+        subvol = local->cached_subvol;
+
+        STACK_WIND (frame, dht_fsync_cbk, subvol, subvol->fops->fsync,
+                    fd, datasync, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+/* TODO: for 'lk()' call, we need some other special error, may be ESTALE to
+   indicate that lock migration happened on the fd, so we can consider it as
+   phase 2 of migration */
+int
+dht_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+            int op_ret, int op_errno, struct gf_flock *flock, dict_t *xdata)
+{
+        DHT_STACK_UNWIND (lk, frame, op_ret, op_errno, flock, xdata);
+
+        return 0;
+}
+
+
+int
+dht_lk (call_frame_t *frame, xlator_t *this,
+        fd_t *fd, int cmd, struct gf_flock *flock, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        subvol = dht_subvol_get_cached (this, fd->inode);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        /* TODO: for rebalance, we need to preserve the fop arguments */
+        STACK_WIND (frame, dht_lk_cbk, subvol, subvol->fops->lk, fd,
+                    cmd, flock, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
+
+        return 0;
+}
+
+/* Symlinks are currently not migrated, so no need for any check here */
+int
+dht_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int op_ret, int op_errno, const char *path,
+                  struct iatt *stbuf, dict_t *xdata)
+{
+        dht_local_t *local = NULL;
+
+        local = frame->local;
+        if (op_ret == -1)
+                goto err;
+
+        if (!local) {
+                op_ret = -1;
+                op_errno = EINVAL;
+        }
+
+err:
+        DHT_STRIP_PHASE1_FLAGS (stbuf);
+        DHT_STACK_UNWIND (readlink, frame, op_ret, op_errno, path, stbuf, xdata);
+
+        return 0;
+}
+
+
+int
+dht_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+              dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+
+        local = dht_local_init (frame, loc, NULL, GF_FOP_READLINK);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_readlink_cbk,
+                    subvol, subvol->fops->readlink,
+                    loc, size, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (readlink, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+/* Currently no translators on top of 'distribute' will be using
+ * below fops, hence not implementing 'migration' related checks
+ */
+
+int
+dht_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+        DHT_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict, xdata);
+        return 0;
+}
+
+
+int
+dht_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+
+        local = dht_local_init (frame, loc, NULL, GF_FOP_XATTROP);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        local->call_cnt = 1;
+
+        STACK_WIND (frame,
+                    dht_xattrop_cbk,
+                    subvol, subvol->fops->xattrop,
+                    loc, flags, dict, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (xattrop, frame, -1, op_errno, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+dht_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+        DHT_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict, xdata);
+        return 0;
+}
+
+
+int
+dht_fxattrop (call_frame_t *frame, xlator_t *this,
+              fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        subvol = dht_subvol_get_cached (this, fd->inode);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame,
+                    dht_fxattrop_cbk,
+                    subvol, subvol->fops->fxattrop,
+                    fd, flags, dict, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fxattrop, frame, -1, op_errno, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+dht_inodelk_cbk (call_frame_t *frame, void *cookie,
+                 xlator_t *this, int32_t op_ret, int32_t op_errno, dict_t *xdata)
+
+{
+        DHT_STACK_UNWIND (inodelk, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+
+int32_t
+dht_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+             loc_t *loc, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+
+        local = dht_local_init (frame, loc, NULL, GF_FOP_INODELK);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        local->call_cnt = 1;
+
+        STACK_WIND (frame,
+                    dht_inodelk_cbk,
+                    subvol, subvol->fops->inodelk,
+                    volume, loc, cmd, lock, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (inodelk, frame, -1, op_errno, NULL);
+
+        return 0;
+}
+
+
+int
+dht_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+
+{
+        DHT_STACK_UNWIND (finodelk, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+
+int
+dht_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+              fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        subvol = dht_subvol_get_cached (this, fd->inode);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+
+        STACK_WIND (frame, dht_finodelk_cbk, subvol, subvol->fops->finodelk,
+                    volume, fd, cmd, lock, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (finodelk, frame, -1, op_errno, NULL);
+
+        return 0;
+}
diff --git a/xlators/cluster/dht/src/dht-inode-write.c b/xlators/cluster/dht/src/dht-inode-write.c
new file mode 100644
index 000000000..576f007e5
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-inode-write.c
@@ -0,0 +1,1017 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "dht-common.h"
+
+int dht_writev2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_truncate2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_setattr2 (xlator_t *this, call_frame_t *frame, int ret);
+int dht_fallocate2(xlator_t *this, call_frame_t *frame, int op_ret);
+int dht_discard2(xlator_t *this, call_frame_t *frame, int op_ret);
+int dht_zerofill2(xlator_t *this, call_frame_t *frame, int op_ret);
+
+int
+dht_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t *local = NULL;
+        int          ret   = -1;
+        xlator_t    *subvol = NULL;
+
+        if (op_ret == -1 && !dht_inode_missing(op_errno)) {
+                goto out;
+        }
+
+        local = frame->local;
+        if (!local) {
+                op_ret = -1;
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                /* preserve the modes of source */
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+
+        local->rebalance.target_op_fn = dht_writev2;
+
+        local->op_errno = op_errno;
+        /* Phase 2 of migration */
+        if (IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Check if the rebalance phase1 is true */
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+
+                ret = dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+                if (subvol) {
+                        dht_writev2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+
+        DHT_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                          xdata);
+
+        return 0;
+}
+
+int
+dht_writev2 (xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+
+        local = frame->local;
+
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+        STACK_WIND (frame, dht_writev_cbk,
+                    subvol, subvol->fops->writev,
+                    local->fd, local->rebalance.vector, local->rebalance.count,
+                    local->rebalance.offset, local->rebalance.flags,
+                    local->rebalance.iobref, NULL);
+
+        return 0;
+}
+
+int
+dht_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
+            struct iovec *vector, int count, off_t off, uint32_t flags,
+            struct iobref *iobref, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_WRITE);
+        if (!local) {
+
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+
+        local->rebalance.vector = iov_dup (vector, count);
+        local->rebalance.offset = off;
+        local->rebalance.count = count;
+        local->rebalance.flags = flags;
+        local->rebalance.iobref = iobref_ref (iobref);
+        local->call_cnt = 1;
+
+        STACK_WIND (frame, dht_writev_cbk,
+                    subvol, subvol->fops->writev,
+                    fd, vector, count, off, flags, iobref, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+
+int
+dht_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int op_ret, int op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           ret = -1;
+        xlator_t    *subvol = NULL;
+        inode_t      *inode = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+
+        local->rebalance.target_op_fn = dht_truncate2;
+
+        local->op_errno = op_errno;
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Check if the rebalance phase1 is true */
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+                inode = (local->fd) ? local->fd->inode : local->loc.inode;
+                dht_inode_ctx_get1 (this, inode, &subvol);
+                if (subvol) {
+                        dht_truncate2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (truncate, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+err:
+        return 0;
+}
+
+
+int
+dht_truncate2 (xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+        inode_t      *inode = NULL;
+
+        local = frame->local;
+
+        inode = local->fd ? local->fd->inode : local->loc.inode;
+
+        dht_inode_ctx_get1 (this, inode, &subvol);
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+        if (local->fop == GF_FOP_TRUNCATE) {
+                STACK_WIND (frame, dht_truncate_cbk, subvol,
+                            subvol->fops->truncate, &local->loc,
+                            local->rebalance.offset, NULL);
+        } else {
+                STACK_WIND (frame, dht_truncate_cbk, subvol,
+                            subvol->fops->ftruncate, local->fd,
+                            local->rebalance.offset, NULL);
+        }
+
+        return 0;
+}
+
+int
+dht_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset,
+              dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+
+        local = dht_local_init (frame, loc, NULL, GF_FOP_TRUNCATE);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->rebalance.offset = offset;
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_truncate_cbk,
+                    subvol, subvol->fops->truncate,
+                    loc, offset, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+dht_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FTRUNCATE);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->rebalance.offset = offset;
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_truncate_cbk,
+                    subvol, subvol->fops->ftruncate,
+                    fd, offset, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+dht_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                  int op_ret, int op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           ret = -1;
+        xlator_t    *subvol = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+        local->rebalance.target_op_fn = dht_fallocate2;
+
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Check if the rebalance phase1 is true */
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+                dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+                if (subvol) {
+                        dht_fallocate2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (fallocate, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+err:
+        return 0;
+}
+
+int
+dht_fallocate2(xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+
+        local = frame->local;
+
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+	STACK_WIND(frame, dht_fallocate_cbk, subvol, subvol->fops->fallocate,
+		   local->fd, local->rebalance.flags, local->rebalance.offset,
+		   local->rebalance.size, NULL);
+
+        return 0;
+}
+
+int
+dht_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+	      off_t offset, size_t len, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FALLOCATE);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+	local->rebalance.flags = mode;
+        local->rebalance.offset = offset;
+	local->rebalance.size = len;
+
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_fallocate_cbk,
+                    subvol, subvol->fops->fallocate,
+                    fd, mode, offset, len, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+dht_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           ret = -1;
+        xlator_t    *subvol = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+        local->rebalance.target_op_fn = dht_discard2;
+
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Check if the rebalance phase1 is true */
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+                dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+                if (subvol) {
+                        dht_discard2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (discard, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+err:
+        return 0;
+}
+
+int
+dht_discard2(xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+
+        local = frame->local;
+
+        dht_inode_ctx_get1 (this, local->fd->inode, &subvol);
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+	STACK_WIND(frame, dht_discard_cbk, subvol, subvol->fops->discard,
+		   local->fd, local->rebalance.offset, local->rebalance.size,
+		   NULL);
+
+        return 0;
+}
+
+int
+dht_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	    size_t len, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        int           op_errno = -1;
+        dht_local_t  *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_DISCARD);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->rebalance.offset = offset;
+	local->rebalance.size = len;
+
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_discard_cbk, subvol, subvol->fops->discard,
+                    fd, offset, len, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+int
+dht_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                int op_ret, int op_errno, struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev  = NULL;
+        int           ret   = -1;
+
+        GF_VALIDATE_OR_GOTO ("dht", frame, err);
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", frame->local, out);
+        GF_VALIDATE_OR_GOTO ("dht", cookie, out);
+
+        local = frame->local;
+        prev = cookie;
+
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+                local->op_errno = op_errno;
+                local->op_ret = -1;
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+                goto out;
+        }
+
+        if (local->call_cnt != 1) {
+                if (local->stbuf.ia_blocks) {
+                        dht_iatt_merge (this, postbuf, &local->stbuf, NULL);
+                        dht_iatt_merge (this, prebuf, &local->prebuf, NULL);
+                }
+                goto out;
+        }
+        local->rebalance.target_op_fn = dht_zerofill2;
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* Check if the rebalance phase1 is true */
+        if (IS_DHT_MIGRATION_PHASE1 (postbuf)) {
+                dht_iatt_merge (this, &local->stbuf, postbuf, NULL);
+                dht_iatt_merge (this, &local->prebuf, prebuf, NULL);
+                ret = fd_ctx_get (local->fd, this, NULL);
+                if (!ret) {
+                        dht_zerofill2 (this, frame, 0);
+                        return 0;
+                }
+                ret = dht_rebalance_in_progress_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (zerofill, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+err:
+        return 0;
+}
+
+int
+dht_zerofill2(xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local          = NULL;
+        xlator_t     *subvol         = NULL;
+        uint64_t      tmp_subvol     = 0;
+        int           ret            = -1;
+
+        local = frame->local;
+
+        if (local->fd)
+                ret = fd_ctx_get (local->fd, this, &tmp_subvol);
+        if (!ret)
+                subvol = (xlator_t *)(long)tmp_subvol;
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+        STACK_WIND(frame, dht_zerofill_cbk, subvol, subvol->fops->zerofill,
+                   local->fd, local->rebalance.offset, local->rebalance.size,
+                   NULL);
+
+        return 0;
+}
+
+int
+dht_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+            off_t len, dict_t *xdata)
+{
+        xlator_t     *subvol       = NULL;
+        int           op_errno     = -1;
+        dht_local_t  *local        = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_ZEROFILL);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->rebalance.offset = offset;
+        local->rebalance.size = len;
+
+        local->call_cnt = 1;
+        subvol = local->cached_subvol;
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no cached subvolume for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        STACK_WIND (frame, dht_zerofill_cbk, subvol, subvol->fops->zerofill,
+                    fd, offset, len, xdata);
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+
+/* handle cases of migration here for 'setattr()' calls */
+int
+dht_file_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int op_ret, int op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           ret = -1;
+
+        local = frame->local;
+        prev = cookie;
+
+        local->op_errno = op_errno;
+        if ((op_ret == -1) && !dht_inode_missing(op_errno)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "subvolume %s returned -1 (%s)",
+                        prev->this->name, strerror (op_errno));
+                goto out;
+        }
+
+        if (local->call_cnt != 1)
+                goto out;
+
+        local->rebalance.target_op_fn = dht_setattr2;
+
+        /* Phase 2 of migration */
+        if ((op_ret == -1) || IS_DHT_MIGRATION_PHASE2 (postbuf)) {
+                ret = dht_rebalance_complete_check (this, frame);
+                if (!ret)
+                        return 0;
+        }
+
+        /* At the end of the migration process, whatever 'attr' we
+           have on source file will be migrated to destination file
+           in one shot, hence we don't need to check for in progress
+           state here (ie, PHASE1) */
+out:
+        DHT_STRIP_PHASE1_FLAGS (postbuf);
+        DHT_STRIP_PHASE1_FLAGS (prebuf);
+        DHT_STACK_UNWIND (setattr, frame, op_ret, op_errno,
+                          prebuf, postbuf, xdata);
+
+        return 0;
+}
+
+int
+dht_setattr2 (xlator_t *this, call_frame_t *frame, int op_ret)
+{
+        dht_local_t  *local  = NULL;
+        xlator_t     *subvol = NULL;
+        inode_t      *inode = NULL;
+
+        local = frame->local;
+
+        inode = (local->fd) ? local->fd->inode : local->loc.inode;
+
+        dht_inode_ctx_get1 (this, inode, &subvol);
+
+        if (!subvol)
+                subvol = local->cached_subvol;
+
+        local->call_cnt = 2; /* This is the second attempt */
+
+        if (local->fop == GF_FOP_SETATTR) {
+                STACK_WIND (frame, dht_file_setattr_cbk, subvol,
+                            subvol->fops->setattr, &local->loc,
+                            &local->rebalance.stbuf, local->rebalance.flags,
+                            NULL);
+        } else {
+                STACK_WIND (frame, dht_file_setattr_cbk, subvol,
+                            subvol->fops->fsetattr, local->fd,
+                            &local->rebalance.stbuf, local->rebalance.flags,
+                            NULL);
+        }
+
+        return 0;
+}
+
+
+/* Keep the existing code same for all the cases other than regular file */
+int
+dht_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int op_ret, int op_errno, struct iatt *statpre,
+                 struct iatt *statpost, dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        call_frame_t *prev = NULL;
+
+
+        local = frame->local;
+        prev = cookie;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "subvolume %s returned -1 (%s)",
+                                prev->this->name, strerror (op_errno));
+                        goto unlock;
+                }
+
+                dht_iatt_merge (this, &local->prebuf, statpre, prev->this);
+                dht_iatt_merge (this, &local->stbuf, statpost, prev->this);
+
+                local->op_ret = 0;
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+		if (local->op_ret == 0)
+			dht_inode_ctx_time_set (local->loc.inode, this,
+						&local->stbuf);
+                DHT_STACK_UNWIND (setattr, frame, local->op_ret, local->op_errno,
+                                  &local->prebuf, &local->stbuf, xdata);
+	}
+
+        return 0;
+}
+
+
+int
+dht_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        dht_layout_t *layout = NULL;
+        dht_local_t  *local  = NULL;
+        int           op_errno = -1;
+        int           i = -1;
+        int           call_cnt = 0;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+
+        local = dht_local_init (frame, loc, NULL, GF_FOP_SETATTR);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        layout = local->layout;
+        if (!layout) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no layout for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        if (!layout_is_sane (layout)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "layout is not sane for path=%s", loc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        if (IA_ISREG (loc->inode->ia_type)) {
+                /* in the regular file _cbk(), we need to check for
+                   migration possibilities */
+                local->rebalance.stbuf = *stbuf;
+                local->rebalance.flags = valid;
+                local->call_cnt = 1;
+                subvol = local->cached_subvol;
+
+                STACK_WIND (frame, dht_file_setattr_cbk, subvol,
+                            subvol->fops->setattr,
+                            loc, stbuf, valid, xdata);
+
+                return 0;
+        }
+
+        local->call_cnt = call_cnt = layout->cnt;
+
+        for (i = 0; i < call_cnt; i++) {
+                STACK_WIND (frame, dht_setattr_cbk,
+                            layout->list[i].xlator,
+                            layout->list[i].xlator->fops->setattr,
+                            loc, stbuf, valid, xdata);
+        }
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
+
+
+int
+dht_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *stbuf,
+              int32_t valid, dict_t *xdata)
+{
+        xlator_t     *subvol = NULL;
+        dht_layout_t *layout = NULL;
+        dht_local_t  *local  = NULL;
+        int           op_errno = -1;
+        int           i = -1;
+        int           call_cnt = 0;
+
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        local = dht_local_init (frame, NULL, fd, GF_FOP_FSETATTR);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        layout = local->layout;
+        if (!layout) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no layout for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        if (!layout_is_sane (layout)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "layout is not sane for fd=%p", fd);
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        if (IA_ISREG (fd->inode->ia_type)) {
+                /* in the regular file _cbk(), we need to check for
+                   migration possibilities */
+                local->rebalance.stbuf = *stbuf;
+                local->rebalance.flags = valid;
+                local->call_cnt = 1;
+                subvol = local->cached_subvol;
+
+                STACK_WIND (frame, dht_file_setattr_cbk, subvol,
+                            subvol->fops->fsetattr,
+                            fd, stbuf, valid, xdata);
+
+                return 0;
+        }
+
+        local->call_cnt = call_cnt = layout->cnt;
+
+        for (i = 0; i < call_cnt; i++) {
+                STACK_WIND (frame, dht_setattr_cbk,
+                            layout->list[i].xlator,
+                            layout->list[i].xlator->fops->fsetattr,
+                            fd, stbuf, valid, xdata);
+        }
+
+        return 0;
+
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
+
+        return 0;
+}
diff --git a/xlators/cluster/dht/src/dht-layout.c b/xlators/cluster/dht/src/dht-layout.c
index a11b78f8d..e1a37b77c 100644
--- a/xlators/cluster/dht/src/dht-layout.c
+++ b/xlators/cluster/dht/src/dht-layout.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -34,32 +25,32 @@
 
 #define layout_size(cnt) (layout_base_size + (cnt * layout_entry_size))
 
-
 dht_layout_t *
 dht_layout_new (xlator_t *this, int cnt)
 {
-	dht_layout_t *layout = NULL;
+        dht_layout_t *layout = NULL;
         dht_conf_t   *conf = NULL;
 
-
         conf = this->private;
 
-	layout = GF_CALLOC (1, layout_size (cnt),
+        layout = GF_CALLOC (1, layout_size (cnt),
                             gf_dht_mt_dht_layout_t);
-	if (!layout) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto out;
-	}
+        if (!layout) {
+                goto out;
+        }
 
         layout->type = DHT_HASH_TYPE_DM;
-	layout->cnt = cnt;
-        if (conf)
+        layout->cnt = cnt;
+
+        if (conf) {
+                layout->spread_cnt = conf->dir_spread_cnt;
                 layout->gen = conf->gen;
+        }
 
         layout->ref = 1;
+
 out:
-	return layout;
+        return layout;
 }
 
 
@@ -67,9 +58,7 @@ dht_layout_t *
 dht_layout_get (xlator_t *this, inode_t *inode)
 {
         dht_conf_t   *conf = NULL;
-        uint64_t      layout_int = 0;
         dht_layout_t *layout = NULL;
-        int           ret    = -1;
 
         conf = this->private;
         if (!conf)
@@ -77,9 +66,8 @@ dht_layout_get (xlator_t *this, inode_t *inode)
 
         LOCK (&conf->layout_lock);
         {
-                ret = inode_ctx_get (inode, this, &layout_int);
-                if (ret == 0) {
-                        layout = (dht_layout_t *) (unsigned long) layout_int;
+                dht_inode_ctx_layout_get (inode, this, &layout);
+                if (layout) {
                         layout->ref++;
                 }
         }
@@ -97,7 +85,6 @@ dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout)
         int           oldret = -1;
         int           ret = 0;
         dht_layout_t *old_layout;
-        uint64_t      old_layout_int;
 
         conf = this->private;
         if (!conf)
@@ -105,16 +92,13 @@ dht_layout_set (xlator_t *this, inode_t *inode, dht_layout_t *layout)
 
         LOCK (&conf->layout_lock);
         {
-                oldret = inode_ctx_get (inode, this, &old_layout_int);
-
+                oldret = dht_inode_ctx_layout_get (inode, this, &old_layout);
                 layout->ref++;
-                ret = inode_ctx_put (inode, this, (uint64_t) (unsigned long)
-                                     layout);
+                dht_inode_ctx_layout_set (inode, this, layout);
         }
         UNLOCK (&conf->layout_lock);
 
-        if (oldret == 0) {
-                old_layout = (dht_layout_t *) (unsigned long) old_layout_int;
+        if (!oldret) {
                 dht_layout_unref (this, old_layout);
         }
 
@@ -129,7 +113,7 @@ dht_layout_unref (xlator_t *this, dht_layout_t *layout)
         dht_conf_t  *conf = NULL;
         int          ref = 0;
 
-        if (layout->preset || !this->private)
+        if (!layout || layout->preset || !this->private)
                 return;
 
         conf = this->private;
@@ -167,532 +151,626 @@ dht_layout_ref (xlator_t *this, dht_layout_t *layout)
 xlator_t *
 dht_layout_search (xlator_t *this, dht_layout_t *layout, const char *name)
 {
-	uint32_t   hash = 0;
+        uint32_t   hash = 0;
         xlator_t  *subvol = NULL;
-	int        i = 0;
-	int        ret = 0;
+        int        i = 0;
+        int        ret = 0;
 
 
-	ret = dht_hash_compute (layout->type, name, &hash);
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"hash computation failed for type=%d name=%s",
-			layout->type, name);
-		goto out;
-	}
+        ret = dht_hash_compute (this, layout->type, name, &hash);
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "hash computation failed for type=%d name=%s",
+                        layout->type, name);
+                goto out;
+        }
 
-	for (i = 0; i < layout->cnt; i++) {
-		if (layout->list[i].start <= hash
-		    && layout->list[i].stop >= hash) {
-			subvol = layout->list[i].xlator;
-			break;
-		}
-	}
+        for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].start <= hash
+                    && layout->list[i].stop >= hash) {
+                        subvol = layout->list[i].xlator;
+                        break;
+                }
+        }
 
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no subvolume for hash (value) = %u", hash);
-	}
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "no subvolume for hash (value) = %u", hash);
+        }
 
 out:
-	return subvol;
+        return subvol;
 }
 
 
 dht_layout_t *
 dht_layout_for_subvol (xlator_t *this, xlator_t *subvol)
 {
-	dht_conf_t   *conf = NULL;
-	dht_layout_t *layout = NULL;
-	int           i = 0;
+        dht_conf_t   *conf = NULL;
+        dht_layout_t *layout = NULL;
+        int           i = 0;
 
-	conf = this->private;
+        conf = this->private;
         if (!conf)
                 goto out;
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		if (conf->subvolumes[i] == subvol) {
-			layout = conf->file_layouts[i];
-			break;
-		}
-	}
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (conf->subvolumes[i] == subvol) {
+                        layout = conf->file_layouts[i];
+                        break;
+                }
+        }
 
 out:
-	return layout;
+        return layout;
 }
 
 
 int
 dht_layouts_init (xlator_t *this, dht_conf_t *conf)
 {
-	dht_layout_t *layout = NULL;
-	int           i = 0;
-	int           ret = -1;
+        dht_layout_t *layout = NULL;
+        int           i = 0;
+        int           ret = -1;
 
         if (!conf)
                 goto out;
 
-	conf->file_layouts = GF_CALLOC (conf->subvolume_cnt,
-				        sizeof (dht_layout_t *),
+        conf->file_layouts = GF_CALLOC (conf->subvolume_cnt,
+                                        sizeof (dht_layout_t *),
                                         gf_dht_mt_dht_layout_t);
-	if (!conf->file_layouts) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto out;
-	}
+        if (!conf->file_layouts) {
+                goto out;
+        }
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		layout = dht_layout_new (this, 1);
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                layout = dht_layout_new (this, 1);
 
-		if (!layout) {
-			goto out;
-		}
+                if (!layout) {
+                        goto out;
+                }
 
-		layout->preset = 1;
+                layout->preset = 1;
 
-		layout->list[0].xlator = conf->subvolumes[i];
+                layout->list[0].xlator = conf->subvolumes[i];
 
-		conf->file_layouts[i] = layout;
-	}
+                conf->file_layouts[i] = layout;
+        }
 
-	ret = 0;
+        ret = 0;
 out:
-	return ret;
+        return ret;
 }
 
 
 int
 dht_disk_layout_extract (xlator_t *this, dht_layout_t *layout,
-			 int pos, int32_t **disk_layout_p)
+                         int pos, int32_t **disk_layout_p)
 {
-	int      ret = -1;
-	int32_t *disk_layout = NULL;
+        int      ret = -1;
+        int32_t *disk_layout = NULL;
 
-	disk_layout = GF_CALLOC (5, sizeof (int),
+        disk_layout = GF_CALLOC (5, sizeof (int),
                                  gf_dht_mt_int32_t);
-	if (!disk_layout) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto out;
-	}
+        if (!disk_layout) {
+                goto out;
+        }
 
-	disk_layout[0] = hton32 (1);
-	disk_layout[1] = hton32 (layout->type);
-	disk_layout[2] = hton32 (layout->list[pos].start);
-	disk_layout[3] = hton32 (layout->list[pos].stop);
+        disk_layout[0] = hton32 (1);
+        disk_layout[1] = hton32 (layout->type);
+        disk_layout[2] = hton32 (layout->list[pos].start);
+        disk_layout[3] = hton32 (layout->list[pos].stop);
 
-	if (disk_layout_p)
-		*disk_layout_p = disk_layout;
-	ret = 0;
+        if (disk_layout_p)
+                *disk_layout_p = disk_layout;
+        else
+                GF_FREE (disk_layout);
+
+        ret = 0;
 
 out:
-	return ret;
+        return ret;
 }
 
 
 int
 dht_disk_layout_merge (xlator_t *this, dht_layout_t *layout,
-		       int pos, void *disk_layout_raw)
+		       int pos, void *disk_layout_raw, int disk_layout_len)
 {
-	int      cnt = 0;
-	int      type = 0;
-	int      start_off = 0;
-	int      stop_off = 0;
+        int      cnt = 0;
+        int      type = 0;
+        int      start_off = 0;
+        int      stop_off = 0;
         int      disk_layout[4];
 
-	/* TODO: assert disk_layout_ptr is of required length */
+	if (!disk_layout_raw) {
+		gf_log (this->name, GF_LOG_CRITICAL,
+                        "error no layout on disk for merge");
+		return -1;
+	}
 
-        memcpy (disk_layout, disk_layout_raw, sizeof (disk_layout));
+	GF_ASSERT (disk_layout_len == sizeof (disk_layout));
+
+        memcpy (disk_layout, disk_layout_raw, disk_layout_len);
 
-	cnt  = ntoh32 (disk_layout[0]);
-	if (cnt != 1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"disk layout has invalid count %d", cnt);
+        cnt  = ntoh32 (disk_layout[0]);
+        if (cnt != 1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "disk layout has invalid count %d", cnt);
+                return -1;
+        }
+
+        type = ntoh32 (disk_layout[1]);
+	switch (type) {
+        case DHT_HASH_TYPE_DM_USER:
+                gf_log (this->name, GF_LOG_DEBUG, "found user-set layout");
+                layout->type = type;
+                /* Fall through. */
+	case DHT_HASH_TYPE_DM:
+		break;
+        default:
+		gf_log (this->name, GF_LOG_CRITICAL,
+			"Catastrophic error layout with unknown type found %d",
+			disk_layout[1]);
 		return -1;
 	}
 
-	/* TODO: assert type is compatible */
-	type      = ntoh32 (disk_layout[1]);
-	start_off = ntoh32 (disk_layout[2]);
-	stop_off  = ntoh32 (disk_layout[3]);
+        start_off = ntoh32 (disk_layout[2]);
+        stop_off  = ntoh32 (disk_layout[3]);
 
-	layout->list[pos].start = start_off;
-	layout->list[pos].stop  = stop_off;
+        layout->list[pos].start = start_off;
+        layout->list[pos].stop  = stop_off;
 
-	gf_log (this->name, GF_LOG_TRACE,
-		"merged to layout: %u - %u (type %d) from %s",
-		start_off, stop_off, type,
-		layout->list[pos].xlator->name);
+        gf_log (this->name, GF_LOG_TRACE,
+                "merged to layout: %u - %u (type %d) from %s",
+                start_off, stop_off, type,
+                layout->list[pos].xlator->name);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_layout_merge (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
-		  int op_ret, int op_errno, dict_t *xattr)
+                  int op_ret, int op_errno, dict_t *xattr)
 {
-	int      i     = 0;
-	int      ret   = -1;
-	int      err   = -1;
-	void    *disk_layout_raw = NULL;
-
-
-	if (op_ret != 0) {
-		err = op_errno;
-	}
+        int         i     = 0;
+        int         ret   = -1;
+        int         err   = -1;
+        void       *disk_layout_raw = NULL;
+        int         disk_layout_len = 0;
+        dht_conf_t *conf = this->private;
+
+        if (op_ret != 0) {
+                err = op_errno;
+        }
 
-	for (i = 0; i < layout->cnt; i++) {
-		if (layout->list[i].xlator == NULL) {
-			layout->list[i].err    = err;
-			layout->list[i].xlator = subvol;
-			break;
-		}
-	}
+        for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].xlator == NULL) {
+                        layout->list[i].err    = err;
+                        layout->list[i].xlator = subvol;
+                        break;
+                }
+        }
 
-	if (op_ret != 0) {
-		ret = 0;
-		goto out;
-	}
+        if (op_ret != 0) {
+                ret = 0;
+                goto out;
+        }
 
-	if (xattr) {
-		/* during lookup and not mkdir */
-		ret = dict_get_ptr (xattr, "trusted.glusterfs.dht",
-				    &disk_layout_raw);
-	}
+        if (xattr) {
+                /* during lookup and not mkdir */
+                ret = dict_get_ptr_and_len (xattr, conf->xattr_name,
+					    &disk_layout_raw, &disk_layout_len);
+        }
 
-	if (ret != 0) {
-		layout->list[i].err = -1;
-		gf_log (this->name, GF_LOG_TRACE,
-			"missing disk layout on %s. err = %d",
-			subvol->name, err);
-		ret = 0;
-		goto out;
-	}
+        if (ret != 0) {
+                layout->list[i].err = 0;
+                gf_log (this->name, GF_LOG_TRACE,
+                        "missing disk layout on %s. err = %d",
+                        subvol->name, err);
+                ret = 0;
+                goto out;
+        }
 
-	ret = dht_disk_layout_merge (this, layout, i, disk_layout_raw);
-	if (ret != 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"layout merge from subvolume %s failed",
-			subvol->name);
-		goto out;
-	}
-	layout->list[i].err = 0;
+        ret = dht_disk_layout_merge (this, layout, i, disk_layout_raw,
+				     disk_layout_len);
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "layout merge from subvolume %s failed",
+                        subvol->name);
+                goto out;
+        }
+        layout->list[i].err = 0;
 
 out:
-	return ret;
+        return ret;
 }
 
 
 void
 dht_layout_entry_swap (dht_layout_t *layout, int i, int j)
 {
-	uint32_t  start_swap = 0;
-	uint32_t  stop_swap = 0;
-	xlator_t *xlator_swap = 0;
-	int       err_swap = 0;
-
-
-	start_swap  = layout->list[i].start;
-	stop_swap   = layout->list[i].stop;
-	xlator_swap = layout->list[i].xlator;
-	err_swap    = layout->list[i].err;
-
-	layout->list[i].start  = layout->list[j].start;
-	layout->list[i].stop   = layout->list[j].stop;
-	layout->list[i].xlator = layout->list[j].xlator;
-	layout->list[i].err    = layout->list[j].err;
-
-	layout->list[j].start  = start_swap;
-	layout->list[j].stop   = stop_swap;
-	layout->list[j].xlator = xlator_swap;
-	layout->list[j].err    = err_swap;
+        uint32_t  start_swap = 0;
+        uint32_t  stop_swap = 0;
+        xlator_t *xlator_swap = 0;
+        int       err_swap = 0;
+
+        start_swap  = layout->list[i].start;
+        stop_swap   = layout->list[i].stop;
+        xlator_swap = layout->list[i].xlator;
+        err_swap    = layout->list[i].err;
+
+        layout->list[i].start  = layout->list[j].start;
+        layout->list[i].stop   = layout->list[j].stop;
+        layout->list[i].xlator = layout->list[j].xlator;
+        layout->list[i].err    = layout->list[j].err;
+
+        layout->list[j].start  = start_swap;
+        layout->list[j].stop   = stop_swap;
+        layout->list[j].xlator = xlator_swap;
+        layout->list[j].err    = err_swap;
+}
+
+void
+dht_layout_range_swap (dht_layout_t *layout, int i, int j)
+{
+        uint32_t  start_swap = 0;
+        uint32_t  stop_swap = 0;
+
+        start_swap  = layout->list[i].start;
+        stop_swap   = layout->list[i].stop;
+
+        layout->list[i].start  = layout->list[j].start;
+        layout->list[i].stop   = layout->list[j].stop;
+
+        layout->list[j].start  = start_swap;
+        layout->list[j].stop   = stop_swap;
 }
 
 int64_t
 dht_layout_entry_cmp_volname (dht_layout_t *layout, int i, int j)
 {
-	return (strcmp (layout->list[i].xlator->name, 
-			layout->list[j].xlator->name));
+        return (strcmp (layout->list[i].xlator->name,
+                        layout->list[j].xlator->name));
+}
+
+
+gf_boolean_t
+dht_is_subvol_in_layout (dht_layout_t *layout, xlator_t *xlator)
+{
+        int i = 0;
+
+        for (i = 0; i < layout->cnt; i++) {
+                /* Check if xlator is already part of layout, and layout is
+                 * non-zero. */
+                if (!strcmp (layout->list[i].xlator->name, xlator->name)) {
+                        if (layout->list[i].start != layout->list[i].stop)
+                                return _gf_true;
+                        break;
+                }
+        }
+        return _gf_false;
 }
 
 int64_t
 dht_layout_entry_cmp (dht_layout_t *layout, int i, int j)
 {
-	int64_t diff = 0;
+        int64_t diff = 0;
 
-	if (layout->list[i].err || layout->list[j].err)
-		diff = layout->list[i].err - layout->list[j].err;
-	else
-		diff = (int64_t) layout->list[i].start
-			- (int64_t) layout->list[j].start;
+        /* swap zero'ed out layouts to front, if needed */
+        if (!layout->list[j].start && !layout->list[j].stop) {
+                diff = (int64_t) layout->list[i].stop
+                       - (int64_t) layout->list[j].stop;
+                       goto out;
+        }
+        diff = (int64_t) layout->list[i].start
+                - (int64_t) layout->list[j].start;
 
-	return diff;
+out:
+        return diff;
 }
 
 
 int
 dht_layout_sort (dht_layout_t *layout)
 {
-	int       i = 0;
-	int       j = 0;
-	int64_t   ret = 0;
-
-	/* TODO: O(n^2) -- bad bad */
-
-	for (i = 0; i < layout->cnt - 1; i++) {
-		for (j = i + 1; j < layout->cnt; j++) {
-			ret = dht_layout_entry_cmp (layout, i, j);
-			if (ret > 0)
-				dht_layout_entry_swap (layout, i, j);
-		}
-	}
+        int       i = 0;
+        int       j = 0;
+        int64_t   ret = 0;
 
-	return 0;
+        /* TODO: O(n^2) -- bad bad */
+
+        for (i = 0; i < layout->cnt - 1; i++) {
+                for (j = i + 1; j < layout->cnt; j++) {
+                        ret = dht_layout_entry_cmp (layout, i, j);
+                        if (ret > 0)
+                                dht_layout_entry_swap (layout, i, j);
+                }
+        }
+
+        return 0;
 }
 
 int
 dht_layout_sort_volname (dht_layout_t *layout)
 {
-	int       i = 0;
-	int       j = 0;
-	int64_t   ret = 0;
-
-	/* TODO: O(n^2) -- bad bad */
-
-	for (i = 0; i < layout->cnt - 1; i++) {
-		for (j = i + 1; j < layout->cnt; j++) {
-			ret = dht_layout_entry_cmp_volname (layout, i, j);
-			if (ret > 0)
-				dht_layout_entry_swap (layout, i, j);
-		}
-	}
+        int       i = 0;
+        int       j = 0;
+        int64_t   ret = 0;
+
+        /* TODO: O(n^2) -- bad bad */
+
+        for (i = 0; i < layout->cnt - 1; i++) {
+                for (j = i + 1; j < layout->cnt; j++) {
+                        ret = dht_layout_entry_cmp_volname (layout, i, j);
+                        if (ret > 0)
+                                dht_layout_entry_swap (layout, i, j);
+                }
+        }
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_layout_anomalies (xlator_t *this, loc_t *loc, dht_layout_t *layout,
-		      uint32_t *holes_p, uint32_t *overlaps_p,
-		      uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p)
+                      uint32_t *holes_p, uint32_t *overlaps_p,
+                      uint32_t *missing_p, uint32_t *down_p, uint32_t *misc_p,
+                      uint32_t *no_space_p)
 {
-	uint32_t    overlaps = 0;
-	uint32_t    missing  = 0;
-	uint32_t    down     = 0;
-	uint32_t    misc     = 0;
-	uint32_t    hole_cnt = 0;
-	uint32_t    overlap_cnt = 0;
-	int         i = 0;
-	int         ret = 0;
-	uint32_t    prev_stop = 0;
-	uint32_t    last_stop = 0;
-	char        is_virgin = 1;
-
-	/* TODO: explain WTF is happening */
-
-	last_stop = layout->list[0].start - 1;
-	prev_stop = last_stop;
-
-	for (i = 0; i < layout->cnt; i++) {
-		if (layout->list[i].err) {
-			switch (layout->list[i].err) {
-			case -1:
-			case ENOENT:
-				missing++;
-				break;
-			case ENOTCONN:
-				down++;
-				break;
-			case ENOSPC:
-				down++;
-				break;
-			default:
-				misc++;
-			}
-			continue;
-		}
-
-		is_virgin = 0;
-
-		if ((prev_stop + 1) < layout->list[i].start) {
-			hole_cnt++;
-		}
-
-		if ((prev_stop + 1) > layout->list[i].start) {
-			overlap_cnt++;
-			overlaps += ((prev_stop + 1) - layout->list[i].start);
-		}
-		prev_stop = layout->list[i].stop;
-	}
+        uint32_t    overlaps = 0;
+        uint32_t    missing  = 0;
+        uint32_t    down     = 0;
+        uint32_t    misc     = 0;
+        uint32_t    hole_cnt = 0;
+        uint32_t    overlap_cnt = 0;
+        int         i = 0;
+        int         ret = 0;
+        uint32_t    prev_stop = 0;
+        uint32_t    last_stop = 0;
+        char        is_virgin = 1;
+        uint32_t    no_space  = 0;
+
+        /* This funtion scans through the layout spread of a directory to
+           check if there are any anomalies. Prior to calling this function
+           the layout entries should be sorted in the ascending order.
+
+           If the layout entry has err != 0
+                then increment the corresponding anomaly.
+           else
+                if (start of the current layout entry > stop + 1 of previous
+                   non erroneous layout entry)
+                        then it indicates a hole in the layout
+                if (start of the current layout entry < stop + 1 of previous
+                    non erroneous layout entry)
+                         then it indicates an overlap in the layout
+        */
+        last_stop = layout->list[0].start - 1;
+        prev_stop = last_stop;
+
+        for (i = 0; i < layout->cnt; i++) {
+                switch (layout->list[i].err) {
+                case -1:
+                case ENOENT:
+                case ESTALE:
+                        missing++;
+                        continue;
+                case ENOTCONN:
+                        down++;
+                        continue;
+                case ENOSPC:
+                        no_space++;
+                        continue;
+                case 0:
+                        /* if err == 0 and start == stop, then it is a non misc++;
+                         * participating subvolume(spread-cnt). Then, do not
+                         * check for anomalies. If start != stop, then treat it
+                         * as misc err */
+                        if (layout->list[i].start == layout->list[i].stop) {
+                                continue;
+                        }
+                        break;
+                default:
+                        misc++;
+                        continue;
+                 }
+
+                is_virgin = 0;
+
+                if ((prev_stop + 1) < layout->list[i].start) {
+                        hole_cnt++;
+                }
+
+                if ((prev_stop + 1) > layout->list[i].start) {
+                        overlap_cnt++;
+                        overlaps += ((prev_stop + 1) - layout->list[i].start);
+                }
+                prev_stop = layout->list[i].stop;
+        }
 
-	if ((last_stop - prev_stop) || is_virgin)
-	    hole_cnt++;
+        if ((last_stop - prev_stop) || is_virgin)
+                hole_cnt++;
 
-	if (holes_p)
-		*holes_p = hole_cnt;
+        if (holes_p)
+                *holes_p = hole_cnt;
 
-	if (overlaps_p)
-		*overlaps_p = overlap_cnt;
+        if (overlaps_p)
+                *overlaps_p = overlap_cnt;
 
-	if (missing_p)
-		*missing_p = missing;
+        if (missing_p)
+                *missing_p = missing;
 
-	if (down_p)
-		*down_p = down;
+        if (down_p)
+                *down_p = down;
 
-	if (misc_p)
-		*misc_p = misc;
+        if (misc_p)
+                *misc_p = misc;
 
-	return ret;
+        if (no_space_p)
+                *no_space_p = no_space;
+
+        return ret;
 }
 
 
 int
 dht_layout_normalize (xlator_t *this, loc_t *loc, dht_layout_t *layout)
 {
-	int          ret   = 0;
-	int          i = 0;
-	uint32_t     holes = 0;
-	uint32_t     overlaps = 0;
-	uint32_t     missing = 0;
-	uint32_t     down = 0;
-	uint32_t     misc = 0;
-
-
-	ret = dht_layout_sort (layout);
-	if (ret == -1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"sort failed?! how the ....");
-		goto out;
-	}
+        int          ret   = 0;
+        int          i = 0;
+        uint32_t     holes = 0;
+        uint32_t     overlaps = 0;
+        uint32_t     missing = 0;
+        uint32_t     down = 0;
+        uint32_t     misc = 0;
+
+        ret = dht_layout_sort (layout);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "sort failed?! how the ....");
+                goto out;
+        }
 
-	ret = dht_layout_anomalies (this, loc, layout,
-				    &holes, &overlaps,
-				    &missing, &down, &misc);
-	if (ret == -1) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"error while finding anomalies in %s -- not good news",
-			loc->path);
-		goto out;
-	}
+        ret = dht_layout_anomalies (this, loc, layout,
+                                    &holes, &overlaps,
+                                    &missing, &down, &misc, NULL);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "error while finding anomalies in %s -- not good news",
+                        loc->path);
+                goto out;
+        }
 
-	if (holes || overlaps) {
-		if (missing == layout->cnt) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"directory %s looked up first time",
-				loc->path);
-		} else {
-			gf_log (this->name, GF_LOG_INFO,
-				"found anomalies in %s. holes=%d overlaps=%d",
-				loc->path, holes, overlaps);
-		}
-		ret = 1;
-	}
+        if (holes || overlaps) {
+                if (missing == layout->cnt) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "directory %s looked up first time",
+                                loc->path);
+                } else {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "found anomalies in %s. holes=%d overlaps=%d",
+                                loc->path, holes, overlaps);
+                }
+                ret = -1;
+        }
+
+        for (i = 0; i < layout->cnt; i++) {
+                /* TODO During DHT selfheal rewrite (almost) find a better place
+                 * to detect this - probably in dht_layout_anomalies()
+                 */
+                if (layout->list[i].err > 0) {
+                        gf_log_callingfn (this->name, GF_LOG_DEBUG,
+                                          "path=%s err=%s on subvol=%s",
+                                          loc->path,
+                                          strerror (layout->list[i].err),
+                                          (layout->list[i].xlator ?
+                                           layout->list[i].xlator->name
+                                           : "<>"));
+                        if ((layout->list[i].err == ENOENT) && (ret >= 0)) {
+                                ret++;
+                        }
+                }
+        }
 
-	for (i = 0; i < layout->cnt; i++) {
-	/* TODO During DHT selfheal rewrite (almost) find a better place to 
-	 * detect this - probably in dht_layout_anomalies() 
-	 */
-		if (layout->list[i].err > 0) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"path=%s err=%s on subvol=%s",
-                                loc->path, strerror (layout->list[i].err),
-                                (layout->list[i].xlator ?
-                                 layout->list[i].xlator->name : "<>"));
-                        if (layout->list[i].err == ENOENT)
-                                ret = 1;
-		}
-	}
 
 out:
-	return ret;
+        return ret;
 }
 
-
 int
-dht_layout_dir_mismatch (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
-			 loc_t *loc, dict_t *xattr)
+dht_dir_has_layout (dict_t *xattr, char *name)
 {
-	int       idx = 0;
-	int       pos = -1;
-	int       ret = 0;
-        int       err = 0;
-        int       dict_ret = 0;
-	int32_t   disk_layout[4];
+
         void     *disk_layout_raw = NULL;
-	int32_t   count = -1;
-	uint32_t  start_off = -1;
-	uint32_t  stop_off = -1;
 
+        return dict_get_ptr (xattr, name, &disk_layout_raw);
+}
 
-	for (idx = 0; idx < layout->cnt; idx++) {
-		if (layout->list[idx].xlator == subvol) {
-			pos = idx;
-			break;
-		}
-	}
-	
-	if (pos == -1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"%s - no layout info for subvolume %s",
-			loc->path, subvol->name);
-		ret = 1;
-		goto out;
-	}
+int
+dht_layout_dir_mismatch (xlator_t *this, dht_layout_t *layout, xlator_t *subvol,
+                         loc_t *loc, dict_t *xattr)
+{
+        int         idx = 0;
+        int         pos = -1;
+        int         ret = 0;
+        int         err = 0;
+        int         dict_ret = 0;
+        int32_t     disk_layout[4];
+        void       *disk_layout_raw = NULL;
+        int32_t     count = -1;
+        uint32_t    start_off = -1;
+        uint32_t    stop_off = -1;
+        dht_conf_t *conf = this->private;
+
+
+        for (idx = 0; idx < layout->cnt; idx++) {
+                if (layout->list[idx].xlator == subvol) {
+                        pos = idx;
+                        break;
+                }
+        }
+
+        if (pos == -1) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "%s - no layout info for subvolume %s",
+                        loc->path, subvol->name);
+                ret = 1;
+                goto out;
+        }
 
         err = layout->list[pos].err;
 
-	if (!xattr) {
+        if (!xattr) {
                 if (err == 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
+                        gf_log (this->name, GF_LOG_INFO,
                                 "%s - xattr dictionary is NULL",
                                 loc->path);
                         ret = -1;
                 }
-		goto out;
-	}
+                goto out;
+        }
 
-	dict_ret = dict_get_ptr (xattr, "trusted.glusterfs.dht",
+        dict_ret = dict_get_ptr (xattr, conf->xattr_name,
                                  &disk_layout_raw);
 
-	if (dict_ret < 0) {
-                if (err == 0) {
-                        gf_log (this->name, GF_LOG_DEBUG,
+        if (dict_ret < 0) {
+                if (err == 0 && layout->list[pos].stop) {
+                        gf_log (this->name, GF_LOG_INFO,
                                 "%s - disk layout missing", loc->path);
                         ret = -1;
                 }
-		goto out;
-	}
+                goto out;
+        }
 
         memcpy (disk_layout, disk_layout_raw, sizeof (disk_layout));
 
-	count  = ntoh32 (disk_layout[0]);
-	if (count != 1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"%s - disk layout has invalid count %d",
-			loc->path, count);
-		ret = -1;
-		goto out;
-	}
+        count  = ntoh32 (disk_layout[0]);
+        if (count != 1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s - disk layout has invalid count %d",
+                        loc->path, count);
+                ret = -1;
+                goto out;
+        }
 
-	start_off = ntoh32 (disk_layout[2]);
-	stop_off  = ntoh32 (disk_layout[3]);
-
-	if ((layout->list[pos].start != start_off)
-	    || (layout->list[pos].stop != stop_off)) {
-		gf_log (this->name, GF_LOG_INFO,
-			"subvol: %s; inode layout - %"PRIu32" - %"PRIu32"; "
-			"disk layout - %"PRIu32" - %"PRIu32,
-			layout->list[pos].xlator->name,
-			layout->list[pos].start, layout->list[pos].stop,
-			start_off, stop_off);
-		ret = 1;
-	} else {
-		ret = 0;
-	}
+        start_off = ntoh32 (disk_layout[2]);
+        stop_off  = ntoh32 (disk_layout[3]);
+
+        if ((layout->list[pos].start != start_off)
+            || (layout->list[pos].stop != stop_off)) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "subvol: %s; inode layout - %"PRIu32" - %"PRIu32"; "
+                        "disk layout - %"PRIu32" - %"PRIu32,
+                        layout->list[pos].xlator->name,
+                        layout->list[pos].start, layout->list[pos].stop,
+                        start_off, stop_off);
+                ret = 1;
+        } else {
+                ret = 0;
+        }
 out:
-	return ret;
+        return ret;
 }
 
 
@@ -707,18 +785,18 @@ dht_layout_preset (xlator_t *this, xlator_t *subvol, inode_t *inode)
         if (!conf)
                 goto out;
 
-	layout = dht_layout_for_subvol (this, subvol);
-	if (!layout) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no pre-set layout for subvolume %s",
-			subvol ? subvol->name : "<nil>");
-		ret = -1;
-		goto out;
-	}
+        layout = dht_layout_for_subvol (this, subvol);
+        if (!layout) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "no pre-set layout for subvolume %s",
+                        subvol ? subvol->name : "<nil>");
+                ret = -1;
+                goto out;
+        }
 
         LOCK (&conf->layout_lock);
         {
-                inode_ctx_put (inode, this, (uint64_t)(long)layout);
+                dht_inode_ctx_layout_set (inode, this, layout);
         }
         UNLOCK (&conf->layout_lock);
 
diff --git a/xlators/cluster/dht/src/dht-linkfile.c b/xlators/cluster/dht/src/dht-linkfile.c
index c72579e47..dbc9d0b3c 100644
--- a/xlators/cluster/dht/src/dht-linkfile.c
+++ b/xlators/cluster/dht/src/dht-linkfile.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -28,220 +19,310 @@
 #include "compat.h"
 #include "dht-common.h"
 
-
-
 int
-dht_linkfile_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int op_ret, int op_errno)
+dht_linkfile_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int op_ret, int op_errno,
+                         inode_t *inode, struct iatt *stbuf, dict_t *xattr,
+                         struct iatt *postparent)
 {
-	dht_local_t *local = NULL;
-
-
-	local = frame->local;
-	local->linkfile.linkfile_cbk (frame, cookie, this, op_ret, op_errno,
-				      local->linkfile.inode,
-				      &local->linkfile.stbuf, NULL, NULL);
-
-	return 0;
+        char          is_linkfile   = 0;
+        dht_conf_t   *conf          = NULL;
+        dht_local_t  *local         = NULL;
+        call_frame_t *prev          = NULL;
+
+        local = frame->local;
+        prev = cookie;
+        conf = this->private;
+
+        if (op_ret)
+                goto out;
+
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
+        if (!is_linkfile)
+                gf_log (this->name, GF_LOG_WARNING, "got non-linkfile %s:%s",
+                        prev->this->name, local->loc.path);
+out:
+        local->linkfile.linkfile_cbk (frame, cookie, this, op_ret, op_errno,
+                                      inode, stbuf, postparent, postparent,
+                                      xattr);
+        return 0;
 }
 
-
+#define is_equal(a, b) (a == b)
 int
 dht_linkfile_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int op_ret, int op_errno, inode_t *inode,
                          struct iatt *stbuf, struct iatt *preparent,
-                         struct iatt *postparent)
+                         struct iatt *postparent, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	call_frame_t *prev = NULL;
-	dict_t       *xattr = NULL;
-	data_t       *str_data = NULL;
-	int           ret = -1;
-
-	local = frame->local;
-	prev  = cookie;
-
-	if (op_ret == -1)
-		goto err;
-
-	xattr = get_new_dict ();
-	if (!xattr) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		op_errno = ENOMEM;
-		goto err;
-	}
-
-	local->linkfile.xattr = dict_ref (xattr);
-	local->linkfile.inode = inode_ref (inode);
-
-	str_data = str_to_data (local->linkfile.srcvol->name);
-	if (!str_data) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		op_errno = ENOMEM;
-		goto err;
-	}
-
-	ret = dict_set (xattr, "trusted.glusterfs.dht.linkto", str_data);
-	if (ret < 0) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"failed to initialize linkfile data");
-	}
-	str_data = NULL;
-
-	local->linkfile.stbuf = *stbuf;
-
-	STACK_WIND (frame, dht_linkfile_xattr_cbk,
-		    prev->this, prev->this->fops->setxattr,
-		    &local->linkfile.loc, local->linkfile.xattr, 0);
-
-	return 0;
+        dht_local_t  *local = NULL;
+        xlator_t     *subvol = NULL;
+        call_frame_t *prev = NULL;
+        dict_t       *xattrs = NULL;
+        dht_conf_t   *conf = NULL;
+        int           ret = -1;
 
-err:
-	if (str_data) {
-		data_destroy (str_data);
-		str_data = NULL;
-	}
+        local = frame->local;
+
+        if (!op_ret)
+                local->linked = _gf_true;
+
+        FRAME_SU_UNDO (frame, dht_local_t);
+
+        if (op_ret && (op_errno == EEXIST)) {
+                conf = this->private;
+                prev = cookie;
+                subvol = prev->this;
+                if (!subvol)
+                        goto out;
+                xattrs = dict_new ();
+                if (!xattrs)
+                        goto out;
+                ret = dict_set_uint32 (xattrs, conf->link_xattr_name, 256);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to set linkto key");
+                        goto out;
+                }
 
+                STACK_WIND (frame, dht_linkfile_lookup_cbk, subvol,
+                            subvol->fops->lookup, &local->loc, xattrs);
+                if (xattrs)
+                        dict_unref (xattrs);
+                return 0;
+        }
+out:
         local->linkfile.linkfile_cbk (frame, cookie, this, op_ret, op_errno,
-                                      inode, stbuf, preparent, postparent);
-	return 0;
+                                      inode, stbuf, preparent, postparent,
+                                      xdata);
+        if (xattrs)
+                dict_unref (xattrs);
+        return 0;
 }
 
 
 int
 dht_linkfile_create (call_frame_t *frame, fop_mknod_cbk_t linkfile_cbk,
-		     xlator_t *tovol, xlator_t *fromvol, loc_t *loc)
+                     xlator_t *this,
+                     xlator_t *tovol, xlator_t *fromvol, loc_t *loc)
 {
-	dht_local_t *local = NULL;
+        dht_local_t *local = NULL;
         dict_t      *dict = NULL;
+        int          need_unref = 0;
         int          ret = 0;
+        dht_conf_t  *conf = this->private;
 
-	local = frame->local;
-	local->linkfile.linkfile_cbk = linkfile_cbk;
-	local->linkfile.srcvol = tovol;
-	loc_copy (&local->linkfile.loc, loc);
+        local = frame->local;
+        local->linkfile.linkfile_cbk = linkfile_cbk;
+        local->linkfile.srcvol = tovol;
 
-        if (!uuid_is_null (local->gfid)) {
+        local->linked = _gf_false;
+
+        dict = local->params;
+        if (!dict) {
                 dict = dict_new ();
                 if (!dict)
                         goto out;
+                need_unref = 1;
+        }
+
+        if (!uuid_is_null (local->gfid)) {
                 ret = dict_set_static_bin (dict, "gfid-req", local->gfid, 16);
                 if (ret)
-                        gf_log ("dht-linkfile", GF_LOG_DEBUG, "gfid set failed");
-        } else if (local->params) {
-                dict = dict_ref (local->params);
+                        gf_log ("dht-linkfile", GF_LOG_INFO,
+                                "%s: gfid set failed", loc->path);
         }
-        if (!dict)
-                gf_log ("", GF_LOG_DEBUG,
-                        "dict is NULL, need to make sure gfid's are same");
 
-	STACK_WIND (frame, dht_linkfile_create_cbk,
-		    fromvol, fromvol->fops->mknod, loc,
-		    S_IFREG | DHT_LINKFILE_MODE, 0, dict);
+        ret = dict_set_str (dict, GLUSTERFS_INTERNAL_FOP_KEY, "yes");
+        if (ret)
+                gf_log ("dht-linkfile", GF_LOG_INFO,
+                        "%s: internal-fop set failed", loc->path);
 
-        if (dict)
+        ret = dict_set_str (dict, conf->link_xattr_name, tovol->name);
+
+        if (ret < 0) {
+                gf_log (frame->this->name, GF_LOG_INFO,
+                        "%s: failed to initialize linkfile data",
+                        loc->path);
+                goto out;
+        }
+
+        local->link_subvol = fromvol;
+        /* Always create as root:root. dht_linkfile_attr_heal fixes the
+         * ownsership */
+        FRAME_SU_DO (frame, dht_local_t);
+        STACK_WIND (frame, dht_linkfile_create_cbk,
+                    fromvol, fromvol->fops->mknod, loc,
+                    S_IFREG | DHT_LINKFILE_MODE, 0, 0, dict);
+
+        if (need_unref && dict)
                 dict_unref (dict);
 
         return 0;
 out:
-	local->linkfile.linkfile_cbk (frame, NULL, frame->this, -1, ENOMEM,
-				      loc->inode, NULL, NULL, NULL);
-	return 0;
+        local->linkfile.linkfile_cbk (frame, NULL, frame->this, -1, ENOMEM,
+                                      loc->inode, NULL, NULL, NULL, NULL);
+
+        if (need_unref && dict)
+                dict_unref (dict);
+
+        return 0;
 }
 
 
 int
 dht_linkfile_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			 int32_t op_ret, int32_t op_errno,
-                         struct iatt *preparent, struct iatt *postparent)
+                         int32_t op_ret, int32_t op_errno,
+                         struct iatt *preparent, struct iatt *postparent,
+                         dict_t *xdata)
 {
-	dht_local_t   *local = NULL;
-	call_frame_t  *prev = NULL;
-	xlator_t      *subvol = NULL;
-
-	local = frame->local;
-	prev = cookie;
-	subvol = prev->this;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"unlinking linkfile %s on %s failed (%s)",
-			local->loc.path, subvol->name, strerror (op_errno));
-	}
+        dht_local_t   *local = NULL;
+        call_frame_t  *prev = NULL;
+        xlator_t      *subvol = NULL;
+
+        local = frame->local;
+        prev = cookie;
+        subvol = prev->this;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "unlinking linkfile %s on %s failed (%s)",
+                        local->loc.path, subvol->name, strerror (op_errno));
+        }
 
-	DHT_STACK_DESTROY (frame);
+        DHT_STACK_DESTROY (frame);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_linkfile_unlink (call_frame_t *frame, xlator_t *this,
-		     xlator_t *subvol, loc_t *loc)
+                     xlator_t *subvol, loc_t *loc)
 {
-	call_frame_t *unlink_frame = NULL;
-	dht_local_t  *unlink_local = NULL;
-
-	unlink_frame = copy_frame (frame);
-	if (!unlink_frame) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
-
-	unlink_local = dht_local_init (unlink_frame);
-	if (!unlink_local) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
-
-	loc_copy (&unlink_local->loc, loc);
-
-	STACK_WIND (unlink_frame, dht_linkfile_unlink_cbk,
-		    subvol, subvol->fops->unlink,
-		    &unlink_local->loc);
-
-	return 0;
+        call_frame_t *unlink_frame = NULL;
+        dht_local_t  *unlink_local = NULL;
+
+        unlink_frame = copy_frame (frame);
+        if (!unlink_frame) {
+                goto err;
+        }
+
+        /* Using non-fop value here, as anyways, 'local->fop' is not used in
+           this particular case */
+        unlink_local = dht_local_init (unlink_frame, loc, NULL,
+                                       GF_FOP_MAXVALUE);
+        if (!unlink_local) {
+                goto err;
+        }
+
+        STACK_WIND (unlink_frame, dht_linkfile_unlink_cbk,
+                    subvol, subvol->fops->unlink,
+                    &unlink_local->loc, 0, NULL);
+
+        return 0;
 err:
-	if (unlink_frame)
-		DHT_STACK_DESTROY (unlink_frame);
+        if (unlink_frame)
+                DHT_STACK_DESTROY (unlink_frame);
 
-	return -1;
+        return -1;
 }
 
 
 xlator_t *
 dht_linkfile_subvol (xlator_t *this, inode_t *inode, struct iatt *stbuf,
-		     dict_t *xattr)
+                     dict_t *xattr)
 {
-	dht_conf_t *conf = NULL;
-	xlator_t   *subvol = NULL;
-	void       *volname = NULL;
-	int         i = 0, ret = 0;
-
+        dht_conf_t *conf = NULL;
+        xlator_t   *subvol = NULL;
+        void       *volname = NULL;
+        int         i = 0, ret = 0;
 
-	conf = this->private;
+        conf = this->private;
 
-	if (!xattr)
-		goto out;
+        if (!xattr)
+                goto out;
 
-	ret = dict_get_ptr (xattr, "trusted.glusterfs.dht.linkto", &volname);
+        ret = dict_get_ptr (xattr, conf->link_xattr_name, &volname);
 
-	if ((-1 == ret) || !volname)
-		goto out;
+        if ((-1 == ret) || !volname)
+                goto out;
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		if (strcmp (conf->subvolumes[i]->name, (char *)volname) == 0) {
-			subvol = conf->subvolumes[i];
-			break;
-		}
-	}
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (strcmp (conf->subvolumes[i]->name, (char *)volname) == 0) {
+                        subvol = conf->subvolumes[i];
+                        break;
+                }
+        }
 
 out:
-	return subvol;
+        return subvol;
 }
 
+int
+dht_linkfile_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int op_ret, int op_errno, struct iatt *statpre,
+                          struct iatt *statpost, dict_t *xdata)
+{
+        dht_local_t *local = NULL;
+        loc_t *loc = NULL;
 
+        local = frame->local;
+        loc = &local->loc;
+
+        if (op_ret)
+                gf_log (this->name, GF_LOG_ERROR, "setattr of uid/gid on %s"
+                        " :<gfid:%s> failed (%s)",
+                        (loc->path? loc->path: "NULL"),
+                        uuid_utoa(local->gfid), strerror(op_errno));
+
+        DHT_STACK_DESTROY (frame);
+
+       return 0;
+}
+
+int
+dht_linkfile_attr_heal (call_frame_t *frame, xlator_t *this)
+{
+        int     ret = -1;
+        call_frame_t *copy = NULL;
+        dht_local_t  *local = NULL;
+        dht_local_t  *copy_local = NULL;
+        xlator_t     *subvol = NULL;
+        struct iatt   stbuf = {0,};
+
+        local = frame->local;
+
+        GF_VALIDATE_OR_GOTO ("dht", local, out);
+        GF_VALIDATE_OR_GOTO ("dht", local->link_subvol, out);
+
+        if (local->stbuf.ia_type == IA_INVAL)
+                return 0;
+
+        uuid_copy (local->loc.gfid, local->stbuf.ia_gfid);
+
+        copy = copy_frame (frame);
+
+        if (!copy)
+                goto out;
+
+       copy_local = dht_local_init (copy, &local->loc, NULL, 0);
+
+       if (!copy_local)
+               goto out;
+
+        stbuf = local->stbuf;
+        subvol = local->link_subvol;
+
+        copy->local = copy_local;
+
+        FRAME_SU_DO (copy, dht_local_t);
+
+        STACK_WIND (copy, dht_linkfile_setattr_cbk, subvol,
+                    subvol->fops->setattr, &copy_local->loc,
+                    &stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID), NULL);
+        ret = 0;
+out:
+        return ret;
+}
diff --git a/xlators/cluster/dht/src/dht-mem-types.h b/xlators/cluster/dht/src/dht-mem-types.h
index a5d28e8fb..e893eb48f 100644
--- a/xlators/cluster/dht/src/dht-mem-types.h
+++ b/xlators/cluster/dht/src/dht-mem-types.h
@@ -1,21 +1,11 @@
-
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -29,14 +19,17 @@ enum gf_dht_mem_types_ {
         gf_dht_mt_dht_conf_t,
         gf_dht_mt_char,
         gf_dht_mt_int32_t,
-        gf_dht_mt_dht_local_t,
         gf_dht_mt_xlator_t,
         gf_dht_mt_dht_layout_t,
         gf_switch_mt_dht_conf_t,
         gf_switch_mt_dht_du_t,
         gf_switch_mt_switch_sched_array,
         gf_switch_mt_switch_struct,
+        gf_dht_mt_subvol_time,
+        gf_dht_mt_loc_t,
+        gf_defrag_info_mt,
+        gf_dht_mt_inode_ctx_t,
+        gf_dht_mt_ctx_stat_time_t,
         gf_dht_mt_end
 };
 #endif
-
diff --git a/xlators/cluster/dht/src/dht-rebalance.c b/xlators/cluster/dht/src/dht-rebalance.c
new file mode 100644
index 000000000..4f78f5203
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-rebalance.c
@@ -0,0 +1,1908 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "dht-common.h"
+#include "xlator.h"
+#include <signal.h>
+#include <fnmatch.h>
+
+#define GF_DISK_SECTOR_SIZE             512
+#define DHT_REBALANCE_PID               4242 /* Change it if required */
+#define DHT_REBALANCE_BLKSIZE           (128 * 1024)
+
+static int
+dht_write_with_holes (xlator_t *to, fd_t *fd, struct iovec *vec, int count,
+                      int32_t size, off_t offset, struct iobref *iobref)
+{
+        int i            = 0;
+        int ret          = -1;
+        int start_idx    = 0;
+        int tmp_offset   = 0;
+        int write_needed = 0;
+        int buf_len      = 0;
+        int size_pending = 0;
+        char *buf        = NULL;
+
+        /* loop through each vector */
+        for (i = 0; i < count; i++) {
+                buf = vec[i].iov_base;
+                buf_len = vec[i].iov_len;
+
+                for (start_idx = 0; (start_idx + GF_DISK_SECTOR_SIZE) <= buf_len;
+                     start_idx += GF_DISK_SECTOR_SIZE) {
+
+                        if (mem_0filled (buf + start_idx, GF_DISK_SECTOR_SIZE) != 0) {
+                                write_needed = 1;
+                                continue;
+                        }
+
+                        if (write_needed) {
+                                ret = syncop_write (to, fd, (buf + tmp_offset),
+                                                    (start_idx - tmp_offset),
+                                                    (offset + tmp_offset),
+                                                    iobref, 0);
+                                /* 'path' will be logged in calling function */
+                                if (ret < 0) {
+                                        gf_log (THIS->name, GF_LOG_WARNING,
+                                                "failed to write (%s)",
+                                                strerror (-ret));
+                                        ret = -1;
+                                        goto out;
+                                }
+
+                                write_needed = 0;
+                        }
+                        tmp_offset = start_idx + GF_DISK_SECTOR_SIZE;
+                }
+
+                if ((start_idx < buf_len) || write_needed) {
+                        /* This means, last chunk is not yet written.. write it */
+                        ret = syncop_write (to, fd, (buf + tmp_offset),
+                                            (buf_len - tmp_offset),
+                                            (offset + tmp_offset), iobref, 0);
+                        if (ret < 0) {
+                                /* 'path' will be logged in calling function */
+                                gf_log (THIS->name, GF_LOG_WARNING,
+                                        "failed to write (%s)",
+                                        strerror (-ret));
+                                ret = -1;
+                                goto out;
+                        }
+                }
+
+                size_pending = (size - buf_len);
+                if (!size_pending)
+                        break;
+        }
+
+        ret = size;
+out:
+        return ret;
+
+}
+
+/*
+   return values:
+   -1 : failure
+   -2 : success
+
+Hard link migration is carried out in three stages.
+
+(Say there are n hardlinks)
+Stage 1: Setting the new hashed subvol information on the 1st hardlink
+         encountered (linkto setxattr)
+
+Stage 2: Creating hardlinks on new hashed subvol for the 2nd to (n-1)th
+         hardlink
+
+Stage 3: Physical migration of the data file for nth hardlink
+
+Why to deem "-2" as success and not "0":
+
+   dht_migrate_file expects return value "0" from _is_file_migratable if
+the file has to be migrated.
+
+   _is_file_migratable returns zero only when it is called with the
+flag "GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS".
+
+   gf_defrag_handle_hardlink calls dht_migrate_file for physical migration
+of the data file with the flag "GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS"
+
+Hence, gf_defrag_handle_hardlink returning "0" for success will force
+"dht_migrate_file" to migrate each of the hardlink which is not intended.
+
+For each of the three stage mentioned above "-2" will be returned and will
+be converted to "0" in dht_migrate_file.
+
+*/
+
+int32_t
+gf_defrag_handle_hardlink (xlator_t *this, loc_t *loc, dict_t  *xattrs,
+                           struct iatt *stbuf)
+{
+        int32_t                 ret             = -1;
+        xlator_t               *cached_subvol   = NULL;
+        xlator_t               *hashed_subvol   = NULL;
+        xlator_t               *linkto_subvol   = NULL;
+        data_t                 *data            = NULL;
+        struct iatt             iatt            = {0,};
+        int32_t                 op_errno        = 0;
+        dht_conf_t             *conf            = NULL;
+
+        GF_VALIDATE_OR_GOTO ("defrag", loc, out);
+        GF_VALIDATE_OR_GOTO ("defrag", loc->name, out);
+        GF_VALIDATE_OR_GOTO ("defrag", stbuf, out);
+        GF_VALIDATE_OR_GOTO ("defrag", this, out);
+        GF_VALIDATE_OR_GOTO ("defrag", xattrs, out);
+        GF_VALIDATE_OR_GOTO ("defrag", this->private, out);
+
+        conf = this->private;
+
+        if (uuid_is_null (loc->pargfid)) {
+                gf_log ("", GF_LOG_ERROR, "loc->pargfid is NULL for "
+                        "%s", loc->path);
+                goto out;
+        }
+
+        if (uuid_is_null (loc->gfid)) {
+                gf_log ("", GF_LOG_ERROR, "loc->gfid is NULL for "
+                        "%s", loc->path);
+                goto out;
+        }
+
+        cached_subvol = dht_subvol_get_cached (this, loc->inode);
+        if (!cached_subvol) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get cached subvol"
+                        " for %s on %s", loc->name, this->name);
+                goto out;
+        }
+
+        hashed_subvol = dht_subvol_get_hashed (this, loc);
+        if (!hashed_subvol) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to get hashed subvol"
+                        " for %s on %s", loc->name, this->name);
+                goto out;
+        }
+
+        gf_log (this->name, GF_LOG_INFO, "Attempting to migrate hardlink %s "
+                "with gfid %s from %s -> %s", loc->name, uuid_utoa (loc->gfid),
+                cached_subvol->name, hashed_subvol->name);
+        data = dict_get (xattrs, conf->link_xattr_name);
+        /* set linkto on cached -> hashed if not present, else link it */
+        if (!data) {
+                ret = dict_set_str (xattrs, conf->link_xattr_name,
+                                    hashed_subvol->name);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set "
+                                "linkto xattr in dict for %s", loc->name);
+                        goto out;
+                }
+
+                ret = syncop_setxattr (cached_subvol, loc, xattrs, 0);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Linkto setxattr "
+                                "failed %s -> %s (%s)", cached_subvol->name,
+                                loc->name, strerror (-ret));
+                        ret = -1;
+                        goto out;
+                }
+                ret = -2;
+                goto out;
+        } else {
+                linkto_subvol = dht_linkfile_subvol (this, NULL, NULL, xattrs);
+                if (!linkto_subvol) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "linkto subvol for %s", loc->name);
+                } else {
+                        hashed_subvol = linkto_subvol;
+                }
+
+                ret = syncop_link (hashed_subvol, loc, loc);
+                if  (ret) {
+                        op_errno = -ret;
+                        ret = -1;
+                        gf_log (this->name, GF_LOG_ERROR, "link of %s -> %s"
+                                " failed on  subvol %s (%s)", loc->name,
+                                uuid_utoa(loc->gfid),
+                                hashed_subvol->name, strerror (op_errno));
+                        if (op_errno != EEXIST)
+                                goto out;
+                }
+        }
+        ret = syncop_lookup (hashed_subvol, loc, NULL, &iatt, NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed lookup %s on %s (%s)"
+                        , loc->name, hashed_subvol->name, strerror (-ret));
+                ret = -1;
+                goto out;
+        }
+
+        if (iatt.ia_nlink == stbuf->ia_nlink) {
+                ret = dht_migrate_file (this, loc, cached_subvol, hashed_subvol,
+                                        GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS);
+                if (ret)
+                        goto out;
+        }
+        ret = -2;
+out:
+        return ret;
+}
+
+/*
+     return values
+     0 : File will be migrated
+    -2 : File will not be migrated
+         (This is the return value from gf_defrag_handle_hardlink. Checkout
+         gf_defrag_handle_hardlink for description of "returning -2")
+    -1 : failure
+*/
+static inline int
+__is_file_migratable (xlator_t *this, loc_t *loc,
+                      struct iatt *stbuf, dict_t *xattrs, int flags)
+{
+        int ret = -1;
+
+        if (IA_ISDIR (stbuf->ia_type)) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: migrate-file called on directory", loc->path);
+                ret = -1;
+                goto out;
+        }
+
+        if (flags == GF_DHT_MIGRATE_HARDLINK_IN_PROGRESS) {
+                ret = 0;
+                goto out;
+        }
+        if (stbuf->ia_nlink > 1) {
+                /* support for decomission */
+                if (flags == GF_DHT_MIGRATE_HARDLINK) {
+                        ret = gf_defrag_handle_hardlink (this, loc,
+                                                         xattrs, stbuf);
+
+                   /*
+                   Returning zero will force the file to be remigrated.
+                   Checkout gf_defrag_handle_hardlink for more information.
+                   */
+                        if (ret && ret != -2) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "%s: failed to migrate file with link",
+                                        loc->path);
+                        }
+                } else {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: file has hardlinks", loc->path);
+                        ret = -ENOTSUP;
+                }
+                goto out;
+        }
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+static inline int
+__dht_rebalance_create_dst_file (xlator_t *to, xlator_t *from, loc_t *loc, struct iatt *stbuf,
+                                 dict_t *dict, fd_t **dst_fd, dict_t *xattr)
+{
+        xlator_t    *this = NULL;
+        int          ret  = -1;
+        fd_t        *fd   = NULL;
+        struct iatt  new_stbuf = {0,};
+        dht_conf_t  *conf = NULL;
+
+        this = THIS;
+        conf = this->private;
+
+        ret = dict_set_static_bin (dict, "gfid-req", stbuf->ia_gfid, 16);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to set gfid in dict for create", loc->path);
+                goto out;
+        }
+
+        ret = dict_set_str (dict, conf->link_xattr_name, from->name);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to set gfid in dict for create", loc->path);
+                goto out;
+        }
+
+        fd = fd_create (loc->inode, DHT_REBALANCE_PID);
+        if (!fd) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: fd create failed (destination) (%s)",
+                        loc->path, strerror (errno));
+                ret = -1;
+                goto out;
+        }
+
+        ret = syncop_lookup (to, loc, NULL, &new_stbuf, NULL, NULL);
+        if (!ret) {
+                /* File exits in the destination, check if gfid matches */
+                if (uuid_compare (stbuf->ia_gfid, new_stbuf.ia_gfid) != 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "file %s exits in %s with different gfid",
+                                loc->path, to->name);
+                        fd_unref (fd);
+                        goto out;
+                }
+        }
+        if ((ret < 0) && (-ret != ENOENT)) {
+                /* File exists in destination, but not accessible */
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "%s: failed to lookup file (%s)",
+                        loc->path, strerror (-ret));
+                ret = -1;
+                goto out;
+        }
+
+        /* Create the destination with LINKFILE mode, and linkto xattr,
+           if the linkfile already exists, it will just open the file */
+        ret = syncop_create (to, loc, O_RDWR, DHT_LINKFILE_MODE, fd,
+                             dict, &new_stbuf);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create %s on %s (%s)",
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
+                goto out;
+        }
+
+        ret = syncop_fsetxattr (to, fd, xattr, 0);
+        if (ret < 0)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to set xattr on %s (%s)",
+                        loc->path, to->name, strerror (-ret));
+
+        ret = syncop_ftruncate (to, fd, stbuf->ia_size);
+        if (ret < 0)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "ftruncate failed for %s on %s (%s)",
+                        loc->path, to->name, strerror (-ret));
+
+        ret = syncop_fsetattr (to, fd, stbuf,
+                               (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
+                                NULL, NULL);
+        if (ret < 0)
+                gf_log (this->name, GF_LOG_ERROR,
+                        "chown failed for %s on %s (%s)",
+                        loc->path, to->name, strerror (-ret));
+
+        if (dst_fd)
+                *dst_fd = fd;
+
+        /* success */
+        ret = 0;
+
+out:
+        return ret;
+}
+
+static inline int
+__dht_check_free_space (xlator_t *to, xlator_t *from, loc_t *loc,
+                        struct iatt *stbuf, int flag)
+{
+        struct statvfs  src_statfs = {0,};
+        struct statvfs  dst_statfs = {0,};
+        int             ret        = -1;
+        xlator_t       *this       = NULL;
+
+        uint64_t        src_statfs_blocks = 1;
+        uint64_t        dst_statfs_blocks = 1;
+
+        this = THIS;
+
+        ret = syncop_statfs (from, loc, &src_statfs);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get statfs of %s on %s (%s)",
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
+                goto out;
+        }
+
+        ret = syncop_statfs (to, loc, &dst_statfs);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to get statfs of %s on %s (%s)",
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
+                goto out;
+        }
+
+        /* if force option is given, do not check for space @ dst.
+         * Check only if space is avail for the file */
+        if (flag != GF_DHT_MIGRATE_DATA)
+                goto check_avail_space;
+
+        /* Check:
+           During rebalance `migrate-data` - Destination subvol experiences
+           a `reduction` in 'blocks' of free space, at the same time source
+           subvol gains certain 'blocks' of free space. A valid check is
+           necessary here to avoid errorneous move to destination where
+           the space could be scantily available.
+         */
+        if (stbuf) {
+                dst_statfs_blocks = ((dst_statfs.f_bavail *
+                                      dst_statfs.f_bsize) /
+                                     GF_DISK_SECTOR_SIZE);
+                src_statfs_blocks = ((src_statfs.f_bavail *
+                                      src_statfs.f_bsize) /
+                                     GF_DISK_SECTOR_SIZE);
+                if ((dst_statfs_blocks - stbuf->ia_blocks) <
+                    (src_statfs_blocks + stbuf->ia_blocks)) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "data movement attempted from node (%s) with"
+                                " higher disk space to a node (%s) with "
+                                "lesser disk space (%s)", from->name,
+                                to->name, loc->path);
+
+                        /* this is not a 'failure', but we don't want to
+                           consider this as 'success' too :-/ */
+                        ret = 1;
+                        goto out;
+                }
+        }
+check_avail_space:
+        if (((dst_statfs.f_bavail * dst_statfs.f_bsize) /
+              GF_DISK_SECTOR_SIZE) < stbuf->ia_blocks) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "data movement attempted from node (%s) with "
+                        "to node (%s) which does not have required free space"
+                        " for %s", from->name, to->name, loc->path);
+                ret = 1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+static inline int
+__dht_rebalance_migrate_data (xlator_t *from, xlator_t *to, fd_t *src, fd_t *dst,
+                             uint64_t ia_size, int hole_exists)
+{
+        int            ret    = 0;
+        int            count  = 0;
+        off_t          offset = 0;
+        struct iovec  *vector = NULL;
+        struct iobref *iobref = NULL;
+        uint64_t       total  = 0;
+        size_t         read_size = 0;
+
+        /* if file size is '0', no need to enter this loop */
+        while (total < ia_size) {
+                read_size = (((ia_size - total) > DHT_REBALANCE_BLKSIZE) ?
+                             DHT_REBALANCE_BLKSIZE : (ia_size - total));
+                ret = syncop_readv (from, src, read_size,
+                                    offset, 0, &vector, &count, &iobref);
+                if (!ret || (ret < 0)) {
+                        break;
+                }
+
+                if (hole_exists)
+                        ret = dht_write_with_holes (to, dst, vector, count,
+                                                    ret, offset, iobref);
+                else
+                        ret = syncop_writev (to, dst, vector, count,
+                                             offset, iobref, 0);
+                if (ret < 0) {
+                        break;
+                }
+                offset += ret;
+                total += ret;
+
+                GF_FREE (vector);
+                if (iobref)
+                        iobref_unref (iobref);
+                iobref = NULL;
+                vector = NULL;
+        }
+        if (iobref)
+                iobref_unref (iobref);
+        GF_FREE (vector);
+
+        if (ret >= 0)
+                ret = 0;
+        else
+                ret = -1;
+
+        return ret;
+}
+
+
+static inline int
+__dht_rebalance_open_src_file (xlator_t *from, xlator_t *to, loc_t *loc,
+                               struct iatt *stbuf, fd_t **src_fd)
+{
+        int          ret  = 0;
+        fd_t        *fd   = NULL;
+        dict_t      *dict = NULL;
+        xlator_t    *this = NULL;
+        struct iatt  iatt = {0,};
+        dht_conf_t  *conf = NULL;
+
+        this = THIS;
+        conf = this->private;
+
+        fd = fd_create (loc->inode, DHT_REBALANCE_PID);
+        if (!fd) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: fd create failed (source)", loc->path);
+                ret = -1;
+                goto out;
+        }
+
+        ret = syncop_open (from, loc, O_RDWR, fd);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to open file %s on %s (%s)",
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
+                goto out;
+        }
+
+        ret = -1;
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_set_str (dict, conf->link_xattr_name, to->name);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to set xattr in dict for %s (linkto:%s)",
+                        loc->path, to->name);
+                goto out;
+        }
+
+        /* Once the migration starts, the source should have 'linkto' key set
+           to show which is the target, so other clients can work around it */
+        ret = syncop_setxattr (from, loc, dict, 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to set xattr on %s in %s (%s)",
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
+                goto out;
+        }
+
+        /* mode should be (+S+T) to indicate migration is in progress */
+        iatt.ia_prot = stbuf->ia_prot;
+        iatt.ia_type = stbuf->ia_type;
+        iatt.ia_prot.sticky = 1;
+        iatt.ia_prot.sgid = 1;
+
+        ret = syncop_setattr (from, loc, &iatt, GF_SET_ATTR_MODE, NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to set mode on %s in %s (%s)",
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
+                goto out;
+        }
+
+        if (src_fd)
+                *src_fd = fd;
+
+        /* success */
+        ret = 0;
+out:
+        if (dict)
+                dict_unref (dict);
+
+        return ret;
+}
+
+int
+migrate_special_files (xlator_t *this, xlator_t *from, xlator_t *to, loc_t *loc,
+                       struct iatt *buf)
+{
+        int          ret      = -1;
+        dict_t      *rsp_dict = NULL;
+        dict_t      *dict     = NULL;
+        char        *link     = NULL;
+        struct iatt  stbuf    = {0,};
+        dht_conf_t  *conf     = this->private;
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_set_int32 (dict, conf->link_xattr_name, 256);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to set 'linkto' key in dict", loc->path);
+                goto out;
+        }
+
+        /* check in the destination if the file is link file */
+        ret = syncop_lookup (to, loc, dict, &stbuf, &rsp_dict, NULL);
+        if ((ret < 0) && (-ret != ENOENT)) {
+                gf_log (this->name, GF_LOG_WARNING, "%s: lookup failed (%s)",
+                        loc->path, strerror (-ret));
+                ret = -1;
+                goto out;
+        }
+
+        /* we no more require this key */
+        dict_del (dict, conf->link_xattr_name);
+
+        /* file exists in target node, only if it is 'linkfile' its valid,
+           otherwise, error out */
+        if (!ret) {
+                if (!check_is_linkfile (loc->inode, &stbuf, rsp_dict,
+                                        conf->link_xattr_name)) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: file exists in destination", loc->path);
+                        ret = -1;
+                        goto out;
+                }
+
+                /* as file is linkfile, delete it */
+                ret = syncop_unlink (to, loc);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: failed to delete the linkfile (%s)",
+                                loc->path, strerror (-ret));
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        /* Set the gfid of the source file in dict */
+        ret = dict_set_static_bin (dict, "gfid-req", buf->ia_gfid, 16);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to set gfid in dict for create", loc->path);
+                goto out;
+        }
+
+        /* Create the file in target */
+        if (IA_ISLNK (buf->ia_type)) {
+                /* Handle symlinks separately */
+                ret = syncop_readlink (from, loc, &link, buf->ia_size);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: readlink on symlink failed (%s)",
+                                loc->path, strerror (-ret));
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = syncop_symlink (to, loc, link, dict, 0);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: creating symlink failed (%s)",
+                                loc->path, strerror (-ret));
+                        ret = -1;
+                        goto out;
+                }
+
+                goto done;
+        }
+
+        ret = syncop_mknod (to, loc, st_mode_from_ia (buf->ia_prot,
+                                                      buf->ia_type),
+                            makedev (ia_major (buf->ia_rdev),
+                                     ia_minor (buf->ia_rdev)), dict, 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "%s: mknod failed (%s)",
+                        loc->path, strerror (-ret));
+                ret = -1;
+                goto out;
+        }
+
+done:
+        ret = syncop_setattr (to, loc, buf,
+                              (GF_SET_ATTR_UID | GF_SET_ATTR_GID |
+                               GF_SET_ATTR_MODE), NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to perform setattr on %s (%s)",
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
+        }
+
+        ret = syncop_unlink (from, loc);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "%s: unlink failed (%s)",
+                        loc->path, strerror (-ret));
+                ret = -1;
+        }
+
+out:
+        if (dict)
+                dict_unref (dict);
+
+        if (rsp_dict)
+                dict_unref (rsp_dict);
+
+        return ret;
+}
+
+/*
+  return values:
+
+   -1 : failure
+    0 : successfully migrated data
+    1 : not a failure, but we can't migrate data as of now
+*/
+int
+dht_migrate_file (xlator_t *this, loc_t *loc, xlator_t *from, xlator_t *to,
+                  int flag)
+{
+        int             ret            = -1;
+        struct iatt     new_stbuf      = {0,};
+        struct iatt     stbuf          = {0,};
+        struct iatt     empty_iatt     = {0,};
+        ia_prot_t       src_ia_prot    = {0,};
+        fd_t           *src_fd         = NULL;
+        fd_t           *dst_fd         = NULL;
+        dict_t         *dict           = NULL;
+        dict_t         *xattr          = NULL;
+        dict_t         *xattr_rsp      = NULL;
+        int             file_has_holes = 0;
+        dht_conf_t     *conf           = this->private;
+
+        gf_log (this->name, GF_LOG_INFO, "%s: attempting to move from %s to %s",
+                loc->path, from->name, to->name);
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        ret = dict_set_int32 (dict, conf->link_xattr_name, 256);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to set 'linkto' key in dict", loc->path);
+                goto out;
+        }
+
+        /* Phase 1 - Data migration is in progress from now on */
+        ret = syncop_lookup (from, loc, dict, &stbuf, &xattr_rsp, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s: lookup failed on %s (%s)",
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
+                goto out;
+        }
+
+        /* we no more require this key */
+        dict_del (dict, conf->link_xattr_name);
+
+        /* preserve source mode, so set the same to the destination */
+        src_ia_prot = stbuf.ia_prot;
+
+        /* Check if file can be migrated */
+        ret = __is_file_migratable (this, loc, &stbuf, xattr_rsp, flag);
+        if (ret) {
+                if (ret == -2)
+                        ret = 0;
+                goto out;
+        }
+        /* Take care of the special files */
+        if (!IA_ISREG (stbuf.ia_type)) {
+                /* Special files */
+                ret = migrate_special_files (this, from, to, loc, &stbuf);
+                goto out;
+        }
+
+        /* TODO: move all xattr related operations to fd based operations */
+        ret = syncop_listxattr (from, loc, &xattr);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to get xattr from %s (%s)",
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
+        }
+
+        /* create the destination, with required modes/xattr */
+        ret = __dht_rebalance_create_dst_file (to, from, loc, &stbuf,
+                                               dict, &dst_fd, xattr);
+        if (ret)
+                goto out;
+
+        ret = __dht_check_free_space (to, from, loc, &stbuf, flag);
+        if (ret) {
+                goto out;
+        }
+
+        /* Open the source, and also update mode/xattr */
+        ret = __dht_rebalance_open_src_file (from, to, loc, &stbuf, &src_fd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to open %s on %s",
+                        loc->path, from->name);
+                goto out;
+        }
+
+
+        ret = syncop_fstat (from, src_fd, &stbuf);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to lookup %s on %s (%s)",
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
+                goto out;
+        }
+
+        /* Try to preserve 'holes' while migrating data */
+        if (stbuf.ia_size > (stbuf.ia_blocks * GF_DISK_SECTOR_SIZE))
+                file_has_holes = 1;
+
+        /* All I/O happens in this function */
+        ret = __dht_rebalance_migrate_data (from, to, src_fd, dst_fd,
+					    stbuf.ia_size, file_has_holes);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "%s: failed to migrate data",
+                        loc->path);
+                /* reset the destination back to 0 */
+                ret = syncop_ftruncate (to, dst_fd, 0);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "%s: failed to reset target size back to 0 (%s)",
+                                loc->path, strerror (-ret));
+                }
+
+                ret = -1;
+                goto out;
+        }
+
+        /* TODO: Sync the locks */
+
+        ret = syncop_fsync (to, dst_fd, 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to fsync on %s (%s)",
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
+        }
+
+
+        /* Phase 2 - Data-Migration Complete, Housekeeping updates pending */
+
+        ret = syncop_fstat (from, src_fd, &new_stbuf);
+        if (ret < 0) {
+                /* Failed to get the stat info */
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to fstat file %s on %s (%s)",
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
+                goto out;
+        }
+
+        /* source would have both sticky bit and sgid bit set, reset it to 0,
+           and set the source permission on destination, if it was not set
+           prior to setting rebalance-modes in source  */
+        if (!src_ia_prot.sticky)
+                new_stbuf.ia_prot.sticky = 0;
+
+        if (!src_ia_prot.sgid)
+                new_stbuf.ia_prot.sgid = 0;
+
+        /* TODO: if the source actually had sticky bit, or sgid bit set,
+           we are not handling it */
+
+        ret = syncop_fsetattr (to, dst_fd, &new_stbuf,
+                               (GF_SET_ATTR_UID | GF_SET_ATTR_GID |
+                                GF_SET_ATTR_MODE), NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to perform setattr on %s (%s)",
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
+                goto out;
+        }
+
+        /* Because 'futimes' is not portable */
+        ret = syncop_setattr (to, loc, &new_stbuf,
+                              (GF_SET_ATTR_MTIME | GF_SET_ATTR_ATIME),
+                              NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to perform setattr on %s (%s)",
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
+        }
+
+        /* Make the source as a linkfile first before deleting it */
+        empty_iatt.ia_prot.sticky = 1;
+        ret = syncop_fsetattr (from, src_fd, &empty_iatt,
+                               GF_SET_ATTR_MODE, NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,             \
+                        "%s: failed to perform setattr on %s (%s)",
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
+                goto out;
+        }
+
+       /* Free up the data blocks on the source node, as the whole
+           file is migrated */
+        ret = syncop_ftruncate (from, src_fd, 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to perform truncate on %s (%s)",
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
+        }
+
+        /* remove the 'linkto' xattr from the destination */
+        ret = syncop_fremovexattr (to, dst_fd, conf->link_xattr_name, 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to perform removexattr on %s (%s)",
+                        loc->path, to->name, strerror (-ret));
+                ret = -1;
+        }
+
+        /* Do a stat and check the gfid before unlink */
+        ret = syncop_stat (from, loc, &empty_iatt);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: failed to do a stat on %s (%s)",
+                        loc->path, from->name, strerror (-ret));
+                ret = -1;
+                goto out;
+        }
+
+        if (uuid_compare (empty_iatt.ia_gfid, loc->gfid) == 0) {
+                /* take out the source from namespace */
+                ret = syncop_unlink (from, loc);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: failed to perform unlink on %s (%s)",
+                                loc->path, from->name, strerror (-ret));
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        ret = syncop_lookup (this, loc, NULL, NULL, NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "%s: failed to lookup the file on subvolumes (%s)",
+                        loc->path, strerror (-ret));
+                ret = -1;
+        }
+
+        gf_log (this->name, GF_LOG_INFO,
+                "completed migration of %s from subvolume %s to %s",
+                loc->path, from->name, to->name);
+
+        ret = 0;
+out:
+        if (dict)
+                dict_unref (dict);
+
+        if (xattr)
+                dict_unref (xattr);
+        if (xattr_rsp)
+                dict_unref (xattr_rsp);
+
+        if (dst_fd)
+                syncop_close (dst_fd);
+        if (src_fd)
+                syncop_close (src_fd);
+
+        return ret;
+}
+
+static int
+rebalance_task (void *data)
+{
+        int           ret   = -1;
+        dht_local_t  *local = NULL;
+        call_frame_t *frame = NULL;
+
+        frame = data;
+
+        local = frame->local;
+
+        /* This function is 'synchrounous', hence if it returns,
+           we are done with the task */
+        ret = dht_migrate_file (THIS, &local->loc, local->rebalance.from_subvol,
+                                local->rebalance.target_node, local->flags);
+
+        return ret;
+}
+
+static int
+rebalance_task_completion (int op_ret, call_frame_t *sync_frame, void *data)
+{
+        int           ret        = -1;
+        uint64_t      layout_int = 0;
+        dht_layout_t *layout     = 0;
+        xlator_t     *this       = NULL;
+        dht_local_t  *local      = NULL;
+        int32_t       op_errno   = EINVAL;
+
+        this = THIS;
+        local = sync_frame->local;
+
+        if (!op_ret) {
+                /* Make sure we have valid 'layout' in inode ctx
+                   after the operation */
+                ret = inode_ctx_del (local->loc.inode, this, &layout_int);
+                if (!ret && layout_int) {
+                        layout = (dht_layout_t *)(long)layout_int;
+                        dht_layout_unref (this, layout);
+                }
+
+                ret = dht_layout_preset (this, local->rebalance.target_node,
+                                         local->loc.inode);
+                if (ret)
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: failed to set inode ctx", local->loc.path);
+        }
+
+        if (op_ret == -1) {
+                /* Failure of migration process, mostly due to write process.
+                   as we can't preserve the exact errno, lets say there was
+                   no space to migrate-data
+                */
+                op_errno = ENOSPC;
+        }
+
+        if (op_ret == 1) {
+                /* migration didn't happen, but is not a failure, let the user
+                   understand that he doesn't have permission to migrate the
+                   file.
+                */
+                op_ret = -1;
+                op_errno = EPERM;
+        }
+
+        DHT_STACK_UNWIND (setxattr, sync_frame, op_ret, op_errno, NULL);
+        return 0;
+}
+
+int
+dht_start_rebalance_task (xlator_t *this, call_frame_t *frame)
+{
+        int         ret     = -1;
+
+        ret = synctask_new (this->ctx->env, rebalance_task,
+                            rebalance_task_completion,
+                            frame, frame);
+        return ret;
+}
+
+int
+gf_listener_stop (xlator_t *this)
+{
+        glusterfs_ctx_t  *ctx = NULL;
+        cmd_args_t       *cmd_args = NULL;
+        int              ret = 0;
+
+        ctx = this->ctx;
+        GF_ASSERT (ctx);
+        cmd_args = &ctx->cmd_args;
+        if (cmd_args->sock_file) {
+                ret = unlink (cmd_args->sock_file);
+                if (ret && (ENOENT == errno)) {
+                        ret = 0;
+                }
+        }
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to unlink listener "
+                        "socket %s, error: %s", cmd_args->sock_file,
+                        strerror (errno));
+        }
+        return ret;
+}
+
+void
+dht_build_root_inode (xlator_t *this, inode_t **inode)
+{
+        inode_table_t    *itable        = NULL;
+        uuid_t            root_gfid     = {0, };
+
+        itable = inode_table_new (0, this);
+        if (!itable)
+                return;
+
+        root_gfid[15] = 1;
+        *inode = inode_find (itable, root_gfid);
+}
+
+void
+dht_build_root_loc (inode_t *inode, loc_t *loc)
+{
+        loc->path = "/";
+        loc->inode = inode;
+        loc->inode->ia_type = IA_IFDIR;
+        memset (loc->gfid, 0, 16);
+        loc->gfid[15] = 1;
+}
+
+
+/* return values: 1 -> error, bug ignore and continue
+                  0 -> proceed
+                 -1 -> error, handle it */
+int32_t
+gf_defrag_handle_migrate_error (int32_t op_errno, gf_defrag_info_t *defrag)
+{
+        /* if errno is not ENOSPC or ENOTCONN, we can still continue
+           with rebalance process */
+        if ((op_errno != ENOSPC) || (op_errno != ENOTCONN))
+                return 1;
+
+        if (op_errno == ENOTCONN) {
+                /* Most probably mount point went missing (mostly due
+                   to a brick down), say rebalance failure to user,
+                   let him restart it if everything is fine */
+                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+                return -1;
+        }
+
+        if (op_errno == ENOSPC) {
+                /* rebalance process itself failed, may be
+                   remote brick went down, or write failed due to
+                   disk full etc etc.. */
+                defrag->defrag_status = GF_DEFRAG_STATUS_FAILED;
+                return -1;
+        }
+
+        return 0;
+}
+
+static gf_boolean_t
+gf_defrag_pattern_match (gf_defrag_info_t *defrag, char *name, uint64_t size)
+{
+        gf_defrag_pattern_list_t *trav = NULL;
+        gf_boolean_t               match = _gf_false;
+        gf_boolean_t               ret = _gf_false;
+
+        GF_VALIDATE_OR_GOTO ("dht", defrag, out);
+
+        trav = defrag->defrag_pattern;
+        while (trav) {
+                if (!fnmatch (trav->path_pattern, name, FNM_NOESCAPE)) {
+                        match = _gf_true;
+                        break;
+                }
+                trav = trav->next;
+        }
+
+        if ((match == _gf_true) && (size >= trav->size))
+                ret = _gf_true;
+
+ out:
+        return ret;
+}
+
+/* We do a depth first traversal of directories. But before we move into
+ * subdirs, we complete the data migration of those directories whose layouts
+ * have been fixed
+ */
+
+int
+gf_defrag_migrate_data (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
+                        dict_t *migrate_data)
+{
+        int                      ret            = -1;
+        loc_t                    entry_loc      = {0,};
+        fd_t                    *fd             = NULL;
+        gf_dirent_t              entries;
+        gf_dirent_t             *tmp            = NULL;
+        gf_dirent_t             *entry          = NULL;
+        gf_boolean_t             free_entries   = _gf_false;
+        off_t                    offset         = 0;
+        dict_t                  *dict           = NULL;
+        struct iatt              iatt           = {0,};
+        int32_t                  op_errno       = 0;
+        char                    *uuid_str       = NULL;
+        uuid_t                   node_uuid      = {0,};
+        struct timeval           dir_start      = {0,};
+        struct timeval           end            = {0,};
+        double                   elapsed        = {0,};
+        struct timeval           start          = {0,};
+        int32_t                  err            = 0;
+        int                      loglevel       = GF_LOG_TRACE;
+
+        gf_log (this->name, GF_LOG_INFO, "migrate data called on %s",
+                loc->path);
+        gettimeofday (&dir_start, NULL);
+
+        fd = fd_create (loc->inode, defrag->pid);
+        if (!fd) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create fd");
+                goto out;
+        }
+
+        ret = syncop_opendir (this, loc, fd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to open dir %s",
+                        loc->path);
+                ret = -1;
+                goto out;
+        }
+
+        INIT_LIST_HEAD (&entries.list);
+
+        while ((ret = syncop_readdirp (this, fd, 131072, offset, NULL,
+                                       &entries)) != 0) {
+
+                if (ret < 0) {
+
+                        gf_log (this->name, GF_LOG_ERROR, "Readdir returned %s."
+                                " Aborting migrate-data",
+                                strerror(-ret));
+                        ret = -1;
+                        goto out;
+                }
+
+                if (list_empty (&entries.list))
+                        break;
+
+                free_entries = _gf_true;
+
+                list_for_each_entry_safe (entry, tmp, &entries.list, list) {
+                        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+                                ret = 1;
+                                goto out;
+                        }
+
+                        offset = entry->d_off;
+
+                        if (!strcmp (entry->d_name, ".") ||
+                            !strcmp (entry->d_name, ".."))
+                                continue;
+
+                        if (IA_ISDIR (entry->d_stat.ia_type))
+                                continue;
+
+                        defrag->num_files_lookedup++;
+                        if (defrag->stats == _gf_true) {
+                                gettimeofday (&start, NULL);
+                        }
+                        if (defrag->defrag_pattern &&
+                            (gf_defrag_pattern_match (defrag, entry->d_name,
+                                                      entry->d_stat.ia_size)
+                             == _gf_false)) {
+                                continue;
+                        }
+                        loc_wipe (&entry_loc);
+                        ret =dht_build_child_loc (this, &entry_loc, loc,
+                                                  entry->d_name);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Child loc"
+                                        " build failed");
+                                goto out;
+                        }
+
+                        if (uuid_is_null (entry->d_stat.ia_gfid)) {
+                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
+                                        " gfid not present", loc->path,
+                                         entry->d_name);
+                                continue;
+                        }
+
+                        uuid_copy (entry_loc.gfid, entry->d_stat.ia_gfid);
+
+                        if (uuid_is_null (loc->gfid)) {
+                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
+                                        " gfid not present", loc->path,
+                                         entry->d_name);
+                                continue;
+                        }
+
+                        uuid_copy (entry_loc.pargfid, loc->gfid);
+
+                        entry_loc.inode->ia_type = entry->d_stat.ia_type;
+
+                        ret = syncop_lookup (this, &entry_loc, NULL, &iatt,
+                                             NULL, NULL);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "%s"
+                                        " lookup failed", entry_loc.path);
+                                ret = -1;
+                                continue;
+                        }
+
+                        ret = syncop_getxattr (this, &entry_loc, &dict,
+                                               GF_XATTR_NODE_UUID_KEY);
+                        if(ret < 0) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "get node-uuid for %s", entry_loc.path);
+                                ret = -1;
+                                continue;
+                        }
+
+                        ret = dict_get_str (dict, GF_XATTR_NODE_UUID_KEY,
+                                            &uuid_str);
+                        if(ret < 0) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "get node-uuid from dict for %s",
+                                        entry_loc.path);
+                                ret = -1;
+                                continue;
+                        }
+
+                        if (uuid_parse (uuid_str, node_uuid)) {
+                                gf_log (this->name, GF_LOG_ERROR, "uuid_parse "
+                                        "failed for %s", entry_loc.path);
+                                continue;
+                        }
+
+                        /* if file belongs to different node, skip migration
+                         * the other node will take responsibility of migration
+                         */
+                        if (uuid_compare (node_uuid, defrag->node_uuid)) {
+                                gf_log (this->name, GF_LOG_TRACE, "%s does not"
+                                        "belong to this node", entry_loc.path);
+                                continue;
+                        }
+
+                        uuid_str = NULL;
+
+                        dict_del (dict, GF_XATTR_NODE_UUID_KEY);
+
+
+                        /* if distribute is present, it will honor this key.
+                         * -1, ENODATA is returned if distribute is not present
+                         * or file doesn't have a link-file. If file has
+                         * link-file, the path of link-file will be the value,
+                         * and also that guarantees that file has to be mostly
+                         * migrated */
+
+                        ret = syncop_getxattr (this, &entry_loc, &dict,
+                                               GF_XATTR_LINKINFO_KEY);
+                        if (ret < 0) {
+                                if (-ret != ENODATA) {
+                                        loglevel = GF_LOG_ERROR;
+                                        defrag->total_failures += 1;
+                                } else {
+                                        loglevel = GF_LOG_TRACE;
+                                }
+                                gf_log (this->name, loglevel, "%s: failed to "
+                                        "get "GF_XATTR_LINKINFO_KEY" key - %s",
+                                        entry_loc.path, strerror (-ret));
+                                ret = -1;
+                                continue;
+                        }
+
+                        ret = syncop_setxattr (this, &entry_loc, migrate_data,
+                                               0);
+                        if (ret) {
+                                err = op_errno;
+                                /* errno is overloaded. See
+                                 * rebalance_task_completion () */
+                                if (err != ENOSPC) {
+                                        gf_log (this->name, GF_LOG_DEBUG,
+                                                "migrate-data skipped for %s"
+                                                " due to space constraints",
+                                                entry_loc.path);
+                                        defrag->skipped +=1;
+                                } else{
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "migrate-data failed for %s",
+                                                entry_loc.path);
+                                        defrag->total_failures +=1;
+                                }
+                        }
+
+                        if (ret < 0) {
+                                op_errno = -ret;
+                                ret = gf_defrag_handle_migrate_error (op_errno,
+                                                                      defrag);
+
+                                if (!ret)
+                                        gf_log (this->name, GF_LOG_DEBUG,
+                                                "migrate-data on %s failed: %s",
+                                                entry_loc.path,
+                                                strerror (op_errno));
+                                else if (ret == 1)
+                                        continue;
+                                else if (ret == -1)
+                                        goto out;
+                        }
+
+                        LOCK (&defrag->lock);
+                        {
+                                defrag->total_files += 1;
+                                defrag->total_data += iatt.ia_size;
+                        }
+                        UNLOCK (&defrag->lock);
+                        if (defrag->stats == _gf_true) {
+                                gettimeofday (&end, NULL);
+                                elapsed = (end.tv_sec - start.tv_sec) * 1e6 +
+                                          (end.tv_usec - start.tv_usec);
+                                gf_log (this->name, GF_LOG_INFO, "Migration of "
+                                        "file:%s size:%"PRIu64" bytes took %.2f"
+                                        "secs", entry_loc.path, iatt.ia_size,
+                                         elapsed/1e6);
+                        }
+                }
+
+                gf_dirent_free (&entries);
+                free_entries = _gf_false;
+                INIT_LIST_HEAD (&entries.list);
+        }
+
+        gettimeofday (&end, NULL);
+        elapsed = (end.tv_sec - dir_start.tv_sec) * 1e6 +
+                  (end.tv_usec - dir_start.tv_usec);
+        gf_log (this->name, GF_LOG_INFO, "Migration operation on dir %s took "
+                "%.2f secs", loc->path, elapsed/1e6);
+        ret = 0;
+out:
+        if (free_entries)
+                gf_dirent_free (&entries);
+
+        loc_wipe (&entry_loc);
+
+        if (dict)
+                dict_unref(dict);
+
+        if (fd)
+                fd_unref (fd);
+        return ret;
+
+}
+
+int
+gf_defrag_fix_layout (xlator_t *this, gf_defrag_info_t *defrag, loc_t *loc,
+                  dict_t *fix_layout, dict_t *migrate_data)
+{
+        int                      ret            = -1;
+        loc_t                    entry_loc      = {0,};
+        fd_t                    *fd             = NULL;
+        gf_dirent_t              entries;
+        gf_dirent_t             *tmp            = NULL;
+        gf_dirent_t             *entry          = NULL;
+        gf_boolean_t             free_entries   = _gf_false;
+        dict_t                  *dict           = NULL;
+        off_t                    offset         = 0;
+        struct iatt              iatt           = {0,};
+
+        ret = syncop_lookup (this, loc, NULL, &iatt, NULL, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Lookup failed on %s",
+                        loc->path);
+                ret = -1;
+                goto out;
+        }
+
+        if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+                ret = gf_defrag_migrate_data (this, defrag, loc, migrate_data);
+                if (ret)
+                        goto out;
+        }
+
+        gf_log (this->name, GF_LOG_TRACE, "fix layout called on %s", loc->path);
+
+        fd = fd_create (loc->inode, defrag->pid);
+        if (!fd) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to create fd");
+                ret = -1;
+                goto out;
+        }
+
+        ret = syncop_opendir (this, loc, fd);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to open dir %s",
+                        loc->path);
+                ret = -1;
+                goto out;
+        }
+
+        INIT_LIST_HEAD (&entries.list);
+        while ((ret = syncop_readdirp (this, fd, 131072, offset, NULL,
+                &entries)) != 0)
+        {
+
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR, "Readdir returned %s"
+                                ". Aborting fix-layout",strerror(-ret));
+                        ret = -1;
+                        goto out;
+                }
+
+                if (list_empty (&entries.list))
+                        break;
+
+                free_entries = _gf_true;
+
+                list_for_each_entry_safe (entry, tmp, &entries.list, list) {
+                        if (defrag->defrag_status != GF_DEFRAG_STATUS_STARTED) {
+                                ret = 1;
+                                goto out;
+                        }
+
+                        offset = entry->d_off;
+
+                        if (!strcmp (entry->d_name, ".") ||
+                            !strcmp (entry->d_name, ".."))
+                                continue;
+
+                        if (!IA_ISDIR (entry->d_stat.ia_type))
+                                continue;
+
+                        loc_wipe (&entry_loc);
+                        ret =dht_build_child_loc (this, &entry_loc, loc,
+                                                  entry->d_name);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Child loc"
+                                        " build failed");
+                                goto out;
+                        }
+
+                        if (uuid_is_null (entry->d_stat.ia_gfid)) {
+                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
+                                        " gfid not present", loc->path,
+                                         entry->d_name);
+                                continue;
+                        }
+
+                        entry_loc.inode->ia_type = entry->d_stat.ia_type;
+
+                        uuid_copy (entry_loc.gfid, entry->d_stat.ia_gfid);
+                        if (uuid_is_null (loc->gfid)) {
+                                gf_log (this->name, GF_LOG_ERROR, "%s/%s"
+                                        " gfid not present", loc->path,
+                                         entry->d_name);
+                                continue;
+                        }
+
+                        uuid_copy (entry_loc.pargfid, loc->gfid);
+
+                        ret = syncop_lookup (this, &entry_loc, NULL, &iatt,
+                                             NULL, NULL);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "%s"
+                                        " lookup failed", entry_loc.path);
+                                ret = -1;
+                                continue;
+                        }
+
+                        ret = syncop_setxattr (this, &entry_loc, fix_layout,
+                                               0);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Setxattr "
+                                        "failed for %s", entry_loc.path);
+                                defrag->defrag_status =
+                                GF_DEFRAG_STATUS_FAILED;
+                                defrag->total_failures ++;
+                                ret = -1;
+                                goto out;
+                        }
+                        ret = gf_defrag_fix_layout (this, defrag, &entry_loc,
+                                                    fix_layout, migrate_data);
+
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Fix layout "
+                                        "failed for %s", entry_loc.path);
+                                defrag->total_failures++;
+                                goto out;
+                        }
+
+                }
+                gf_dirent_free (&entries);
+                free_entries = _gf_false;
+                INIT_LIST_HEAD (&entries.list);
+        }
+
+        ret = 0;
+out:
+        if (free_entries)
+                gf_dirent_free (&entries);
+
+        loc_wipe (&entry_loc);
+
+        if (dict)
+                dict_unref(dict);
+
+        if (fd)
+                fd_unref (fd);
+
+        return ret;
+
+}
+
+
+int
+gf_defrag_start_crawl (void *data)
+{
+        xlator_t                *this   = NULL;
+        dht_conf_t              *conf   = NULL;
+        gf_defrag_info_t        *defrag = NULL;
+        int                      ret    = -1;
+        loc_t                    loc    = {0,};
+        struct iatt              iatt   = {0,};
+        struct iatt              parent = {0,};
+        dict_t                  *fix_layout = NULL;
+        dict_t                  *migrate_data = NULL;
+        dict_t                  *status = NULL;
+        glusterfs_ctx_t         *ctx = NULL;
+
+        this = data;
+        if (!this)
+                goto out;
+
+        ctx = this->ctx;
+        if (!ctx)
+                goto out;
+
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        defrag = conf->defrag;
+        if (!defrag)
+                goto out;
+
+        gettimeofday (&defrag->start_time, NULL);
+        dht_build_root_inode (this, &defrag->root_inode);
+        if (!defrag->root_inode)
+                goto out;
+
+        dht_build_root_loc (defrag->root_inode, &loc);
+
+        /* fix-layout on '/' first */
+
+        ret = syncop_lookup (this, &loc, NULL, &iatt, NULL, &parent);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "look up on / failed");
+                ret = -1;
+                goto out;
+        }
+
+        fix_layout = dict_new ();
+        if (!fix_layout) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_set_str (fix_layout, GF_XATTR_FIX_LAYOUT_KEY, "yes");
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Failed to set dict str");
+                goto out;
+        }
+
+        ret = syncop_setxattr (this, &loc, fix_layout, 0);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "fix layout on %s failed",
+                        loc.path);
+                defrag->total_failures++;
+                ret = -1;
+                goto out;
+        }
+
+        if (defrag->cmd != GF_DEFRAG_CMD_START_LAYOUT_FIX) {
+                migrate_data = dict_new ();
+                if (!migrate_data) {
+                        ret = -1;
+                        goto out;
+                }
+                if (defrag->cmd == GF_DEFRAG_CMD_START_FORCE)
+                        ret = dict_set_str (migrate_data,
+                                            "distribute.migrate-data", "force");
+                else
+                        ret = dict_set_str (migrate_data,
+                                            "distribute.migrate-data",
+                                            "non-force");
+                if (ret)
+                        goto out;
+        }
+        ret = gf_defrag_fix_layout (this, defrag, &loc, fix_layout,
+                                    migrate_data);
+        if ((defrag->defrag_status != GF_DEFRAG_STATUS_STOPPED) &&
+            (defrag->defrag_status != GF_DEFRAG_STATUS_FAILED)) {
+                defrag->defrag_status = GF_DEFRAG_STATUS_COMPLETE;
+        }
+
+
+
+out:
+        LOCK (&defrag->lock);
+        {
+                status = dict_new ();
+                gf_defrag_status_get (defrag, status);
+                if (ctx->notify)
+                        ctx->notify (GF_EN_DEFRAG_STATUS, status);
+                if (status)
+                        dict_unref (status);
+                defrag->is_exiting = 1;
+        }
+        UNLOCK (&defrag->lock);
+
+        if (defrag) {
+                GF_FREE (defrag);
+                conf->defrag = NULL;
+        }
+
+        return ret;
+}
+
+
+static int
+gf_defrag_done  (int ret, call_frame_t *sync_frame, void *data)
+{
+        gf_listener_stop (sync_frame->this);
+
+        STACK_DESTROY (sync_frame->root);
+        kill (getpid(), SIGTERM);
+        return 0;
+}
+
+void *
+gf_defrag_start (void *data)
+{
+        int                      ret    = -1;
+        call_frame_t            *frame  = NULL;
+        dht_conf_t              *conf   = NULL;
+        gf_defrag_info_t        *defrag = NULL;
+        xlator_t                *this  = NULL;
+
+        this = data;
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        defrag = conf->defrag;
+        if (!defrag)
+                goto out;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame)
+                goto out;
+
+        frame->root->pid = GF_CLIENT_PID_DEFRAG;
+
+        defrag->pid = frame->root->pid;
+
+        defrag->defrag_status = GF_DEFRAG_STATUS_STARTED;
+
+        ret = synctask_new (this->ctx->env, gf_defrag_start_crawl,
+                            gf_defrag_done, frame, this);
+
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Could not create"
+                        " task for rebalance");
+out:
+        return NULL;
+}
+
+int
+gf_defrag_status_get (gf_defrag_info_t *defrag, dict_t *dict)
+{
+        int      ret    = 0;
+        uint64_t files  = 0;
+        uint64_t size   = 0;
+        uint64_t lookup = 0;
+        uint64_t failures = 0;
+        uint64_t skipped = 0;
+        char     *status = "";
+        double   elapsed = 0;
+        struct timeval end = {0,};
+
+
+        if (!defrag)
+                goto out;
+
+        ret = 0;
+        if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED)
+                goto out;
+
+        files  = defrag->total_files;
+        size   = defrag->total_data;
+        lookup = defrag->num_files_lookedup;
+        failures = defrag->total_failures;
+        skipped = defrag->skipped;
+
+        gettimeofday (&end, NULL);
+
+        elapsed = end.tv_sec - defrag->start_time.tv_sec;
+
+        if (!dict)
+                goto log;
+
+        ret = dict_set_uint64 (dict, "files", files);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set file count");
+
+        ret = dict_set_uint64 (dict, "size", size);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set size of xfer");
+
+        ret = dict_set_uint64 (dict, "lookups", lookup);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set lookedup file count");
+
+
+        ret = dict_set_int32 (dict, "status", defrag->defrag_status);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set status");
+        if (elapsed) {
+                ret = dict_set_double (dict, "run-time", elapsed);
+                if (ret)
+                        gf_log (THIS->name, GF_LOG_WARNING,
+                                "failed to set run-time");
+        }
+
+        ret = dict_set_uint64 (dict, "failures", failures);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set failure count");
+
+        ret = dict_set_uint64 (dict, "skipped", skipped);
+        if (ret)
+                gf_log (THIS->name, GF_LOG_WARNING,
+                        "failed to set skipped file count");
+log:
+        switch (defrag->defrag_status) {
+        case GF_DEFRAG_STATUS_NOT_STARTED:
+                status = "not started";
+                break;
+        case GF_DEFRAG_STATUS_STARTED:
+                status = "in progress";
+                break;
+        case GF_DEFRAG_STATUS_STOPPED:
+                status = "stopped";
+                break;
+        case GF_DEFRAG_STATUS_COMPLETE:
+                status = "completed";
+                break;
+        case GF_DEFRAG_STATUS_FAILED:
+                status = "failed";
+                break;
+        default:
+                break;
+        }
+
+        gf_log (THIS->name, GF_LOG_INFO, "Rebalance is %s. Time taken is %.2f "
+                "secs", status, elapsed);
+        gf_log (THIS->name, GF_LOG_INFO, "Files migrated: %"PRIu64", size: %"
+                PRIu64", lookups: %"PRIu64", failures: %"PRIu64", skipped: "
+                "%"PRIu64, files, size, lookup, failures, skipped);
+
+
+out:
+        return 0;
+}
+
+int
+gf_defrag_stop (gf_defrag_info_t *defrag, gf_defrag_status_t status,
+                dict_t *output)
+{
+        /* TODO: set a variable 'stop_defrag' here, it should be checked
+           in defrag loop */
+        int     ret = -1;
+        GF_ASSERT (defrag);
+
+        if (defrag->defrag_status == GF_DEFRAG_STATUS_NOT_STARTED) {
+                goto out;
+        }
+
+        gf_log ("", GF_LOG_INFO, "Received stop command on rebalance");
+        defrag->defrag_status = status;
+
+        if (output)
+                gf_defrag_status_get (defrag, output);
+        ret = 0;
+out:
+        gf_log ("", GF_LOG_DEBUG, "Returning %d", ret);
+        return ret;
+}
diff --git a/xlators/cluster/dht/src/dht-rename.c b/xlators/cluster/dht/src/dht-rename.c
index dcdd5c7ec..925538cc8 100644
--- a/xlators/cluster/dht/src/dht-rename.c
+++ b/xlators/cluster/dht/src/dht-rename.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 /* TODO: link(oldpath, newpath) fails if newpath already exists. DHT should
@@ -33,232 +24,348 @@
 
 int
 dht_rename_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                    int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
                     struct iatt *preoldparent, struct iatt *postoldparent,
-                    struct iatt *prenewparent, struct iatt *postnewparent)
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = 0;
-	call_frame_t *prev = NULL;
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = 0;
+        call_frame_t *prev = NULL;
 
+        local = frame->local;
+        prev = cookie;
 
-	local = frame->local;
-	prev = cookie;
-
-	if (op_ret == -1) {
-		/* TODO: undo the damage */
-
-		gf_log (this->name, GF_LOG_DEBUG,
-			"rename %s -> %s on %s failed (%s)",
-			local->loc.path, local->loc2.path,
-			prev->this->name, strerror (op_errno));
-
-		local->op_ret   = op_ret;
-		local->op_errno = op_errno;
-	} else {
-		/* TODO: construct proper stbuf for dir */
-                /* 
-                 * FIXME: is this the correct way to build stbuf and
-                 * parent bufs?
-                */
-                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-                dht_iatt_merge (this, &local->preoldparent, preoldparent,
-                                prev->this);
-                dht_iatt_merge (this, &local->postoldparent, postoldparent,
-                                prev->this);
-                dht_iatt_merge (this, &local->preparent, prenewparent,
-                                prev->this);
-                dht_iatt_merge (this, &local->postparent, postnewparent,
-                                prev->this);
-	}
+        if (op_ret == -1) {
+                /* TODO: undo the damage */
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt)) {
-                local->stbuf.ia_ino = local->loc.inode->ino;
+                gf_log (this->name, GF_LOG_INFO,
+                        "rename %s -> %s on %s failed (%s)",
+                        local->loc.path, local->loc2.path,
+                        prev->this->name, strerror (op_errno));
 
-                local->preoldparent.ia_ino = local->loc.parent->ino;
-                local->postoldparent.ia_ino = local->loc.parent->ino;
+                local->op_ret   = op_ret;
+                local->op_errno = op_errno;
+                goto unwind;
+        }
+        /* TODO: construct proper stbuf for dir */
+        /*
+         * FIXME: is this the correct way to build stbuf and
+         * parent bufs?
+         */
+        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+        dht_iatt_merge (this, &local->preoldparent, preoldparent,
+                        prev->this);
+        dht_iatt_merge (this, &local->postoldparent, postoldparent,
+                        prev->this);
+        dht_iatt_merge (this, &local->preparent, prenewparent,
+                        prev->this);
+        dht_iatt_merge (this, &local->postparent, postnewparent,
+                        prev->this);
 
-                local->preparent.ia_ino = local->loc2.parent->ino;
-                local->postparent.ia_ino = local->loc2.parent->ino;
 
+unwind:
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
                 WIPE (&local->preoldparent);
                 WIPE (&local->postoldparent);
                 WIPE (&local->preparent);
                 WIPE (&local->postparent);
 
-		DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-				  &local->stbuf, &local->preoldparent,
+                DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+                DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
+                                  &local->stbuf, &local->preoldparent,
                                   &local->postoldparent,
-                                  &local->preparent, &local->postparent);
-	}
+                                  &local->preparent, &local->postparent, xdata);
+        }
 
-	return 0;
+        return 0;
 }
 
 
+int
+dht_rename_hashed_dir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                           struct iatt *preoldparent,
+                           struct iatt *postoldparent,
+                           struct iatt *prenewparent,
+                           struct iatt *postnewparent, dict_t *xdata)
+{
+        dht_conf_t   *conf = NULL;
+        dht_local_t  *local = NULL;
+        int           call_cnt = 0;
+        call_frame_t *prev = NULL;
+        int           i = 0;
+
+        conf = this->private;
+        local = frame->local;
+        prev = cookie;
+
+        if (op_ret == -1) {
+                /* TODO: undo the damage */
+
+                gf_log (this->name, GF_LOG_INFO,
+                        "rename %s -> %s on %s failed (%s)",
+                        local->loc.path, local->loc2.path,
+                        prev->this->name, strerror (op_errno));
+
+                local->op_ret   = op_ret;
+                local->op_errno = op_errno;
+                goto unwind;
+        }
+        /* TODO: construct proper stbuf for dir */
+        /*
+         * FIXME: is this the correct way to build stbuf and
+         * parent bufs?
+         */
+        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+        dht_iatt_merge (this, &local->preoldparent, preoldparent,
+                        prev->this);
+        dht_iatt_merge (this, &local->postoldparent, postoldparent,
+                        prev->this);
+        dht_iatt_merge (this, &local->preparent, prenewparent,
+                        prev->this);
+        dht_iatt_merge (this, &local->postparent, postnewparent,
+                        prev->this);
+
+        call_cnt = local->call_cnt = conf->subvolume_cnt - 1;
+
+        if (!local->call_cnt)
+                goto unwind;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (conf->subvolumes[i] == local->dst_hashed)
+                        continue;
+                STACK_WIND (frame, dht_rename_dir_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->rename,
+                            &local->loc, &local->loc2, NULL);
+                if (!--call_cnt)
+                        break;
+        }
+
+
+        return 0;
+unwind:
+        WIPE (&local->preoldparent);
+        WIPE (&local->postoldparent);
+        WIPE (&local->preparent);
+        WIPE (&local->postparent);
+
+        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
+                          &local->stbuf, &local->preoldparent,
+                          &local->postoldparent,
+                          &local->preparent, &local->postparent, NULL);
+
+        return 0;
+}
+
 
 int
 dht_rename_dir_do (call_frame_t *frame, xlator_t *this)
 {
-	dht_local_t  *local = NULL;
-	dht_conf_t   *conf = NULL;
-	int           i = 0;
+        dht_local_t  *local = NULL;
 
-	conf = this->private;
-	local = frame->local;
+        local = frame->local;
 
-	if (local->op_ret == -1)
-		goto err;
+        if (local->op_ret == -1)
+                goto err;
 
-	local->call_cnt = conf->subvolume_cnt;
-	local->op_ret = 0;
+        local->op_ret = 0;
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		STACK_WIND (frame, dht_rename_dir_cbk,
-			    conf->subvolumes[i],
-			    conf->subvolumes[i]->fops->rename,
-			    &local->loc, &local->loc2);
-	}
-
-	return 0;
+        STACK_WIND (frame, dht_rename_hashed_dir_cbk,
+                    local->dst_hashed,
+                    local->dst_hashed->fops->rename,
+                    &local->loc, &local->loc2, NULL);
+        return 0;
 
 err:
-	DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno, NULL, NULL,
-                          NULL, NULL, NULL);
-	return 0;
+        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno, NULL, NULL,
+                          NULL, NULL, NULL, NULL);
+        return 0;
 }
 
 
 int
 dht_rename_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int op_ret, int op_errno, gf_dirent_t *entries)
+                        int op_ret, int op_errno, gf_dirent_t *entries,
+                        dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = -1;
-	call_frame_t *prev = NULL;
-
-	local = frame->local;
-	prev  = cookie;
-
-	if (op_ret > 2) {
-		gf_log (this->name, GF_LOG_TRACE,
-			"readdir on %s for %s returned %d entries",
-			prev->this->name, local->loc.path, op_ret);
-		local->op_ret = -1;
-		local->op_errno = ENOTEMPTY;
-	}
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = -1;
+        call_frame_t *prev = NULL;
+
+        local = frame->local;
+        prev  = cookie;
+
+        if (op_ret > 2) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "readdir on %s for %s returned %d entries",
+                        prev->this->name, local->loc.path, op_ret);
+                local->op_ret = -1;
+                local->op_errno = ENOTEMPTY;
+        }
 
-	this_call_cnt = dht_frame_return (frame);
+        this_call_cnt = dht_frame_return (frame);
 
-	if (is_last_call (this_call_cnt)) {
-		dht_rename_dir_do (frame, this);
-	}
+        if (is_last_call (this_call_cnt)) {
+                dht_rename_dir_do (frame, this);
+        }
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_rename_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int op_ret, int op_errno, fd_t *fd)
+                        int op_ret, int op_errno, fd_t *fd, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	int           this_call_cnt = -1;
-	call_frame_t *prev = NULL;
+        dht_local_t  *local = NULL;
+        int           this_call_cnt = -1;
+        call_frame_t *prev = NULL;
 
 
-	local = frame->local;
-	prev  = cookie;
+        local = frame->local;
+        prev  = cookie;
 
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"opendir on %s for %s failed (%s)",
-			prev->this->name, local->loc.path,
-			strerror (op_errno));
-		goto err;
-	}
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "opendir on %s for %s failed (%s)",
+                        prev->this->name, local->loc.path,
+                        strerror (op_errno));
+                goto err;
+        }
 
-	STACK_WIND (frame, dht_rename_readdir_cbk,
-		    prev->this, prev->this->fops->readdir,
-		    local->fd, 4096, 0);
+        STACK_WIND (frame, dht_rename_readdir_cbk,
+                    prev->this, prev->this->fops->readdir,
+                    local->fd, 4096, 0, NULL);
 
-	return 0;
+        return 0;
 
 err:
-	this_call_cnt = dht_frame_return (frame);
+        this_call_cnt = dht_frame_return (frame);
 
-	if (is_last_call (this_call_cnt)) {
-		dht_rename_dir_do (frame, this);
-	}
+        if (is_last_call (this_call_cnt)) {
+                dht_rename_dir_do (frame, this);
+        }
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_rename_dir (call_frame_t *frame, xlator_t *this)
 {
-	dht_conf_t  *conf = NULL;
-	dht_local_t *local = NULL;
-	int          i = 0;
-	int          op_errno = -1;
+        dht_conf_t  *conf = NULL;
+        dht_local_t *local = NULL;
+        int          i = 0;
+        int          op_errno = -1;
 
 
-	conf = frame->this->private;
-	local = frame->local;
+        conf = frame->this->private;
+        local = frame->local;
 
-	local->call_cnt = conf->subvolume_cnt;
+        local->call_cnt = conf->subvolume_cnt;
 
         for (i = 0; i < conf->subvolume_cnt; i++) {
                 if (!conf->subvolume_status[i]) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                "one of the subvolumes down (%s)",
+                                conf->subvolumes[i]->name);
                         op_errno = ENOTCONN;
                         goto err;
                 }
         }
 
-	local->fd = fd_create (local->loc.inode, frame->root->pid);
-	if (!local->fd) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		op_errno = ENOMEM;
-		goto err;
-	}
+        local->fd = fd_create (local->loc.inode, frame->root->pid);
+        if (!local->fd) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	local->op_ret = 0;
+        local->op_ret = 0;
 
-	if (!local->dst_cached) {
-		dht_rename_dir_do (frame, this);
-		return 0;
-	}
+        if (!local->dst_cached) {
+                dht_rename_dir_do (frame, this);
+                return 0;
+        }
 
-	for (i = 0; i < conf->subvolume_cnt; i++) {
-		STACK_WIND (frame, dht_rename_opendir_cbk,
-			    conf->subvolumes[i],
-			    conf->subvolumes[i]->fops->opendir,
-			    &local->loc2, local->fd);
-	}
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                STACK_WIND (frame, dht_rename_opendir_cbk,
+                            conf->subvolumes[i],
+                            conf->subvolumes[i]->fops->opendir,
+                            &local->loc2, local->fd, NULL);
+        }
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
-	return 0;
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                          NULL, NULL);
+        return 0;
 }
+#define DHT_MARK_FOP_INTERNAL(xattr) do {                                      \
+                int tmp = -1;                                                  \
+                if (!xattr) {                                                  \
+                        xattr = dict_new ();                                   \
+                        if (!xattr)                                            \
+                                break;                                         \
+                }                                                              \
+                tmp = dict_set_str (xattr, GLUSTERFS_INTERNAL_FOP_KEY, "yes"); \
+                if (tmp) {                                                     \
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set"      \
+                                " internal dict key for %s", local->loc.path); \
+                }                                                              \
+        }while (0)
+
+#define DHT_MARKER_DONT_ACCOUNT(xattr) do {                             \
+                int tmp = -1;                                                  \
+                if (!xattr) {                                                  \
+                        xattr = dict_new ();                                   \
+                        if (!xattr)                                            \
+                                break;                                         \
+                }                                                              \
+                tmp = dict_set_str (xattr, GLUSTERFS_MARKER_DONT_ACCOUNT_KEY,  \
+                                    "yes");                                    \
+                if (tmp) {                                                     \
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to set"      \
+                                " marker dont account key for %s", local->loc.path); \
+                }                                                              \
+        }while (0)
+
+int
+dht_rename_done (call_frame_t *frame, xlator_t *this)
+{
+        dht_local_t     *local = NULL;
+
+        local = frame->local;
+
+        if (local->linked == _gf_true) {
+                local->linked = _gf_false;
+                dht_linkfile_attr_heal (frame, this);
+        }
+        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
+                          &local->stbuf, &local->preoldparent,
+                          &local->postoldparent, &local->preparent,
+                          &local->postparent, NULL);
 
+        return 0;
+}
 
 int
 dht_rename_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                       struct iatt *postparent)
+                       int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                       struct iatt *postparent, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	call_frame_t *prev = NULL;
-	int           this_call_cnt = 0;
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           this_call_cnt = 0;
 
-	local = frame->local;
-	prev  = cookie;
+        local = frame->local;
+        prev  = cookie;
 
         if (!local) {
                 gf_log (this->name, GF_LOG_ERROR,
@@ -266,82 +373,108 @@ dht_rename_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto out;
         }
 
-	this_call_cnt = dht_frame_return (frame);
+        this_call_cnt = dht_frame_return (frame);
 
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"unlink on %s failed (%s)",
-			prev->this->name, strerror (op_errno));
-	}
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: unlink on %s failed (%s)",
+                        local->loc.path, prev->this->name, strerror (op_errno));
+        }
 
         WIPE (&local->preoldparent);
         WIPE (&local->postoldparent);
         WIPE (&local->preparent);
         WIPE (&local->postparent);
 
-	if (is_last_call (this_call_cnt)) {
-		DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-				  &local->stbuf, &local->preoldparent,
-                                  &local->postoldparent, &local->preparent,
-                                  &local->postparent);
+        if (is_last_call (this_call_cnt)) {
+                dht_rename_done (frame, this);
         }
 
 out:
-	return 0;
+        return 0;
 }
 
 
 int
 dht_rename_cleanup (call_frame_t *frame)
 {
-	dht_local_t *local = NULL;
-	xlator_t    *this = NULL;
-	xlator_t    *src_hashed = NULL;
-	xlator_t    *src_cached = NULL;
-	xlator_t    *dst_hashed = NULL;
-	xlator_t    *dst_cached = NULL;
-	int          call_cnt = 0;
+        dht_local_t *local = NULL;
+        xlator_t    *this = NULL;
+        xlator_t    *src_hashed = NULL;
+        xlator_t    *src_cached = NULL;
+        xlator_t    *dst_hashed = NULL;
+        xlator_t    *dst_cached = NULL;
+        int          call_cnt = 0;
+        dict_t      *xattr      = NULL;
+
+        local = frame->local;
+        this  = frame->this;
+
+        src_hashed = local->src_hashed;
+        src_cached = local->src_cached;
+        dst_hashed = local->dst_hashed;
+        dst_cached = local->dst_cached;
+
+        if (src_cached == dst_cached)
+                goto nolinks;
 
+        if (dst_hashed != src_hashed && dst_hashed != src_cached)
+                call_cnt++;
 
-	local = frame->local;
-	this  = frame->this;
+        if (src_cached != dst_hashed)
+                call_cnt++;
 
-	src_hashed = local->src_hashed;
-	src_cached = local->src_cached;
-	dst_hashed = local->dst_hashed;
-	dst_cached = local->dst_cached;
+        local->call_cnt = call_cnt;
 
-	if (src_cached == dst_cached)
-		goto nolinks;
+        if (!call_cnt)
+                goto nolinks;
 
-	if (dst_hashed != src_hashed && dst_hashed != src_cached)
-		call_cnt++;
+        DHT_MARK_FOP_INTERNAL (xattr);
 
-	if (src_cached != dst_hashed)
-		call_cnt++;
+        if (dst_hashed != src_hashed && dst_hashed != src_cached) {
+                dict_t *xattr_new = NULL;
 
-	local->call_cnt = call_cnt;
+                gf_log (this->name, GF_LOG_TRACE,
+                        "unlinking linkfile %s @ %s => %s",
+                        local->loc.path, dst_hashed->name, src_cached->name);
 
-        if (!call_cnt)
-                goto nolinks;
+                xattr_new = dict_copy_with_ref (xattr, NULL);
+
+
+                DHT_MARKER_DONT_ACCOUNT(xattr_new);
 
-	if (dst_hashed != src_hashed && dst_hashed != src_cached) {
-		gf_log (this->name, GF_LOG_TRACE,
-			"unlinking linkfile %s @ %s => %s",
-			local->loc.path, dst_hashed->name, src_cached->name);
                 STACK_WIND (frame, dht_rename_unlink_cbk,
                             dst_hashed, dst_hashed->fops->unlink,
-                            &local->loc);
-	}
+                            &local->loc, 0, xattr_new);
 
-	if (src_cached != dst_hashed) {
-		gf_log (this->name, GF_LOG_TRACE,
-			"unlinking link %s => %s (%s)", local->loc.path,
-			local->loc2.path, src_cached->name);
-		STACK_WIND (frame, dht_rename_unlink_cbk,
-			    src_cached, src_cached->fops->unlink,
-			    &local->loc2);
-	}
+                dict_unref (xattr_new);
+                xattr_new = NULL;
+        }
+
+        if (src_cached != dst_hashed) {
+                dict_t *xattr_new = NULL;
+
+                gf_log (this->name, GF_LOG_TRACE,
+                        "unlinking link %s => %s (%s)", local->loc.path,
+                        local->loc2.path, src_cached->name);
+
+                xattr_new = dict_copy_with_ref (xattr, NULL);
+
+                if (uuid_compare (local->loc.pargfid,
+                                  local->loc2.pargfid) == 0) {
+                        DHT_MARKER_DONT_ACCOUNT(xattr_new);
+                }
+
+                STACK_WIND (frame, dht_rename_unlink_cbk,
+                            src_cached, src_cached->fops->unlink,
+                            &local->loc2, 0, xattr_new);
+
+                dict_unref (xattr_new);
+                xattr_new = NULL;
+        }
+
+        if (xattr)
+                dict_unref (xattr);
 
         return 0;
 
@@ -351,10 +484,40 @@ nolinks:
         WIPE (&local->preparent);
         WIPE (&local->postparent);
 
-	DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-			  &local->stbuf, &local->preoldparent,
+        DHT_STRIP_PHASE1_FLAGS (&local->stbuf);
+        DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
+                          &local->stbuf, &local->preoldparent,
                           &local->postoldparent, &local->preparent,
-                          &local->postparent);
+                          &local->postparent, NULL);
+
+        return 0;
+}
+
+
+int
+dht_rename_links_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno,
+                             inode_t *inode, struct iatt *stbuf,
+                             struct iatt *preparent, struct iatt *postparent,
+                             dict_t *xdata)
+{
+        call_frame_t *prev = NULL;
+        dht_local_t  *local = NULL;
+
+        prev = cookie;
+        local = frame->local;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "link/file %s on %s failed (%s)",
+                        local->loc.path, prev->this->name, strerror (op_errno));
+        }
+
+        if (local->linked == _gf_true) {
+                local->linked = _gf_false;
+                dht_linkfile_attr_heal (frame, this);
+        }
+        DHT_STACK_DESTROY (frame);
 
         return 0;
 }
@@ -362,119 +525,177 @@ nolinks:
 
 int
 dht_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
+                int32_t op_ret, int32_t op_errno, struct iatt *stbuf,
                 struct iatt *preoldparent, struct iatt *postoldparent,
-                struct iatt *prenewparent, struct iatt *postnewparent)
+                struct iatt *prenewparent, struct iatt *postnewparent,
+                dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	call_frame_t *prev = NULL;
-	xlator_t     *src_hashed = NULL;
-	xlator_t     *src_cached = NULL;
-	xlator_t     *dst_hashed = NULL;
-	xlator_t     *dst_cached = NULL;
-	xlator_t     *rename_subvol = NULL;
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        xlator_t     *src_hashed = NULL;
+        xlator_t     *src_cached = NULL;
+        xlator_t     *dst_hashed = NULL;
+        xlator_t     *dst_cached = NULL;
+        xlator_t     *rename_subvol = NULL;
+        call_frame_t *link_frame = NULL;
+        dht_local_t *link_local = NULL;
+        dict_t       *xattr     = NULL;
+
+        local = frame->local;
+        prev = cookie;
+
+        src_hashed = local->src_hashed;
+        src_cached = local->src_cached;
+        dst_hashed = local->dst_hashed;
+        dst_cached = local->dst_cached;
+
+        if (local->linked == _gf_true)
+                FRAME_SU_UNDO (frame, dht_local_t);
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: rename on %s failed (%s)", local->loc.path,
+                        prev->this->name, strerror (op_errno));
+                local->op_ret   = op_ret;
+                local->op_errno = op_errno;
+                goto cleanup;
+        }
 
-	local = frame->local;
-	prev = cookie;
+        if ((src_cached == dst_cached) && (dst_hashed != dst_cached)) {
+                link_frame = copy_frame (frame);
+                if (!link_frame) {
+                        goto err;
+                }
 
-	src_hashed = local->src_hashed;
-	src_cached = local->src_cached;
-	dst_hashed = local->dst_hashed;
-	dst_cached = local->dst_cached;
+                /* fop value sent as maxvalue because it is not used
+                   anywhere in this case */
+                link_local = dht_local_init (link_frame, &local->loc2, NULL,
+                                             GF_FOP_MAXVALUE);
+                if (!link_local) {
+                        goto err;
+                }
 
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"rename on %s failed (%s)", prev->this->name,
-			strerror (op_errno));
-		local->op_ret   = op_ret;
-		local->op_errno = op_errno;
-		goto cleanup;
-	}
+                if (link_local->loc.inode)
+                        inode_unref (link_local->loc.inode);
+                link_local->loc.inode = inode_ref (local->loc.inode);
+                uuid_copy (link_local->gfid, local->loc.inode->gfid);
 
-        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-        dht_iatt_merge (this, &local->preoldparent, preoldparent, prev->this);
-        dht_iatt_merge (this, &local->postoldparent, postoldparent, prev->this);
-        dht_iatt_merge (this, &local->preparent, prenewparent, prev->this);
-        dht_iatt_merge (this, &local->postparent, postnewparent, prev->this);
+                dht_linkfile_create (link_frame, dht_rename_links_create_cbk,
+                                     this, src_cached, dst_hashed,
+                                     &link_local->loc);
+        }
+
+err:
+        /* Merge attrs only from src_cached. In case there of src_cached !=
+         * dst_hashed, this ignores linkfile attrs. */
+        if (prev->this == src_cached) {
+                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+                dht_iatt_merge (this, &local->preoldparent, preoldparent,
+                                prev->this);
+                dht_iatt_merge (this, &local->postoldparent, postoldparent,
+                                prev->this);
+                dht_iatt_merge (this, &local->preparent, prenewparent,
+                                prev->this);
+                dht_iatt_merge (this, &local->postparent, postnewparent,
+                                prev->this);
+        }
 
-        local->stbuf.ia_ino = local->loc.inode->ino;
 
-        local->preoldparent.ia_ino = local->loc.parent->ino;
-        local->postoldparent.ia_ino = local->loc.parent->ino;
+        /* NOTE: rename_subvol is the same subvolume from which dht_rename_cbk
+         *       is called. since rename has already happened on rename_subvol,
+         *       unlink should not be sent for oldpath (either linkfile or cached-file)
+         *       on rename_subvol. */
+        if (src_cached == dst_cached)
+                rename_subvol = src_cached;
+        else
+                rename_subvol = dst_hashed;
 
-        local->preparent.ia_ino = local->loc2.parent->ino;
-        local->postparent.ia_ino = local->loc2.parent->ino;
-	
-	/* NOTE: rename_subvol is the same subvolume from which dht_rename_cbk
-	 *       is called. since rename has already happened on rename_subvol,
-	 *       unlink should not be sent for oldpath (either linkfile or cached-file)
-	 *       on rename_subvol. */
-	if (src_cached == dst_cached)
-		rename_subvol = src_cached;
-	else
-		rename_subvol = dst_hashed;
+        /* TODO: delete files in background */
 
-	/* TODO: delete files in background */
+        if (src_cached != dst_hashed && src_cached != dst_cached)
+                local->call_cnt++;
 
-	if (src_cached != dst_hashed && src_cached != dst_cached)
-		local->call_cnt++;
+        if (src_hashed != rename_subvol && src_hashed != src_cached)
+                local->call_cnt++;
 
-	if (src_hashed != rename_subvol && src_hashed != src_cached)
-		local->call_cnt++;
+        if (dst_cached && dst_cached != dst_hashed && dst_cached != src_cached)
+                local->call_cnt++;
 
-	if (dst_cached && dst_cached != dst_hashed && dst_cached != src_cached)
-		local->call_cnt++;
+        if (local->call_cnt == 0)
+                goto unwind;
 
-	if (local->call_cnt == 0)
-		goto unwind;
+        DHT_MARK_FOP_INTERNAL (xattr);
 
-	if (src_cached != dst_hashed && src_cached != dst_cached) {
-		gf_log (this->name, GF_LOG_TRACE,
-			"deleting old src datafile %s @ %s",
-			local->loc.path, src_cached->name);
+        if (src_cached != dst_hashed && src_cached != dst_cached) {
+                dict_t *xattr_new = NULL;
 
-		STACK_WIND (frame, dht_rename_unlink_cbk,
-			    src_cached, src_cached->fops->unlink,
-			    &local->loc);
-	}
+                xattr_new = dict_copy_with_ref (xattr, NULL);
 
-	if (src_hashed != rename_subvol && src_hashed != src_cached) {
-		gf_log (this->name, GF_LOG_TRACE,
-			"deleting old src linkfile %s @ %s",
-			local->loc.path, src_hashed->name);
+                gf_log (this->name, GF_LOG_TRACE,
+                        "deleting old src datafile %s @ %s",
+                        local->loc.path, src_cached->name);
 
-		STACK_WIND (frame, dht_rename_unlink_cbk,
-			    src_hashed, src_hashed->fops->unlink,
-			    &local->loc);
-	}
+                if (uuid_compare (local->loc.pargfid,
+                                  local->loc2.pargfid) == 0) {
+                        DHT_MARKER_DONT_ACCOUNT(xattr_new);
+                }
 
-	if (dst_cached
-	    && (dst_cached != dst_hashed)
-	    && (dst_cached != src_cached)) {
-		gf_log (this->name, GF_LOG_TRACE,
-			"deleting old dst datafile %s @ %s",
-			local->loc2.path, dst_cached->name);
+                STACK_WIND (frame, dht_rename_unlink_cbk,
+                            src_cached, src_cached->fops->unlink,
+                            &local->loc, 0, xattr_new);
 
-		STACK_WIND (frame, dht_rename_unlink_cbk,
-			    dst_cached, dst_cached->fops->unlink,
-			    &local->loc2);
-	}
-	return 0;
+                dict_unref (xattr_new);
+                xattr_new = NULL;
+        }
+
+        if (src_hashed != rename_subvol && src_hashed != src_cached) {
+                dict_t *xattr_new = NULL;
+
+                xattr_new = dict_copy_with_ref (xattr, NULL);
+
+                gf_log (this->name, GF_LOG_TRACE,
+                        "deleting old src linkfile %s @ %s",
+                        local->loc.path, src_hashed->name);
+
+                DHT_MARKER_DONT_ACCOUNT(xattr_new);
+
+                STACK_WIND (frame, dht_rename_unlink_cbk,
+                            src_hashed, src_hashed->fops->unlink,
+                            &local->loc, 0, xattr_new);
+
+                dict_unref (xattr_new);
+                xattr_new = NULL;
+        }
+
+        if (dst_cached
+            && (dst_cached != dst_hashed)
+            && (dst_cached != src_cached)) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "deleting old dst datafile %s @ %s",
+                        local->loc2.path, dst_cached->name);
+
+                STACK_WIND (frame, dht_rename_unlink_cbk,
+                            dst_cached, dst_cached->fops->unlink,
+                            &local->loc2, 0, xattr);
+        }
+        if (xattr)
+                dict_unref (xattr);
+        return 0;
 
 unwind:
         WIPE (&local->preoldparent);
         WIPE (&local->postoldparent);
         WIPE (&local->preparent);
         WIPE (&local->postparent);
+        if (xattr)
+                dict_unref (xattr);
 
-	DHT_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
-			  &local->stbuf, &local->preoldparent,
-                          &local->postoldparent, &local->preparent,
-                          &local->postparent);
+        dht_rename_done (frame, this);
 
-	return 0;
+        return 0;
 
 cleanup:
+        if (xattr)
+                dict_unref (xattr);
         dht_rename_cleanup (frame);
 
         return 0;
@@ -484,67 +705,115 @@ cleanup:
 int
 dht_do_rename (call_frame_t *frame)
 {
-	dht_local_t *local = NULL;
-	xlator_t    *dst_hashed = NULL;
-	xlator_t    *src_cached = NULL;
-	xlator_t    *dst_cached = NULL;
-	xlator_t    *this = NULL;
-	xlator_t    *rename_subvol = NULL;
+        dht_local_t *local         = NULL;
+        xlator_t    *dst_hashed    = NULL;
+        xlator_t    *src_cached    = NULL;
+        xlator_t    *dst_cached    = NULL;
+        xlator_t    *this          = NULL;
+        xlator_t    *rename_subvol = NULL;
+        dict_t      *dict          = NULL;
 
 
-	local = frame->local;
-	this  = frame->this;
+        local = frame->local;
+        this  = frame->this;
 
-	dst_hashed = local->dst_hashed;
-	dst_cached = local->dst_cached;
-	src_cached = local->src_cached;
+        dst_hashed = local->dst_hashed;
+        dst_cached = local->dst_cached;
+        src_cached = local->src_cached;
 
-	if (src_cached == dst_cached)
-		rename_subvol = src_cached;
-	else
-		rename_subvol = dst_hashed;
+        if (src_cached == dst_cached)
+                rename_subvol = src_cached;
+        else
+                rename_subvol = dst_hashed;
 
-	gf_log (this->name, GF_LOG_TRACE,
-		"renaming %s => %s (%s)",
-		local->loc.path, local->loc2.path, rename_subvol->name);
+        if ((src_cached != dst_hashed) && (rename_subvol == dst_hashed)) {
+                DHT_MARKER_DONT_ACCOUNT(dict);
+        }
 
-	STACK_WIND (frame, dht_rename_cbk,
-		    rename_subvol, rename_subvol->fops->rename,
-		    &local->loc, &local->loc2);
+        gf_log (this->name, GF_LOG_TRACE,
+                "renaming %s => %s (%s)",
+                local->loc.path, local->loc2.path, rename_subvol->name);
 
-	return 0;
+        if (local->linked == _gf_true)
+                FRAME_SU_DO (frame, dht_local_t);
+        STACK_WIND (frame, dht_rename_cbk,
+                    rename_subvol, rename_subvol->fops->rename,
+                    &local->loc, &local->loc2, dict);
+
+        return 0;
 }
 
 
 int
 dht_rename_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno,
+                      int32_t op_ret, int32_t op_errno,
                       inode_t *inode, struct iatt *stbuf,
-                      struct iatt *preparent, struct iatt *postparent)
+                      struct iatt *preparent, struct iatt *postparent,
+                      dict_t *xdata)
+{
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        int           this_call_cnt  = 0;
+
+
+        local = frame->local;
+        prev = cookie;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "link/file on %s failed (%s)",
+                        prev->this->name, strerror (op_errno));
+                local->op_ret   = -1;
+                if (op_errno != ENOENT)
+                        local->op_errno = op_errno;
+        } else if (local->src_cached == prev->this) {
+                /* merge of attr returned only from linkfile creation */
+                dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
+        }
+
+        this_call_cnt = dht_frame_return (frame);
+        if (is_last_call (this_call_cnt)) {
+                if (local->op_ret == -1)
+                        goto cleanup;
+
+                dht_do_rename (frame);
+        }
+
+        return 0;
+
+cleanup:
+        dht_rename_cleanup (frame);
+
+        return 0;
+}
+
+
+int
+dht_rename_unlink_links_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                             int32_t op_ret, int32_t op_errno,
+                             struct iatt *preparent, struct iatt *postparent,
+                             dict_t *xdata)
 {
 	dht_local_t  *local = NULL;
 	call_frame_t *prev = NULL;
-	int           this_call_cnt  = 0;
 
 
 	local = frame->local;
 	prev = cookie;
-	
-	if (op_ret == -1) {
+
+	if ((op_ret == -1) && (op_errno != ENOENT)) {
 		gf_log (this->name, GF_LOG_DEBUG,
-			"link/file on %s failed (%s)",
-			prev->this->name, strerror (op_errno));
+			"unlink of %s on %s failed (%s)",
+			local->loc2.path, prev->this->name,
+                        strerror (op_errno));
 		local->op_ret   = -1;
 		local->op_errno = op_errno;
 	}
 
-	this_call_cnt = dht_frame_return (frame);
-	if (is_last_call (this_call_cnt)) {
-		if (local->op_ret == -1)
-			goto cleanup;
-		
-		dht_do_rename (frame);
-	}
+        if (local->op_ret == -1)
+                goto cleanup;
+
+        dht_do_rename (frame);
 
 	return 0;
 
@@ -558,156 +827,183 @@ cleanup:
 int
 dht_rename_create_links (call_frame_t *frame)
 {
-	dht_local_t *local = NULL;
-	xlator_t    *this = NULL;
-	xlator_t    *src_hashed = NULL;
-	xlator_t    *src_cached = NULL;
-	xlator_t    *dst_hashed = NULL;
-	xlator_t    *dst_cached = NULL;
-	int          call_cnt = 0;
+        dht_local_t *local = NULL;
+        xlator_t    *this = NULL;
+        xlator_t    *src_hashed = NULL;
+        xlator_t    *src_cached = NULL;
+        xlator_t    *dst_hashed = NULL;
+        xlator_t    *dst_cached = NULL;
+        int          call_cnt = 0;
+        dict_t      *xattr = NULL;
 
 
-	local = frame->local;
-	this  = frame->this;
+        local = frame->local;
+        this  = frame->this;
 
-	src_hashed = local->src_hashed;
-	src_cached = local->src_cached;
-	dst_hashed = local->dst_hashed;
-	dst_cached = local->dst_cached;
+        src_hashed = local->src_hashed;
+        src_cached = local->src_cached;
+        dst_hashed = local->dst_hashed;
+        dst_cached = local->dst_cached;
 
-	if (src_cached == dst_cached)
-		goto nolinks;
+        DHT_MARK_FOP_INTERNAL (xattr);
 
-	if (dst_hashed != src_hashed && dst_hashed != src_cached)
-		call_cnt++;
+        if (src_cached == dst_cached) {
+                dict_t *xattr_new = NULL;
 
-	if (src_cached != dst_hashed)
-		call_cnt++;
+                if (dst_hashed == dst_cached)
+                        goto nolinks;
 
-	local->call_cnt = call_cnt;
+                xattr_new = dict_copy_with_ref (xattr, NULL);
 
-	if (dst_hashed != src_hashed && dst_hashed != src_cached) {
 		gf_log (this->name, GF_LOG_TRACE,
-			"linkfile %s @ %s => %s",
-			local->loc.path, dst_hashed->name, src_cached->name);
+			"unlinking dst linkfile %s @ %s",
+			local->loc2.path, dst_hashed->name);
+
+                DHT_MARKER_DONT_ACCOUNT(xattr_new);
+
+		STACK_WIND (frame, dht_rename_unlink_links_cbk,
+			    dst_hashed, dst_hashed->fops->unlink,
+			    &local->loc2, 0, xattr_new);
+
+                dict_unref (xattr_new);
+                return 0;
+        }
+
+        if (dst_hashed != src_hashed && dst_hashed != src_cached)
+                call_cnt++;
+
+        if (src_cached != dst_hashed)
+                call_cnt++;
+
+        local->call_cnt = call_cnt;
+
+        if (dst_hashed != src_hashed && dst_hashed != src_cached) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "linkfile %s @ %s => %s",
+                        local->loc.path, dst_hashed->name, src_cached->name);
                 memcpy (local->gfid, local->loc.inode->gfid, 16);
-		dht_linkfile_create (frame, dht_rename_links_cbk,
+		dht_linkfile_create (frame, dht_rename_links_cbk, this,
 				     src_cached, dst_hashed, &local->loc);
 	}
 
 	if (src_cached != dst_hashed) {
+                dict_t *xattr_new = NULL;
+
+                xattr_new = dict_copy_with_ref (xattr, NULL);
+
 		gf_log (this->name, GF_LOG_TRACE,
 			"link %s => %s (%s)", local->loc.path,
 			local->loc2.path, src_cached->name);
+                if (uuid_compare (local->loc.pargfid,
+                                  local->loc2.pargfid) == 0) {
+                        DHT_MARKER_DONT_ACCOUNT(xattr_new);
+                }
+
 		STACK_WIND (frame, dht_rename_links_cbk,
 			    src_cached, src_cached->fops->link,
-			    &local->loc, &local->loc2);
+			    &local->loc, &local->loc2, xattr_new);
+
+                dict_unref (xattr_new);
 	}
 
 nolinks:
-	if (!call_cnt) {
-		/* skip to next step */
-		dht_do_rename (frame);
-	}
+        if (!call_cnt) {
+                /* skip to next step */
+                dht_do_rename (frame);
+        }
+        if (xattr)
+                dict_unref (xattr);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_rename (call_frame_t *frame, xlator_t *this,
-	    loc_t *oldloc, loc_t *newloc)
+            loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
-	xlator_t    *src_cached = NULL;
-	xlator_t    *src_hashed = NULL;
-	xlator_t    *dst_cached = NULL;
-	xlator_t    *dst_hashed = NULL;
-	int          op_errno = -1;
-	int          ret = -1;
-	dht_local_t *local = NULL;
-
-
-	VALIDATE_OR_GOTO (frame, err);
-	VALIDATE_OR_GOTO (this, err);
-	VALIDATE_OR_GOTO (oldloc, err);
-	VALIDATE_OR_GOTO (newloc, err);
-
-	src_hashed = dht_subvol_get_hashed (this, oldloc);
-	if (!src_hashed) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no subvolume in layout for path=%s",
-			oldloc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	src_cached = dht_subvol_get_cached (this, oldloc->inode);
-	if (!src_cached) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no cached subvolume for path=%s", oldloc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
-
-	dst_hashed = dht_subvol_get_hashed (this, newloc);
-	if (!dst_hashed) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no subvolume in layout for path=%s",
-			newloc->path);
-		op_errno = EINVAL;
-		goto err;
-	}
+        xlator_t    *src_cached = NULL;
+        xlator_t    *src_hashed = NULL;
+        xlator_t    *dst_cached = NULL;
+        xlator_t    *dst_hashed = NULL;
+        int          op_errno = -1;
+        int          ret = -1;
+        dht_local_t *local = NULL;
+
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (oldloc, err);
+        VALIDATE_OR_GOTO (newloc, err);
+
+        src_hashed = dht_subvol_get_hashed (this, oldloc);
+        if (!src_hashed) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "no subvolume in layout for path=%s",
+                        oldloc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	if (newloc->inode)
-		dst_cached = dht_subvol_get_cached (this, newloc->inode);
+        src_cached = dht_subvol_get_cached (this, oldloc->inode);
+        if (!src_cached) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "no cached subvolume for path=%s", oldloc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
+        dst_hashed = dht_subvol_get_hashed (this, newloc);
+        if (!dst_hashed) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "no subvolume in layout for path=%s",
+                        newloc->path);
+                op_errno = EINVAL;
+                goto err;
+        }
 
-	ret = loc_copy (&local->loc, oldloc);
-	if (ret == -1) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
+        if (newloc->inode)
+                dst_cached = dht_subvol_get_cached (this, newloc->inode);
 
-	ret = loc_copy (&local->loc2, newloc);
-	if (ret == -1) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
+        local = dht_local_init (frame, oldloc, NULL, GF_FOP_RENAME);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        /* cached_subvol will be set from dht_local_init, reset it to NULL,
+           as the logic of handling rename is different  */
+        local->cached_subvol = NULL;
+
+        ret = loc_copy (&local->loc2, newloc);
+        if (ret == -1) {
+                op_errno = ENOMEM;
+                goto err;
+        }
 
-	local->src_hashed = src_hashed;
-	local->src_cached = src_cached;
-	local->dst_hashed = dst_hashed;
-	local->dst_cached = dst_cached;
-
-	gf_log (this->name, GF_LOG_TRACE,
-		"renaming %s (hash=%s/cache=%s) => %s (hash=%s/cache=%s)",
-		oldloc->path, src_hashed->name, src_cached->name,
-		newloc->path, dst_hashed->name,
-		dst_cached ? dst_cached->name : "<nul>");
-
-	if (IA_ISDIR (oldloc->inode->ia_type)) {
-		dht_rename_dir (frame, this);
-	} else {
-		local->op_ret = 0;
-		dht_rename_create_links (frame);
-	}
+        local->src_hashed = src_hashed;
+        local->src_cached = src_cached;
+        local->dst_hashed = dst_hashed;
+        local->dst_cached = dst_cached;
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "renaming %s (hash=%s/cache=%s) => %s (hash=%s/cache=%s)",
+                oldloc->path, src_hashed->name, src_cached->name,
+                newloc->path, dst_hashed->name,
+                dst_cached ? dst_cached->name : "<nul>");
+
+        if (IA_ISDIR (oldloc->inode->ia_type)) {
+                dht_rename_dir (frame, this);
+        } else {
+                local->op_ret = 0;
+                dht_rename_create_links (frame);
+        }
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL, NULL,
+                          NULL, NULL);
 
-	return 0;
+        return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-selfheal.c b/xlators/cluster/dht/src/dht-selfheal.c
index c12847f58..0e6527544 100644
--- a/xlators/cluster/dht/src/dht-selfheal.c
+++ b/xlators/cluster/dht/src/dht-selfheal.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 #ifndef _CONFIG_H
@@ -26,247 +17,489 @@
 #include "glusterfs.h"
 #include "xlator.h"
 #include "dht-common.h"
+#include "glusterfs-acl.h"
+
+#define DHT_SET_LAYOUT_RANGE(layout,i,srt,chunk,cnt,path)    do {       \
+                layout->list[i].start = srt;                            \
+                layout->list[i].stop  = srt + chunk - 1;                \
+                                                                        \
+                gf_log (this->name, GF_LOG_TRACE,                       \
+                        "gave fix: %u - %u on %s for %s",               \
+                        layout->list[i].start, layout->list[i].stop,    \
+                        layout->list[i].xlator->name, path);            \
+        } while (0)
+
+#define DHT_RESET_LAYOUT_RANGE(layout)    do {                          \
+                int cnt = 0;                                            \
+                for (cnt = 0; cnt < layout->cnt; cnt++ ) {              \
+                        layout->list[cnt].start = 0;                    \
+                        layout->list[cnt].stop  = 0;                    \
+                }                                                       \
+        } while (0)
+
+static uint32_t
+dht_overlap_calc (dht_layout_t *old, int o, dht_layout_t *new, int n)
+{
+	if (o >= old->cnt || n >= new->cnt)
+		return 0;
+
+	if (old->list[o].err > 0 || new->list[n].err > 0)
+		return 0;
+
+        if (old->list[o].start == old->list[o].stop) {
+                return 0;
+        }
+
+        if (new->list[n].start == new->list[n].stop) {
+                return 0;
+        }
+
+	if ((old->list[o].start > new->list[n].stop) ||
+	    (old->list[o].stop < new->list[n].start))
+		return 0;
+
+	return min (old->list[o].stop, new->list[n].stop) -
+	        max (old->list[o].start, new->list[n].start) + 1;
+}
 
 
 int
 dht_selfheal_dir_finish (call_frame_t *frame, xlator_t *this, int ret)
 {
-	dht_local_t  *local = NULL;
-
+        dht_local_t  *local = NULL;
 
-	local = frame->local;
-	local->selfheal.dir_cbk (frame, NULL, frame->this, ret,
-				 local->op_errno);
+        local = frame->local;
+        local->selfheal.dir_cbk (frame, NULL, frame->this, ret,
+                                 local->op_errno, NULL);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_selfheal_dir_xattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			    int op_ret, int op_errno)
+                            int op_ret, int op_errno, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-	call_frame_t *prev = NULL;
-	xlator_t     *subvol = NULL;
-	int           i = 0;
-	dht_layout_t *layout = NULL;
-	int           err = 0;
-	int           this_call_cnt = 0;
-
-	local = frame->local;
-	layout = local->selfheal.layout;
-	prev = cookie;
-	subvol = prev->this;
-
-	if (op_ret == 0)
-		err = 0;
-	else
-		err = op_errno;
-
-	for (i = 0; i < layout->cnt; i++) {
-		if (layout->list[i].xlator == subvol) {
-			layout->list[i].err = err;
-			break;
-		}
-	}
+        dht_local_t  *local = NULL;
+        call_frame_t *prev = NULL;
+        xlator_t     *subvol = NULL;
+        int           i = 0;
+        dht_layout_t *layout = NULL;
+        int           err = 0;
+        int           this_call_cnt = 0;
+
+        local = frame->local;
+        layout = local->selfheal.layout;
+        prev = cookie;
+        subvol = prev->this;
+
+        if (op_ret == 0)
+                err = 0;
+        else
+                err = op_errno;
+
+        for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].xlator == subvol) {
+                        layout->list[i].err = err;
+                        break;
+                }
+        }
 
-	this_call_cnt = dht_frame_return (frame);
+        this_call_cnt = dht_frame_return (frame);
 
-	if (is_last_call (this_call_cnt)) {
-		dht_selfheal_dir_finish (frame, this, 0);
-	}
+        if (is_last_call (this_call_cnt)) {
+                dht_selfheal_dir_finish (frame, this, 0);
+        }
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_selfheal_dir_xattr_persubvol (call_frame_t *frame, loc_t *loc,
-				  dht_layout_t *layout, int i)
+                                  dht_layout_t *layout, int i,
+                                  xlator_t *req_subvol)
 {
-	xlator_t          *subvol = NULL;
-	dict_t            *xattr = NULL;
-	int                ret = 0;
-	xlator_t          *this = NULL;
-	int32_t           *disk_layout = NULL;
+        xlator_t          *subvol = NULL;
+        dict_t            *xattr = NULL;
+        int                ret = 0;
+        xlator_t          *this = NULL;
+        int32_t           *disk_layout = NULL;
+        dht_local_t       *local = NULL;
+        dht_conf_t        *conf = NULL;
+        data_t            *data = NULL;
+
+        local = frame->local;
+        if (req_subvol)
+                subvol = req_subvol;
+        else
+                subvol = layout->list[i].xlator;
+        this = frame->this;
+
+        GF_VALIDATE_OR_GOTO ("", this, err);
+        GF_VALIDATE_OR_GOTO (this->name, layout, err);
+        GF_VALIDATE_OR_GOTO (this->name, local, err);
+        GF_VALIDATE_OR_GOTO (this->name, subvol, err);
+        VALIDATE_OR_GOTO (this->private, err);
+
+        conf = this->private;
+
+        xattr = get_new_dict ();
+        if (!xattr) {
+                goto err;
+        }
 
+        ret = dht_disk_layout_extract (this, layout, i, &disk_layout);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: (subvol %s) failed to extract disk layout",
+                        loc->path, subvol->name);
+                goto err;
+        }
 
-	subvol = layout->list[i].xlator;
-	this = frame->this;
+        ret = dict_set_bin (xattr, conf->xattr_name, disk_layout, 4 * 4);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%s: (subvol %s) failed to set xattr dictionary",
+                        loc->path, subvol->name);
+                goto err;
+        }
+        disk_layout = NULL;
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "setting hash range %u - %u (type %d) on subvolume %s for %s",
+                layout->list[i].start, layout->list[i].stop,
+                layout->type, subvol->name, loc->path);
+
+        dict_ref (xattr);
+        if (local->xattr) {
+                data = dict_get (local->xattr, QUOTA_LIMIT_KEY);
+                if (data) {
+                        ret = dict_add (xattr, QUOTA_LIMIT_KEY, data);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "Failed to "
+                                        "set quota limit key on %s",loc->path);
+                        }
+                }
+        }
+        if (!uuid_is_null (local->gfid))
+                uuid_copy (loc->gfid, local->gfid);
 
-	xattr = get_new_dict ();
-	if (!xattr) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
+        STACK_WIND (frame, dht_selfheal_dir_xattr_cbk,
+                    subvol, subvol->fops->setxattr,
+                    loc, xattr, 0, NULL);
 
-	ret = dht_disk_layout_extract (this, layout, i, &disk_layout);
-	if (ret == -1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"failed to extract disk layout");
-		goto err;
-	}
+        dict_unref (xattr);
 
-	ret = dict_set_bin (xattr, "trusted.glusterfs.dht",
-			    disk_layout, 4 * 4);
-	if (ret == -1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"failed to set xattr dictionary");
-		goto err;
-	}
-	disk_layout = NULL;
+        return 0;
 
-	gf_log (this->name, GF_LOG_TRACE,
-		"setting hash range %u - %u (type %d) on subvolume %s for %s",
-		layout->list[i].start, layout->list[i].stop,
-		layout->type, subvol->name, loc->path);
+err:
+        if (xattr)
+                dict_destroy (xattr);
 
-	dict_ref (xattr);
+        GF_FREE (disk_layout);
 
-	STACK_WIND (frame, dht_selfheal_dir_xattr_cbk,
-		    subvol, subvol->fops->setxattr,
-		    loc, xattr, 0);
+        dht_selfheal_dir_xattr_cbk (frame, subvol, frame->this,
+                                    -1, ENOMEM, NULL);
+        return 0;
+}
 
-	dict_unref (xattr);
+int
+dht_fix_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
+{
+        dht_local_t *local = NULL;
+        int          i = 0;
+        int          count = 0;
+        xlator_t    *this = NULL;
+        dht_conf_t  *conf = NULL;
+        dht_layout_t *dummy = NULL;
 
-	return 0;
+        local = frame->local;
+        this = frame->this;
+        conf = this->private;
 
-err:
-	if (xattr)
-		dict_destroy (xattr);
+        gf_log (this->name, GF_LOG_DEBUG,
+                "writing the new range for all subvolumes");
 
-	if (disk_layout)
-		GF_FREE (disk_layout);
+        local->call_cnt = count = conf->subvolume_cnt;
 
-	dht_selfheal_dir_xattr_cbk (frame, subvol, frame->this,
-				    -1, ENOMEM);
-	return 0;
-}
+        for (i = 0; i < layout->cnt; i++) {
+                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i, NULL);
+
+                if (--count == 0)
+                        goto out;
+        }
+        /* if we are here, subvolcount > layout_count. subvols-per-directory
+         * option might be set here. We need to clear out layout from the
+         * non-participating subvolumes, else it will result in overlaps */
+        dummy = dht_layout_new (this, 1);
+        if (!dummy)
+                goto out;
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (_gf_false ==
+                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
+                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
+                                                          conf->subvolumes[i]);
+                        if (--count == 0)
+                                break;
+                }
+        }
 
+        dht_layout_unref (this, dummy);
+out:
+        return 0;
+}
 
 int
 dht_selfheal_dir_xattr (call_frame_t *frame, loc_t *loc, dht_layout_t *layout)
 {
-	dht_local_t *local = NULL;
-	int          missing_xattr = 0;
-	int          i = 0;
-	xlator_t    *this = NULL;
-
-	local = frame->local;
-	this = frame->this;
-
-	for (i = 0; i < layout->cnt; i++) {
-		if (layout->list[i].err != -1 || !layout->list[i].stop) {
-			/* err != -1 would mean xattr present on the directory
-			 * or the directory is itself non existant.
-			 * !layout->list[i].stop would mean layout absent
-			 */
-			continue;
-		}
-		missing_xattr++;
-	}
+        dht_local_t *local = NULL;
+        int          missing_xattr = 0;
+        int          i = 0;
+        xlator_t    *this = NULL;
+        dht_conf_t   *conf = NULL;
+        dht_layout_t *dummy = NULL;
+
+        local = frame->local;
+        this = frame->this;
+        conf = this->private;
+
+        for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].err != -1 || !layout->list[i].stop) {
+                        /* err != -1 would mean xattr present on the directory
+                         * or the directory is non existent.
+                         * !layout->list[i].stop would mean layout absent
+                         */
+
+                        continue;
+                }
+                missing_xattr++;
+        }
+        /* Also account for subvolumes with no-layout. Used for zero'ing out
+         * the layouts and for setting quota key's if present */
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (_gf_false ==
+                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
+                        missing_xattr++;
+                }
+        }
+        gf_log (this->name, GF_LOG_TRACE,
+                "%d subvolumes missing xattr for %s",
+                missing_xattr, loc->path);
 
-	gf_log (this->name, GF_LOG_TRACE,
-		"%d subvolumes missing xattr for %s",
-		missing_xattr, loc->path);
+        if (missing_xattr == 0) {
+                dht_selfheal_dir_finish (frame, this, 0);
+                return 0;
+        }
 
-	if (missing_xattr == 0) {
-		dht_selfheal_dir_finish (frame, this, 0);
-		return 0;
-	}
+        local->call_cnt = missing_xattr;
+        for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].err != -1 || !layout->list[i].stop)
+                        continue;
 
-	local->call_cnt = missing_xattr;
+                dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i, NULL);
 
-	for (i = 0; i < layout->cnt; i++) {
-		if (layout->list[i].err != -1 || !layout->list[i].stop)
-			continue;
+                if (--missing_xattr == 0)
+                        break;
+        }
+        dummy = dht_layout_new (this, 1);
+        if (!dummy)
+                goto out;
+        for (i = 0; i < conf->subvolume_cnt && missing_xattr; i++) {
+                if (_gf_false ==
+                    dht_is_subvol_in_layout (layout, conf->subvolumes[i])) {
+                        dht_selfheal_dir_xattr_persubvol (frame, loc, dummy, 0,
+                                                          conf->subvolumes[i]);
+                        missing_xattr--;
+                }
+        }
+
+        dht_layout_unref (this, dummy);
+out:
+        return 0;
+}
 
-		dht_selfheal_dir_xattr_persubvol (frame, loc, layout, i);
+int
+dht_selfheal_dir_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                              int op_ret, int op_errno, struct iatt *statpre,
+                              struct iatt *statpost, dict_t *xdata)
+{
+        dht_local_t   *local = NULL;
+        dht_layout_t  *layout = NULL;
+        int            this_call_cnt = 0;
 
-		if (--missing_xattr == 0)
-			break;
-	}
-	return 0;
+        local  = frame->local;
+        layout = local->selfheal.layout;
+
+        this_call_cnt = dht_frame_return (frame);
+
+        if (is_last_call (this_call_cnt)) {
+                dht_selfheal_dir_xattr (frame, &local->loc, layout);
+        }
+
+        return 0;
 }
 
 
 int
+dht_selfheal_dir_setattr (call_frame_t *frame, loc_t *loc, struct iatt *stbuf,
+                          int32_t valid, dht_layout_t *layout)
+{
+        int           missing_attr = 0;
+        int           i     = 0;
+        dht_local_t  *local = NULL;
+        xlator_t     *this = NULL;
+
+        local = frame->local;
+        this = frame->this;
+
+        for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].err == -1)
+                        missing_attr++;
+        }
+
+        if (missing_attr == 0) {
+                dht_selfheal_dir_xattr (frame, loc, layout);
+                return 0;
+        }
+
+        if (!uuid_is_null (local->gfid))
+                uuid_copy (loc->gfid, local->gfid);
+
+        local->call_cnt = missing_attr;
+        for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].err == -1) {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "setattr for %s on subvol %s",
+                                loc->path, layout->list[i].xlator->name);
+
+                        STACK_WIND (frame, dht_selfheal_dir_setattr_cbk,
+                                    layout->list[i].xlator,
+                                    layout->list[i].xlator->fops->setattr,
+                                    loc, stbuf, valid, NULL);
+                }
+        }
+
+        return 0;
+}
+
+int
 dht_selfheal_dir_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			    int op_ret, int op_errno,
+                            int op_ret, int op_errno,
                             inode_t *inode, struct iatt *stbuf,
-                            struct iatt *preparent, struct iatt *postparent)
+                            struct iatt *preparent, struct iatt *postparent,
+                            dict_t *xdata)
 {
-	dht_local_t   *local = NULL;
-	dht_layout_t  *layout = NULL;
-	call_frame_t  *prev = NULL;
-	xlator_t      *subvol = NULL;
-	int            i = 0;
-	int            this_call_cnt = 0;
-
-
-	local  = frame->local;
-	layout = local->selfheal.layout;
-	prev   = cookie;
-	subvol = prev->this;
-
-	if ((op_ret == 0) || ((op_ret == -1) && (op_errno == EEXIST))) {
-		for (i = 0; i < layout->cnt; i++) {
-			if (layout->list[i].xlator == subvol) {
-				layout->list[i].err = -1;
-				break;
-			}
-		}
-	}
+        dht_local_t   *local = NULL;
+        dht_layout_t  *layout = NULL;
+        call_frame_t  *prev = NULL;
+        xlator_t      *subvol = NULL;
+        int            i = 0;
+        int            this_call_cnt = 0;
 
-        if (op_ret)
-                goto out;
 
+        local  = frame->local;
+        layout = local->selfheal.layout;
+        prev   = cookie;
+        subvol = prev->this;
 
-        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
-        if (prev->this == local->hashed_subvol)
-                local->ia_ino = local->stbuf.ia_ino;
+        if ((op_ret == 0) || ((op_ret == -1) && (op_errno == EEXIST))) {
+                for (i = 0; i < layout->cnt; i++) {
+                        if (layout->list[i].xlator == subvol) {
+                                layout->list[i].err = -1;
+                                break;
+                        }
+                }
+        }
+
+        if (op_ret) {
+                gf_log (this->name, ((op_errno == EEXIST) ? GF_LOG_DEBUG :
+                                     GF_LOG_WARNING),
+                        "selfhealing directory %s failed: %s",
+                        local->loc.path, strerror (op_errno));
+                goto out;
+        }
 
+        dht_iatt_merge (this, &local->stbuf, stbuf, prev->this);
         dht_iatt_merge (this, &local->preparent, preparent, prev->this);
         dht_iatt_merge (this, &local->postparent, postparent, prev->this);
 
 out:
-	this_call_cnt = dht_frame_return (frame);
+        this_call_cnt = dht_frame_return (frame);
 
-	if (is_last_call (this_call_cnt)) {
-		dht_selfheal_dir_xattr (frame, &local->loc, layout);
-	}
+        if (is_last_call (this_call_cnt)) {
+                dht_selfheal_dir_setattr (frame, &local->loc, &local->stbuf, 0xffffff, layout);
+        }
 
-	return 0;
+        return 0;
 }
 
+void
+dht_selfheal_dir_mkdir_setacl (dict_t *xattr, dict_t *dict)
+{
+        data_t          *acl_default = NULL;
+        data_t          *acl_access = NULL;
+        xlator_t        *this = NULL;
+        int     ret = -1;
+
+        GF_ASSERT (xattr);
+        GF_ASSERT (dict);
+
+        this = THIS;
+        GF_ASSERT (this);
+
+        acl_default = dict_get (xattr, POSIX_ACL_DEFAULT_XATTR);
+
+        if (!acl_default) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "ACL_DEFAULT xattr not present");
+                goto cont;
+        }
+        ret = dict_set (dict, POSIX_ACL_DEFAULT_XATTR, acl_default);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Could not set ACL_DEFAULT xattr");
+cont:
+        acl_access = dict_get (xattr, POSIX_ACL_ACCESS_XATTR);
+        if (!acl_access) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "ACL_ACCESS xattr not present");
+                goto out;
+        }
+        ret = dict_set (dict, POSIX_ACL_ACCESS_XATTR, acl_access);
+        if (ret)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Could not set ACL_ACCESS xattr");
+
+out:
+        return;
+}
 
 int
 dht_selfheal_dir_mkdir (call_frame_t *frame, loc_t *loc,
-			dht_layout_t *layout, int force)
+                        dht_layout_t *layout, int force)
 {
-	int           missing_dirs = 0;
-	int           i     = 0;
-	int           ret   = -1;
-	dht_local_t  *local = NULL;
-	xlator_t     *this = NULL;
+        int           missing_dirs = 0;
+        int           i     = 0;
+        int           ret   = -1;
+        dht_local_t  *local = NULL;
+        xlator_t     *this = NULL;
         dict_t       *dict = NULL;
 
-	local = frame->local;
-	this = frame->this;
+        local = frame->local;
+        this = frame->this;
 
-	for (i = 0; i < layout->cnt; i++) {
-		if (layout->list[i].err == ENOENT || force)
-			missing_dirs++;
-	}
+        for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].err == ENOENT || force)
+                        missing_dirs++;
+        }
 
-	if (missing_dirs == 0) {
-		dht_selfheal_dir_xattr (frame, loc, layout);
-		return 0;
-	}
+        if (missing_dirs == 0) {
+                dht_selfheal_dir_setattr (frame, loc, &local->stbuf, 0xffffffff, layout);
+                return 0;
+        }
 
-	local->call_cnt = missing_dirs;
+        local->call_cnt = missing_dirs;
         if (!uuid_is_null (local->gfid)) {
                 dict = dict_new ();
                 if (!dict)
@@ -274,37 +507,40 @@ dht_selfheal_dir_mkdir (call_frame_t *frame, loc_t *loc,
 
                 ret = dict_set_static_bin (dict, "gfid-req", local->gfid, 16);
                 if (ret)
-                        gf_log (this->name, GF_LOG_INFO,
-                                "failed to set gfid in dict");
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s: failed to set gfid in dict", loc->path);
         } else if (local->params) {
                 /* Send the dictionary from higher layers directly */
                 dict = dict_ref (local->params);
         }
+        /* Set acls */
+        if (local->xattr && dict)
+                dht_selfheal_dir_mkdir_setacl (local->xattr, dict);
 
         if (!dict)
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "dict is NULL, need to make sure gfid's are same");
-
-	for (i = 0; i < layout->cnt; i++) {
-		if (layout->list[i].err == ENOENT || force) {
-			gf_log (this->name, GF_LOG_TRACE,
-				"creating directory %s on subvol %s",
-				loc->path, layout->list[i].xlator->name);
-
-			STACK_WIND (frame, dht_selfheal_dir_mkdir_cbk,
-				    layout->list[i].xlator,
-				    layout->list[i].xlator->fops->mkdir,
-				    loc,
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dict is NULL, need to make sure gfids are same");
+
+        for (i = 0; i < layout->cnt; i++) {
+                if (layout->list[i].err == ENOENT || force) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "creating directory %s on subvol %s",
+                                loc->path, layout->list[i].xlator->name);
+
+                        STACK_WIND (frame, dht_selfheal_dir_mkdir_cbk,
+                                    layout->list[i].xlator,
+                                    layout->list[i].xlator->fops->mkdir,
+                                    loc,
                                     st_mode_from_ia (local->stbuf.ia_prot,
                                                      local->stbuf.ia_type),
-                                    dict);
-		}
-	}
+                                    0, dict);
+                }
+        }
 
         if (dict)
                 dict_unref (dict);
 
-	return 0;
+        return 0;
 }
 
 
@@ -316,7 +552,7 @@ dht_selfheal_layout_alloc_start (xlator_t *this, loc_t *loc,
         uint32_t      hashval = 0;
         int           ret = 0;
 
-        ret = dht_hash_compute (layout->type, loc->path, &hashval);
+        ret = dht_hash_compute (this, layout->type, loc->path, &hashval);
         if (ret == 0) {
                 start = (hashval % layout->cnt);
         }
@@ -324,235 +560,489 @@ dht_selfheal_layout_alloc_start (xlator_t *this, loc_t *loc,
         return start;
 }
 
-
-void
-dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
-				   dht_layout_t *layout)
+static inline int
+dht_get_layout_count (xlator_t *this, dht_layout_t *layout, int new_layout)
 {
-	xlator_t    *this = NULL;
-	uint32_t     chunk = 0;
-	int          i = 0;
-	uint32_t     start = 0;
-	int          cnt = 0;
-	int          err = 0;
-        int          start_subvol = 0;
-
-	this = frame->this;
+        int i = 0;
+        int j = 0;
+        int err = 0;
+        int count = 0;
+        dht_conf_t *conf = NULL;
+
+        /* Gets in use only for replace-brick, remove-brick */
+        conf = this->private;
+        for (i = 0; i < layout->cnt; i++) {
+                for (j = 0; j < conf->subvolume_cnt; j++) {
+                        if (conf->decommissioned_bricks[j] &&
+                            conf->decommissioned_bricks[j] == layout->list[i].xlator) {
+                                layout->list[i].err = EINVAL;
+                                break;
+                        }
+                }
+        }
 
-	for (i = 0; i < layout->cnt; i++) {
-		err = layout->list[i].err;
-		if (err == -1 || err == 0) {
-			layout->list[i].err = -1;
-			cnt++;
-		}
-	}
+        for (i = 0; i < layout->cnt; i++) {
+                err = layout->list[i].err;
+                if (err == -1 || err == 0 || err == ENOENT) {
+			/* Setting list[i].err = -1 is an indication for
+			   dht_selfheal_layout_new_directory() to assign
+			   a range. We set it to -1 based on any one of
+			   the three criteria:
+
+			   - err == -1 already, which means directory
+			     existed but layout was not set on it.
+
+			   - err == 0, which means directory exists and
+			     has an old layout piece which will be
+			     overwritten now.
+
+			   - err == ENOENT, which means directory does
+			     not exist (possibly racing with mkdir or
+			     finishing half done mkdir). The missing
+			     directory will be attempted to be recreated.
+
+			     It is important to note that it is safe
+			     to race with mkdir() as self-heal and
+			     mkdir are idempotent operations. Both will
+			     strive to set the directory and layouts to
+			     the same final state.
+			*/
+                        count++;
+			if (!err)
+				layout->list[i].err = -1;
+                }
+        }
 
         /* no subvolume has enough space, but can't stop directory creation */
-        if (!cnt) {
+        if (!count || !new_layout) {
                 for (i = 0; i < layout->cnt; i++) {
                         err = layout->list[i].err;
                         if (err == ENOSPC) {
                                 layout->list[i].err = -1;
-                                cnt++;
+                                count++;
                         }
                 }
         }
 
-	chunk = ((unsigned long) 0xffffffff) / ((cnt) ? cnt : 1);
+        /* if layout->spread_cnt is set, check if it is <= available
+         * subvolumes (down brick and decommissioned bricks are considered
+         * un-availbale). Else return count (available up bricks) */
+        count = ((layout->spread_cnt &&
+                 (layout->spread_cnt <= count)) ?
+                 layout->spread_cnt : ((count) ? count : 1));
 
-	start_subvol = dht_selfheal_layout_alloc_start (this, loc, layout);
+        return count;
+}
 
-	for (i = start_subvol; i < layout->cnt; i++) {
-		err = layout->list[i].err;
-		if (err == -1) {
-			layout->list[i].start = start;
-			layout->list[i].stop  = start + chunk - 1;
-			
-			start = start + chunk;
 
-			gf_log (this->name, GF_LOG_TRACE,
-				"gave fix: %u - %u on %s for %s",
-				layout->list[i].start, layout->list[i].stop,
-				layout->list[i].xlator->name, loc->path);
-			if (--cnt == 0) {
-				layout->list[i].stop = 0xffffffff;
-				break;
-			}
-		}
-	}
+void dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
+					dht_layout_t *new_layout);
+
+void dht_layout_entry_swap (dht_layout_t *layout, int i, int j);
+void dht_layout_range_swap (dht_layout_t *layout, int i, int j);
+
+/*
+ * It's a bit icky using local variables in a macro, but it makes the rest
+ * of the code a lot clearer.
+ */
+#define OV_ENTRY(x,y)      table[x*new->cnt+y]
+
+void
+dht_selfheal_layout_maximize_overlap (call_frame_t *frame, loc_t *loc,
+				      dht_layout_t *new, dht_layout_t *old)
+{
+	int           i            = 0;
+	int           j            = 0;
+	uint32_t      curr_overlap = 0;
+	uint32_t      max_overlap  = 0;
+	int           max_overlap_idx = -1;
+	uint32_t      overlap      = 0;
+        uint32_t     *table = NULL;
+
+	dht_layout_sort_volname (old);
+	/* Now both old_layout->list[] and new_layout->list[]
+	   are match the same xlators/subvolumes. i.e,
+	   old_layout->[i] and new_layout->[i] are referring
+	   to the same subvolumes
+	*/
+
+        /* Build a table of overlaps between new[i] and old[j]. */
+        table = alloca(sizeof(overlap)*old->cnt*new->cnt);
+        if (!table) {
+                return;
+        }
+        memset(table,0,sizeof(overlap)*old->cnt*new->cnt);
+        for (i = 0; i < new->cnt; ++i) {
+                for (j = 0; j < old->cnt; ++j) {
+                        OV_ENTRY(i,j) = dht_overlap_calc(old,j,new,i);
+                }
+        }
+
+	for (i = 0; i < new->cnt; i++) {
+		if (new->list[i].err > 0) {
+			/* Subvol might be marked for decommission
+			   with EINVAL, or some other serious error
+			   marked with positive errno.
+			*/
+			continue;
+                }
+
+                max_overlap = 0;
+                max_overlap_idx = i;
+                for (j = (i + 1); j < new->cnt; ++j) {
+                        if (new->list[j].err > 0) {
+			        /* Subvol might be marked for decommission
+			        with EINVAL, or some other serious error
+			        marked with positive errno.
+			        */
+			        continue;
+                        }
+                        /* Calculate the overlap now. */
+                        curr_overlap = OV_ENTRY(i,i) + OV_ENTRY(j,j);
+                        /* Calculate the overlap after the proposed swap. */
+                        overlap = OV_ENTRY(i,j) + OV_ENTRY(j,i);
+                        /* Are we better than status quo? */
+                        if (overlap > curr_overlap) {
+                                overlap -= curr_overlap;
+                                /* Are we better than the previous choice? */
+                                if (overlap > max_overlap) {
+                                        max_overlap = overlap;
+                                        max_overlap_idx = j;
+                                }
+                        }
+                }
 
-	for (i = 0; i < start_subvol; i++) {
-		err = layout->list[i].err;
-		if (err == -1) {
-			layout->list[i].start = start;
-			layout->list[i].stop  = start + chunk - 1;
-			
-			start = start + chunk;
-
-			gf_log (this->name, GF_LOG_TRACE,
-				"gave fix: %u - %u on %s for %s",
-				layout->list[i].start, layout->list[i].stop,
-				layout->list[i].xlator->name, loc->path);
-			if (--cnt == 0) {
-				layout->list[i].stop = 0xffffffff;
-				break;
-			}
-		}
+		if (max_overlap_idx != i) {
+ 			dht_layout_range_swap (new, i, max_overlap_idx);
+                        /* Need to swap the table values too. */
+                        for (j = 0; j < old->cnt; ++j) {
+                                overlap = OV_ENTRY(i,j);
+                                OV_ENTRY(i,j) = OV_ENTRY(max_overlap_idx,j);
+                                OV_ENTRY(max_overlap_idx,j) = overlap;
+                        }
+                }
 	}
 }
 
 
+dht_layout_t *
+dht_fix_layout_of_directory (call_frame_t *frame, loc_t *loc,
+                             dht_layout_t *layout)
+{
+        int           i            = 0;
+        xlator_t     *this         = NULL;
+        dht_layout_t *new_layout   = NULL;
+        dht_conf_t   *priv         = NULL;
+        dht_local_t  *local        = NULL;
+        uint32_t      subvol_down  = 0;
+        int           ret          = 0;
+
+        this  = frame->this;
+        priv  = this->private;
+        local = frame->local;
+
+        if (layout->type == DHT_HASH_TYPE_DM_USER) {
+                gf_log (THIS->name, GF_LOG_DEBUG, "leaving %s alone",
+                        loc->path);
+                goto done;
+        }
+
+        new_layout = dht_layout_new (this, priv->subvolume_cnt);
+        if (!new_layout)
+                goto done;
+
+        /* If a subvolume is down, do not re-write the layout. */
+        ret = dht_layout_anomalies (this, loc, layout, NULL, NULL, NULL,
+                                    &subvol_down, NULL, NULL);
+
+        if (subvol_down || (ret == -1)) {
+                gf_log (this->name, GF_LOG_WARNING, "%u subvolume(s) are down"
+                        ". Skipping fix layout.", subvol_down);
+                GF_FREE (new_layout);
+                return NULL;
+        }
+
+        for (i = 0; i < new_layout->cnt; i++) {
+		if (layout->list[i].err != ENOSPC)
+			new_layout->list[i].err = layout->list[i].err;
+		else
+			new_layout->list[i].err = -1;
+
+		new_layout->list[i].xlator = layout->list[i].xlator;
+        }
+
+	/* First give it a layout as though it is a new directory. This
+	   ensures rotation to kick in */
+        dht_layout_sort_volname (new_layout);
+	dht_selfheal_layout_new_directory (frame, loc, new_layout);
+
+	/* Now selectively re-assign ranges only when it helps */
+	dht_selfheal_layout_maximize_overlap (frame, loc, new_layout, layout);
+
+done:
+        if (new_layout) {
+                /* Now that the new layout has all the proper layout, change the
+                   inode context */
+                dht_layout_set (this, loc->inode, new_layout);
+
+                /* Make sure the extra 'ref' for existing layout is removed */
+                dht_layout_unref (this, local->layout);
+
+                local->layout = new_layout;
+        }
+
+        return local->layout;
+}
+
+
+void
+dht_selfheal_layout_new_directory (call_frame_t *frame, loc_t *loc,
+                                   dht_layout_t *layout)
+{
+        xlator_t    *this = NULL;
+        uint32_t     chunk = 0;
+        int          i = 0;
+        uint32_t     start = 0;
+        int          cnt = 0;
+        int          err = 0;
+        int          start_subvol = 0;
+
+        this = frame->this;
+
+        cnt = dht_get_layout_count (this, layout, 1);
+
+        chunk = ((unsigned long) 0xffffffff) / ((cnt) ? cnt : 1);
+
+        start_subvol = dht_selfheal_layout_alloc_start (this, loc, layout);
+
+        /* clear out the range, as we are re-computing here */
+        DHT_RESET_LAYOUT_RANGE (layout);
+        for (i = start_subvol; i < layout->cnt; i++) {
+                err = layout->list[i].err;
+                if (err == -1 || err == ENOENT) {
+                        DHT_SET_LAYOUT_RANGE(layout, i, start, chunk,
+                                             cnt, loc->path);
+                        if (--cnt == 0) {
+                                layout->list[i].stop = 0xffffffff;
+                                goto done;
+                        }
+                        start += chunk;
+                }
+        }
+
+        for (i = 0; i < start_subvol; i++) {
+                err = layout->list[i].err;
+                if (err == -1 || err == ENOENT) {
+                        DHT_SET_LAYOUT_RANGE(layout, i, start, chunk,
+                                             cnt, loc->path);
+                        if (--cnt == 0) {
+                                layout->list[i].stop = 0xffffffff;
+                                goto done;
+                        }
+                        start += chunk;
+                }
+        }
+
+done:
+        return;
+}
+
 int
 dht_selfheal_dir_getafix (call_frame_t *frame, loc_t *loc,
-			  dht_layout_t *layout)
+                          dht_layout_t *layout)
 {
-	dht_conf_t  *conf = NULL;
-	xlator_t    *this = NULL;
-	dht_local_t *local = NULL;
-	int          missing = -1;
-	int          down = -1;
-	int          holes = -1;
-	int          ret = -1;
-	int          i = -1;
-	int          overlaps = -1;
-
-	this = frame->this;
-	conf = this->private;
-	local = frame->local;
-
-	missing = local->selfheal.missing;
-	down = local->selfheal.down;
-	holes = local->selfheal.hole_cnt;
-	overlaps = local->selfheal.overlaps_cnt;
-
-	if ((missing + down) == conf->subvolume_cnt) {
-		dht_selfheal_layout_new_directory (frame, loc, layout);
-		ret = 0;
-	}
+        dht_local_t *local = NULL;
+        uint32_t     holes = 0;
+        int          ret = -1;
+        int          i = -1;
+        uint32_t     overlaps = 0;
 
-	if (holes <= down) {
-		/* the down subvol might fill up the holes */
-		ret = 0;
-	}
+        local = frame->local;
 
-	if (holes || overlaps) {
-		dht_selfheal_layout_new_directory (frame, loc, layout);
-		ret = 0;
-	}
+        holes = local->selfheal.hole_cnt;
+        overlaps = local->selfheal.overlaps_cnt;
 
-	for (i = 0; i < layout->cnt; i++) {
-		/* directory not present */
-		if (layout->list[i].err == ENOENT) {
-			ret = 0;
-			break;
-		}
-	}
+        if (holes || overlaps) {
+                dht_selfheal_layout_new_directory (frame, loc, layout);
+                ret = 0;
+        }
+
+        for (i = 0; i < layout->cnt; i++) {
+                /* directory not present */
+                if (layout->list[i].err == ENOENT) {
+                        ret = 0;
+                        break;
+                }
+        }
 
-	/* TODO: give a fix to these non-virgins */
+        /* TODO: give a fix to these non-virgins */
 
-	return ret;
+        return ret;
 }
 
 int
-dht_selfheal_new_directory (call_frame_t *frame, 
-			    dht_selfheal_dir_cbk_t dir_cbk,
-			    dht_layout_t *layout)
+dht_selfheal_new_directory (call_frame_t *frame,
+                            dht_selfheal_dir_cbk_t dir_cbk,
+                            dht_layout_t *layout)
 {
-	dht_local_t *local = NULL;
+        dht_local_t *local = NULL;
 
-	local = frame->local;
+        local = frame->local;
 
-	local->selfheal.dir_cbk = dir_cbk;
-	local->selfheal.layout = dht_layout_ref (frame->this, layout);
+        local->selfheal.dir_cbk = dir_cbk;
+        local->selfheal.layout = dht_layout_ref (frame->this, layout);
 
-	dht_layout_sort_volname (layout);
-	dht_selfheal_layout_new_directory (frame, &local->loc, layout);
-	dht_selfheal_dir_xattr (frame, &local->loc, layout);
-	return 0;
+        dht_layout_sort_volname (layout);
+        dht_selfheal_layout_new_directory (frame, &local->loc, layout);
+        dht_selfheal_dir_xattr (frame, &local->loc, layout);
+        return 0;
+}
+
+int
+dht_fix_directory_layout (call_frame_t *frame,
+                          dht_selfheal_dir_cbk_t dir_cbk,
+                          dht_layout_t *layout)
+{
+        dht_local_t  *local = NULL;
+        dht_layout_t *tmp_layout = NULL;
+
+        local = frame->local;
+
+        local->selfheal.dir_cbk = dir_cbk;
+        local->selfheal.layout = dht_layout_ref (frame->this, layout);
+
+        /* No layout sorting required here */
+        tmp_layout = dht_fix_layout_of_directory (frame, &local->loc, layout);
+        if (!tmp_layout) {
+                return -1;
+        }
+        dht_fix_dir_xattr (frame, &local->loc, tmp_layout);
+
+        return 0;
 }
 
 
 int
 dht_selfheal_directory (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
-			loc_t *loc, dht_layout_t *layout)
+                        loc_t *loc, dht_layout_t *layout)
 {
-	dht_local_t *local    = NULL;
-	uint32_t     holes    = 0;
-	uint32_t     down     = 0;
-	uint32_t     misc     = 0;
-	int          ret      = 0;
-	xlator_t    *this     = NULL;
+        dht_local_t *local    = NULL;
+        uint32_t     down     = 0;
+        uint32_t     misc     = 0;
+        int          ret      = 0;
+        xlator_t    *this     = NULL;
 
-	local = frame->local;
-	this = frame->this;
+        local = frame->local;
+        this = frame->this;
 
-	dht_layout_anomalies (this, loc, layout,
+        dht_layout_anomalies (this, loc, layout,
                               &local->selfheal.hole_cnt,
                               &local->selfheal.overlaps_cnt,
-                              &local->selfheal.missing,
-                              &local->selfheal.down,
-                              &local->selfheal.misc);
-
-	holes    = local->selfheal.hole_cnt;
-	down     = local->selfheal.down;
-	misc     = local->selfheal.misc;
-
-	local->selfheal.dir_cbk = dir_cbk;
-	local->selfheal.layout = dht_layout_ref (this, layout);
-
-	if (down) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"%d subvolumes down -- not fixing", down);
-                ret = -1;
-                local->op_errno = ESTALE;
-		goto sorry_no_fix;
-	}
+                              NULL, &local->selfheal.down,
+                              &local->selfheal.misc, NULL);
 
-	if (misc) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"%d subvolumes have unrecoverable errors", misc);
-                ret = -1;
-                local->op_errno = ESTALE;
-		goto sorry_no_fix;
-	}
+        down     = local->selfheal.down;
+        misc     = local->selfheal.misc;
 
-	dht_layout_sort_volname (layout);
-	ret = dht_selfheal_dir_getafix (frame, loc, layout);
+        local->selfheal.dir_cbk = dir_cbk;
+        local->selfheal.layout = dht_layout_ref (this, layout);
 
-	if (ret == -1) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"not able to form layout for the directory");
-                ret = -1;
-                local->op_errno = ESTALE;
-		goto sorry_no_fix;
-	}
+        if (down) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%d subvolumes down -- not fixing", down);
+                ret = 0;
+                goto sorry_no_fix;
+        }
+
+        if (misc) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "%d subvolumes have unrecoverable errors", misc);
+                ret = 0;
+                goto sorry_no_fix;
+        }
+
+        dht_layout_sort_volname (layout);
+        ret = dht_selfheal_dir_getafix (frame, loc, layout);
+
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "not able to form layout for the directory");
+                goto sorry_no_fix;
+        }
 
-	dht_selfheal_dir_mkdir (frame, loc, layout, 0);
+        dht_selfheal_dir_mkdir (frame, loc, layout, 0);
 
-	return 0;
+        return 0;
 
 sorry_no_fix:
-	/* TODO: need to put appropriate local->op_errno */
-	dht_selfheal_dir_finish (frame, this, ret);
+        /* TODO: need to put appropriate local->op_errno */
+        dht_selfheal_dir_finish (frame, this, ret);
 
-	return 0;
+        return 0;
 }
 
 
 int
 dht_selfheal_restore (call_frame_t *frame, dht_selfheal_dir_cbk_t dir_cbk,
-		      loc_t *loc, dht_layout_t *layout)
+                      loc_t *loc, dht_layout_t *layout)
 {
-	int          ret = 0;
-	dht_local_t *local    = NULL;
+        int          ret = 0;
+        dht_local_t *local    = NULL;
 
+        local = frame->local;
 
-	local = frame->local;
+        local->selfheal.dir_cbk = dir_cbk;
+        local->selfheal.layout = dht_layout_ref (frame->this, layout);
 
-	local->selfheal.dir_cbk = dir_cbk;
-	local->selfheal.layout = dht_layout_ref (frame->this, layout);
+        ret = dht_selfheal_dir_mkdir (frame, loc, layout, 1);
 
-	ret = dht_selfheal_dir_mkdir (frame, loc, layout, 1);
+        return ret;
+}
+
+int
+dht_dir_attr_heal (void *data)
+{
+        call_frame_t    *frame = NULL;
+        dht_local_t     *local = NULL;
+        xlator_t        *subvol = NULL;
+        xlator_t        *this  = NULL;
+        dht_conf_t      *conf  = NULL;
+        int              call_cnt = 0;
+        int              ret   = -1;
+        int              i     = 0;
+
+        GF_VALIDATE_OR_GOTO ("dht", data, out);
+
+        frame = data;
+        local = frame->local;
+        this = frame->this;
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", local, out);
+        conf = this->private;
+        GF_VALIDATE_OR_GOTO ("dht", conf, out);
+
+        call_cnt = conf->subvolume_cnt;
+
+        for (i = 0; i < call_cnt; i++) {
+                subvol = conf->subvolumes[i];
+                if (!subvol || (subvol == dht_first_up_subvol (this)))
+                        continue;
+                ret = syncop_setattr (subvol, &local->loc, &local->stbuf,
+                                      (GF_SET_ATTR_UID | GF_SET_ATTR_GID),
+                                      NULL, NULL);
+                if (ret) {
+                        gf_log ("dht", GF_LOG_ERROR, "Failed to set uid/gid on"
+                                " %s on %s subvol (%s)", local->loc.path,
+                                subvol->name, strerror (-ret));
+                }
+        }
+out:
+        return 0;
+}
 
-	return ret;
+int
+dht_dir_attr_heal_done (int ret, call_frame_t *sync_frame, void *data)
+{
+        DHT_STACK_DESTROY (sync_frame);
+        return 0;
 }
diff --git a/xlators/cluster/dht/src/dht-shared.c b/xlators/cluster/dht/src/dht-shared.c
new file mode 100644
index 000000000..f2e7467ab
--- /dev/null
+++ b/xlators/cluster/dht/src/dht-shared.c
@@ -0,0 +1,780 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+/* TODO: add NS locking */
+
+#include "statedump.h"
+#include "dht-common.h"
+
+/* TODO:
+   - use volumename in xattr instead of "dht"
+   - use NS locks
+   - handle all cases in self heal layout reconstruction
+   - complete linkfile selfheal
+*/
+struct volume_options options[];
+
+void
+dht_layout_dump (dht_layout_t  *layout, const char *prefix)
+{
+
+        char    key[GF_DUMP_MAX_BUF_LEN];
+        int     i = 0;
+
+        if (!layout)
+                goto out;
+        if (!prefix)
+                goto out;
+
+        gf_proc_dump_build_key(key, prefix, "cnt");
+        gf_proc_dump_write(key, "%d", layout->cnt);
+        gf_proc_dump_build_key(key, prefix, "preset");
+        gf_proc_dump_write(key, "%d", layout->preset);
+        gf_proc_dump_build_key(key, prefix, "gen");
+        gf_proc_dump_write(key, "%d", layout->gen);
+        if (layout->type != IA_INVAL) {
+                gf_proc_dump_build_key(key, prefix, "inode type");
+                gf_proc_dump_write(key, "%d", layout->type);
+        }
+
+        if  (!IA_ISDIR (layout->type))
+                goto out;
+
+        for (i = 0; i < layout->cnt; i++) {
+                gf_proc_dump_build_key(key, prefix,"list[%d].err", i);
+                gf_proc_dump_write(key, "%d", layout->list[i].err);
+                gf_proc_dump_build_key(key, prefix,"list[%d].start", i);
+                gf_proc_dump_write(key, "%u", layout->list[i].start);
+                gf_proc_dump_build_key(key, prefix,"list[%d].stop", i);
+                gf_proc_dump_write(key, "%u", layout->list[i].stop);
+                if (layout->list[i].xlator) {
+                        gf_proc_dump_build_key(key, prefix,
+                                               "list[%d].xlator.type", i);
+                        gf_proc_dump_write(key, "%s",
+                                           layout->list[i].xlator->type);
+                        gf_proc_dump_build_key(key, prefix,
+                                               "list[%d].xlator.name", i);
+                        gf_proc_dump_write(key, "%s",
+                                           layout->list[i].xlator->name);
+                }
+        }
+
+out:
+        return;
+}
+
+
+int32_t
+dht_priv_dump (xlator_t *this)
+{
+        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
+        char            key[GF_DUMP_MAX_BUF_LEN];
+        int             i = 0;
+        dht_conf_t      *conf = NULL;
+        int             ret = -1;
+
+        if (!this)
+                goto out;
+
+        conf = this->private;
+        if (!conf)
+                goto out;
+
+        ret = TRY_LOCK(&conf->subvolume_lock);
+        if (ret != 0) {
+                return ret;
+        }
+
+        gf_proc_dump_add_section("xlator.cluster.dht.%s.priv", this->name);
+        gf_proc_dump_build_key(key_prefix,"xlator.cluster.dht","%s.priv",
+                               this->name);
+        gf_proc_dump_write("subvol_cnt","%d", conf->subvolume_cnt);
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                snprintf (key, sizeof (key), "subvolumes[%d]", i);
+                gf_proc_dump_write(key, "%s.%s", conf->subvolumes[i]->type,
+                                   conf->subvolumes[i]->name);
+                if (conf->file_layouts && conf->file_layouts[i]){
+                        snprintf (key, sizeof (key), "file_layouts[%d]", i);
+                        dht_layout_dump(conf->file_layouts[i], key);
+                }
+                if (conf->dir_layouts && conf->dir_layouts[i]) {
+                        snprintf (key, sizeof (key), "dir_layouts[%d]", i);
+                        dht_layout_dump(conf->dir_layouts[i], key);
+                }
+                if (conf->subvolume_status) {
+
+                        snprintf (key, sizeof (key), "subvolume_status[%d]", i);
+                        gf_proc_dump_write(key, "%d",
+                                           (int)conf->subvolume_status[i]);
+                }
+
+        }
+
+        gf_proc_dump_write("search_unhashed", "%d", conf->search_unhashed);
+        gf_proc_dump_write("gen", "%d", conf->gen);
+        gf_proc_dump_write("min_free_disk", "%lf", conf->min_free_disk);
+	gf_proc_dump_write("min_free_inodes", "%lf", conf->min_free_inodes);
+        gf_proc_dump_write("disk_unit", "%c", conf->disk_unit);
+        gf_proc_dump_write("refresh_interval", "%d", conf->refresh_interval);
+        gf_proc_dump_write("unhashed_sticky_bit", "%d", conf->unhashed_sticky_bit);
+
+        if (conf->du_stats) {
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        if (!conf->subvolume_status[i])
+                                continue;
+
+                        snprintf (key, sizeof (key), "subvolumes[%d]", i);
+                        gf_proc_dump_write (key, "%s",
+                                            conf->subvolumes[i]->name);
+
+                        snprintf (key, sizeof (key),
+                                  "du_stats[%d].avail_percent", i);
+                        gf_proc_dump_write (key, "%lf",
+                                            conf->du_stats[i].avail_percent);
+
+                        snprintf (key, sizeof (key), "du_stats[%d].avail_space",
+                                  i);
+                        gf_proc_dump_write (key, "%lu",
+                                            conf->du_stats[i].avail_space);
+
+                        snprintf (key, sizeof (key),
+                                  "du_stats[%d].avail_inodes", i);
+                        gf_proc_dump_write (key, "%lf",
+                                            conf->du_stats[i].avail_inodes);
+
+                        snprintf (key, sizeof (key), "du_stats[%d].log", i);
+                        gf_proc_dump_write (key, "%lu",
+                                            conf->du_stats[i].log);
+                }
+        }
+
+        if (conf->last_stat_fetch.tv_sec)
+                gf_proc_dump_write("last_stat_fetch", "%s",
+                                    ctime(&conf->last_stat_fetch.tv_sec));
+
+        UNLOCK(&conf->subvolume_lock);
+
+out:
+        return ret;
+}
+
+int32_t
+dht_inodectx_dump (xlator_t *this, inode_t *inode)
+{
+        int             ret = -1;
+        dht_layout_t    *layout = NULL;
+
+        if (!this)
+                goto out;
+        if (!inode)
+                goto out;
+
+        ret = dht_inode_ctx_layout_get (inode, this, &layout);
+
+        if ((ret != 0) || !layout)
+                return ret;
+
+        gf_proc_dump_add_section("xlator.cluster.dht.%s.inode", this->name);
+        dht_layout_dump(layout, "layout");
+
+out:
+        return ret;
+}
+
+void
+dht_fini (xlator_t *this)
+{
+        int         i = 0;
+        dht_conf_t *conf = NULL;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+
+        conf = this->private;
+        this->private = NULL;
+        if (conf) {
+                if (conf->file_layouts) {
+                        for (i = 0; i < conf->subvolume_cnt; i++) {
+                                GF_FREE (conf->file_layouts[i]);
+                        }
+                        GF_FREE (conf->file_layouts);
+                }
+
+                GF_FREE (conf->subvolumes);
+
+                GF_FREE (conf->subvolume_status);
+
+                GF_FREE (conf);
+        }
+out:
+        return;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+
+        ret = xlator_mem_acct_init (this, gf_dht_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                        "failed");
+                return ret;
+        }
+out:
+        return ret;
+}
+
+
+int
+dht_parse_decommissioned_bricks (xlator_t *this, dht_conf_t *conf,
+                                 const char *bricks)
+{
+        int         i  = 0;
+        int         ret  = -1;
+        char       *tmpstr = NULL;
+        char       *dup_brick = NULL;
+        char       *node = NULL;
+
+        if (!conf || !bricks)
+                goto out;
+
+        dup_brick = gf_strdup (bricks);
+        node = strtok_r (dup_brick, ",", &tmpstr);
+        while (node) {
+                for (i = 0; i < conf->subvolume_cnt; i++) {
+                        if (!strcmp (conf->subvolumes[i]->name, node)) {
+                                conf->decommissioned_bricks[i] =
+                                        conf->subvolumes[i];
+                                        conf->decommission_subvols_cnt++;
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "decommissioning subvolume %s",
+                                        conf->subvolumes[i]->name);
+                                break;
+                        }
+                }
+                if (i == conf->subvolume_cnt) {
+                        /* Wrong node given. */
+                        goto out;
+                }
+                node = strtok_r (NULL, ",", &tmpstr);
+        }
+
+        ret = 0;
+        conf->decommission_in_progress = 1;
+out:
+        GF_FREE (dup_brick);
+
+        return ret;
+}
+
+
+int
+dht_decommissioned_remove (xlator_t *this, dht_conf_t *conf)
+{
+        int         i  = 0;
+        int         ret  = -1;
+
+        if (!conf)
+                goto out;
+
+        for (i = 0; i < conf->subvolume_cnt; i++) {
+                if (conf->decommissioned_bricks[i]) {
+                        conf->decommissioned_bricks[i] = NULL;
+                        conf->decommission_subvols_cnt--;
+                }
+        }
+
+        ret = 0;
+out:
+
+        return ret;
+}
+void
+dht_init_regex (xlator_t *this, dict_t *odict, char *name,
+                regex_t *re, gf_boolean_t *re_valid)
+{
+        char    *temp_str;
+
+        if (dict_get_str (odict, name, &temp_str) != 0) {
+                if (strcmp(name,"rsync-hash-regex")) {
+                        return;
+                }
+                temp_str = "^\\.(.+)\\.[^.]+$";
+        }
+
+        if (*re_valid) {
+                regfree(re);
+                *re_valid = _gf_false;
+        }
+
+        if (!strcmp(temp_str,"none")) {
+                return;
+        }
+
+        if (regcomp(re,temp_str,REG_EXTENDED) == 0) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "using regex %s = %s", name, temp_str);
+                *re_valid = _gf_true;
+        }
+        else {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "compiling regex %s failed", temp_str);
+        }
+}
+
+int
+dht_reconfigure (xlator_t *this, dict_t *options)
+{
+        dht_conf_t      *conf = NULL;
+        char            *temp_str = NULL;
+        gf_boolean_t     search_unhashed;
+        int              ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("dht", this, out);
+        GF_VALIDATE_OR_GOTO ("dht", options, out);
+
+        conf = this->private;
+        if (!conf)
+                return 0;
+
+        if (dict_get_str (options, "lookup-unhashed", &temp_str) == 0) {
+                /* If option is not "auto", other options _should_ be boolean*/
+                if (strcasecmp (temp_str, "auto")) {
+                        if (!gf_string2boolean (temp_str, &search_unhashed)) {
+                                gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
+                                       " lookup-unhashed reconfigured (%s)",
+                                       temp_str);
+                                conf->search_unhashed = search_unhashed;
+                        } else {
+                                gf_log(this->name, GF_LOG_ERROR, "Reconfigure:"
+                                       " lookup-unhashed should be boolean,"
+                                       " not (%s), defaulting to (%d)",
+                                       temp_str, conf->search_unhashed);
+                                ret = -1;
+                                goto out;
+                        }
+                } else {
+                        gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
+                               " lookup-unhashed reconfigured auto ");
+                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
+                }
+        }
+
+	GF_OPTION_RECONF ("min-free-disk", conf->min_free_disk, options,
+                          percent_or_size, out);
+        /* option can be any one of percent or bytes */
+        conf->disk_unit = 0;
+        if (conf->min_free_disk < 100.0)
+                conf->disk_unit = 'p';
+
+	GF_OPTION_RECONF ("min-free-inodes", conf->min_free_inodes, options,
+                          percent, out);
+
+        GF_OPTION_RECONF ("directory-layout-spread", conf->dir_spread_cnt,
+                          options, uint32, out);
+
+        GF_OPTION_RECONF ("readdir-optimize", conf->readdir_optimize, options,
+                          bool, out);
+        if (conf->defrag) {
+                GF_OPTION_RECONF ("rebalance-stats", conf->defrag->stats,
+                                  options, bool, out);
+        }
+
+        if (dict_get_str (options, "decommissioned-bricks", &temp_str) == 0) {
+                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
+                if (ret == -1)
+                        goto out;
+        } else {
+                ret = dht_decommissioned_remove (this, conf);
+                if (ret == -1)
+                        goto out;
+        }
+
+        dht_init_regex (this, options, "rsync-hash-regex",
+                        &conf->rsync_regex, &conf->rsync_regex_valid);
+        dht_init_regex (this, options, "extra-hash-regex",
+                        &conf->extra_regex, &conf->extra_regex_valid);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+gf_defrag_pattern_list_fill (xlator_t *this, gf_defrag_info_t *defrag, char *data)
+{
+        int                    ret = -1;
+        char                  *tmp_str = NULL;
+        char                  *tmp_str1 = NULL;
+        char                  *dup_str = NULL;
+        char                  *num = NULL;
+        char                  *pattern_str = NULL;
+        char                  *pattern = NULL;
+        gf_defrag_pattern_list_t *temp_list = NULL;
+        gf_defrag_pattern_list_t *pattern_list = NULL;
+
+        if (!this || !defrag || !data)
+                goto out;
+
+        /* Get the pattern for pattern list. "pattern:<optional-size>"
+         * eg: *avi, *pdf:10MB, *:1TB
+         */
+        pattern_str = strtok_r (data, ",", &tmp_str);
+        while (pattern_str) {
+                dup_str = gf_strdup (pattern_str);
+                pattern_list = GF_CALLOC (1, sizeof (gf_defrag_pattern_list_t),
+                                        1);
+                if (!pattern_list) {
+                        goto out;
+                }
+                pattern = strtok_r (dup_str, ":", &tmp_str1);
+                num = strtok_r (NULL, ":", &tmp_str1);
+                if (!pattern)
+                        goto out;
+                if (!num) {
+                        if (gf_string2bytesize_uint64(pattern, &pattern_list->size)
+                             == 0) {
+                                pattern = "*";
+                        }
+                } else if (gf_string2bytesize_uint64 (num, &pattern_list->size) != 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "invalid number format \"%s\"", num);
+                        goto out;
+                }
+                memcpy (pattern_list->path_pattern, pattern, strlen (dup_str));
+
+                if (!defrag->defrag_pattern)
+                        temp_list = NULL;
+                else
+                        temp_list = defrag->defrag_pattern;
+
+                pattern_list->next = temp_list;
+
+                defrag->defrag_pattern = pattern_list;
+                pattern_list = NULL;
+
+                GF_FREE (dup_str);
+                dup_str = NULL;
+
+                pattern_str = strtok_r (NULL, ",", &tmp_str);
+        }
+
+        ret = 0;
+out:
+        if (ret)
+                GF_FREE (pattern_list);
+        GF_FREE (dup_str);
+
+        return ret;
+}
+
+int
+dht_init (xlator_t *this)
+{
+        dht_conf_t                      *conf           = NULL;
+        char                            *temp_str       = NULL;
+        int                              ret            = -1;
+        int                              i              = 0;
+        gf_defrag_info_t                *defrag         = NULL;
+        int                              cmd            = 0;
+        char                            *node_uuid      = NULL;
+
+
+        GF_VALIDATE_OR_GOTO ("dht", this, err);
+
+        if (!this->children) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Distribute needs more than one subvolume");
+                return -1;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dangling volume. check volfile");
+        }
+
+        conf = GF_CALLOC (1, sizeof (*conf), gf_dht_mt_dht_conf_t);
+        if (!conf) {
+                goto err;
+        }
+
+        ret = dict_get_int32 (this->options, "rebalance-cmd", &cmd);
+
+        if (cmd) {
+                defrag = GF_CALLOC (1, sizeof (gf_defrag_info_t),
+                                    gf_defrag_info_mt);
+
+                GF_VALIDATE_OR_GOTO (this->name, defrag, err);
+
+                LOCK_INIT (&defrag->lock);
+
+                defrag->is_exiting = 0;
+
+                conf->defrag = defrag;
+
+                ret = dict_get_str (this->options, "node-uuid", &node_uuid);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "node-uuid not "
+                                "specified");
+                        goto err;
+                }
+
+                if (uuid_parse (node_uuid, defrag->node_uuid)) {
+                        gf_log (this->name, GF_LOG_ERROR, "Cannot parse "
+                                "glusterd node uuid");
+                        goto err;
+                }
+
+                defrag->cmd = cmd;
+
+                defrag->stats = _gf_false;
+        }
+
+        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
+        if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
+                /* If option is not "auto", other options _should_ be boolean */
+                if (strcasecmp (temp_str, "auto"))
+                        gf_string2boolean (temp_str, &conf->search_unhashed);
+                else
+                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
+        }
+
+        GF_OPTION_INIT ("unhashed-sticky-bit", conf->unhashed_sticky_bit, bool,
+                        err);
+
+        GF_OPTION_INIT ("use-readdirp", conf->use_readdirp, bool, err);
+
+	GF_OPTION_INIT ("min-free-disk", conf->min_free_disk, percent_or_size,
+			err);
+
+        GF_OPTION_INIT ("min-free-inodes", conf->min_free_inodes, percent,
+                        err);
+
+        conf->dir_spread_cnt = conf->subvolume_cnt;
+        GF_OPTION_INIT ("directory-layout-spread", conf->dir_spread_cnt,
+                        uint32, err);
+
+        GF_OPTION_INIT ("assert-no-child-down", conf->assert_no_child_down,
+                        bool, err);
+
+        GF_OPTION_INIT ("readdir-optimize", conf->readdir_optimize, bool, err);
+
+        if (defrag) {
+                GF_OPTION_INIT ("rebalance-stats", defrag->stats, bool, err);
+                if (dict_get_str (this->options, "rebalance-filter", &temp_str)
+                    == 0) {
+                        if (gf_defrag_pattern_list_fill (this, defrag, temp_str)
+                            == -1) {
+                                gf_log (this->name, GF_LOG_ERROR, "Cannot parse"
+                                        " rebalance-filter (%s)", temp_str);
+                                goto err;
+                        }
+                }
+        }
+
+        /* option can be any one of percent or bytes */
+        conf->disk_unit = 0;
+        if (conf->min_free_disk < 100)
+                conf->disk_unit = 'p';
+
+        ret = dht_init_subvolumes (this, conf);
+        if (ret == -1) {
+                goto err;
+        }
+
+        if (dict_get_str (this->options, "decommissioned-bricks", &temp_str) == 0) {
+                ret = dht_parse_decommissioned_bricks (this, conf, temp_str);
+                if (ret == -1)
+                        goto err;
+        }
+
+        dht_init_regex (this, this->options, "rsync-hash-regex",
+                        &conf->rsync_regex, &conf->rsync_regex_valid);
+        dht_init_regex (this, this->options, "extra-hash-regex",
+                        &conf->extra_regex, &conf->extra_regex_valid);
+
+        ret = dht_layouts_init (this, conf);
+        if (ret == -1) {
+                goto err;
+        }
+
+        LOCK_INIT (&conf->subvolume_lock);
+        LOCK_INIT (&conf->layout_lock);
+
+        conf->gen = 1;
+
+        this->local_pool = mem_pool_new (dht_local_t, 512);
+        if (!this->local_pool) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto err;
+        }
+
+        GF_OPTION_INIT ("xattr-name", conf->xattr_name, str, err);
+        gf_asprintf (&conf->link_xattr_name, "%s."DHT_LINKFILE_STR,
+                     conf->xattr_name);
+        gf_asprintf (&conf->wild_xattr_name, "%s*", conf->xattr_name);
+        if (!conf->link_xattr_name || !conf->wild_xattr_name) {
+                goto err;
+        }
+
+        this->private = conf;
+
+        return 0;
+
+err:
+        if (conf) {
+                if (conf->file_layouts) {
+                        for (i = 0; i < conf->subvolume_cnt; i++) {
+                                GF_FREE (conf->file_layouts[i]);
+                        }
+                        GF_FREE (conf->file_layouts);
+                }
+
+                GF_FREE (conf->subvolumes);
+
+                GF_FREE (conf->subvolume_status);
+
+                GF_FREE (conf->du_stats);
+
+                GF_FREE (conf->defrag);
+
+                GF_FREE (conf->xattr_name);
+                GF_FREE (conf->link_xattr_name);
+                GF_FREE (conf->wild_xattr_name);
+
+                GF_FREE (conf);
+        }
+
+        return -1;
+}
+
+
+struct volume_options options[] = {
+        { .key  = {"lookup-unhashed"},
+          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
+                    "on", "off"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "on",
+          .description = "This option if set to ON, does a lookup through "
+          "all the sub-volumes, in case a lookup didn't return any result "
+          "from the hash subvolume. If set to OFF, it does not do a lookup "
+          "on the remaining subvolumes."
+        },
+        { .key  = {"min-free-disk"},
+          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
+          .default_value = "10%",
+          .description = "Percentage/Size of disk space, after which the "
+          "process starts balancing out the cluster, and logs will appear "
+          "in log files",
+        },
+	{ .key  = {"min-free-inodes"},
+          .type = GF_OPTION_TYPE_PERCENT,
+          .default_value = "5%",
+          .description = "after system has only N% of inodes, warnings "
+          "starts to appear in log files",
+        },
+        { .key = {"unhashed-sticky-bit"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+        },
+        { .key = {"use-readdirp"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "on",
+          .description = "This option if set to ON, forces the use of "
+          "readdirp, and hence also displays the stats of the files."
+        },
+        { .key = {"assert-no-child-down"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option if set to ON, in the event of "
+          "CHILD_DOWN, will call exit."
+        },
+        { .key  = {"directory-layout-spread"},
+          .type = GF_OPTION_TYPE_INT,
+          .min  = 1,
+          .validate = GF_OPT_VALIDATE_MIN,
+          .description = "Specifies the directory layout spread. Takes number "
+                         "of subvolumes as default value."
+        },
+        { .key  = {"decommissioned-bricks"},
+          .type = GF_OPTION_TYPE_ANY,
+          .description = "This option if set to ON, decommissions "
+          "the brick, so that no new data is allowed to be created "
+          "on that brick."
+        },
+        { .key  = {"rebalance-cmd"},
+          .type = GF_OPTION_TYPE_INT,
+        },
+        { .key = {"node-uuid"},
+          .type = GF_OPTION_TYPE_STR,
+        },
+        { .key = {"rebalance-stats"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option if set to ON displays and logs the "
+          " time taken for migration of each file, during the rebalance "
+          "process. If set to OFF, the rebalance logs will only display the "
+          "time spent in each directory."
+        },
+        { .key = {"readdir-optimize"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "This option if set to ON enables the optimization "
+          "that allows DHT to requests non-first subvolumes to filter out "
+          "directory entries."
+        },
+        { .key = {"rsync-hash-regex"},
+          .type = GF_OPTION_TYPE_STR,
+          /* Setting a default here doesn't work.  See dht_init_regex. */
+          .description = "Regular expression for stripping temporary-file "
+          "suffix and prefix used by rsync, to prevent relocation when the "
+          "file is renamed."
+        },
+        { .key = {"extra-hash-regex"},
+          .type = GF_OPTION_TYPE_STR,
+          /* Setting a default here doesn't work.  See dht_init_regex. */
+          .description = "Regular expression for stripping temporary-file "
+          "suffix and prefix used by an application, to prevent relocation when "
+          "the file is renamed."
+        },
+        { .key = {"rebalance-filter"},
+          .type = GF_OPTION_TYPE_STR,
+        },
+
+        { .key = {"xattr-name"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "trusted.glusterfs.dht",
+          .description = "Base for extended attributes used by this "
+          "translator instance, to avoid conflicts with others above or "
+          "below it."
+        },
+
+        /* NUFA option */
+        { .key  = {"local-volume-name"},
+          .type = GF_OPTION_TYPE_XLATOR
+        },
+
+        /* switch option */
+        { .key  = {"pattern.switch.case"},
+          .type = GF_OPTION_TYPE_ANY
+        },
+
+        { .key  = {NULL} },
+};
diff --git a/xlators/cluster/dht/src/dht.c b/xlators/cluster/dht/src/dht.c
index 2c8f2e9a2..fc0ca2f77 100644
--- a/xlators/cluster/dht/src/dht.c
+++ b/xlators/cluster/dht/src/dht.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -23,501 +14,65 @@
 #include "config.h"
 #endif
 
-/* TODO: add NS locking */
-
 #include "statedump.h"
-#include "dht-common.c"
-
-/* TODO:
-   - use volumename in xattr instead of "dht"
-   - use NS locks
-   - handle all cases in self heal layout reconstruction
-   - complete linkfile selfheal
-*/
-
-
-void
-dht_layout_dump (dht_layout_t  *layout, const char *prefix)
-{
-
-        char    key[GF_DUMP_MAX_BUF_LEN];
-        int     i = 0;
-
-        if (!layout)
-                return;
-
-        gf_proc_dump_build_key(key, prefix, "cnt");
-        gf_proc_dump_write(key, "%d", layout->cnt);
-        gf_proc_dump_build_key(key, prefix, "preset");
-        gf_proc_dump_write(key, "%d", layout->preset);
-        gf_proc_dump_build_key(key, prefix, "gen");
-        gf_proc_dump_write(key, "%d", layout->gen);
-        gf_proc_dump_build_key(key, prefix, "type");
-        gf_proc_dump_write(key, "%d", layout->type);
-
-        for (i = 0; i < layout->cnt; i++) {
-                gf_proc_dump_build_key(key, prefix,"list[%d].err", i);
-                gf_proc_dump_write(key, "%d", layout->list[i].err);
-                gf_proc_dump_build_key(key, prefix,"list[%d].start", i);
-                gf_proc_dump_write(key, "%u", layout->list[i].start);
-                gf_proc_dump_build_key(key, prefix,"list[%d].stop", i);
-                gf_proc_dump_write(key, "%u", layout->list[i].stop);
-                if (layout->list[i].xlator) {
-                        gf_proc_dump_build_key(key, prefix,
-                                               "list[%d].xlator.type", i);
-                        gf_proc_dump_write(key, "%s",
-                                           layout->list[i].xlator->type);
-                        gf_proc_dump_build_key(key, prefix,
-                                               "list[%d].xlator.name", i);
-                        gf_proc_dump_write(key, "%s",
-                                           layout->list[i].xlator->name);
-                }
-        }
-}
-
-
-int32_t
-dht_priv_dump (xlator_t *this)
-{
-	char            key_prefix[GF_DUMP_MAX_BUF_LEN];
-	char            key[GF_DUMP_MAX_BUF_LEN];
-        int             i = 0;
-        dht_conf_t      *conf = NULL;
-        int             ret = 0;
-
-        if (!this)
-                return -1;
-
-        conf = this->private;
-
-        if (!conf)
-                return -1;
-
-        ret = TRY_LOCK(&conf->subvolume_lock);
-
-        if (ret != 0) {
-                gf_log("", GF_LOG_WARNING, "Unable to lock dht subvolume %s",
-                                this->name);
-                return ret;
-        }
-
-        gf_proc_dump_add_section("xlator.cluster.dht.%s.priv", this->name);
-        gf_proc_dump_build_key(key_prefix,"xlator.cluster.dht","%s.priv",
-                               this->name);
-        gf_proc_dump_build_key(key, key_prefix, "subvolume_cnt");
-        gf_proc_dump_write(key,"%d", conf->subvolume_cnt);
-        for (i = 0; i < conf->subvolume_cnt; i++) {
-                gf_proc_dump_build_key(key, key_prefix, "subvolumes[%d]", i);
-                gf_proc_dump_write(key, "%s.%s", conf->subvolumes[i]->type,
-                                   conf->subvolumes[i]->name);
-                if (conf->file_layouts && conf->file_layouts[i]){
-                        gf_proc_dump_build_key(key, key_prefix,
-                                               "file_layouts[%d]",i);
-                        dht_layout_dump(conf->file_layouts[i], key);
-                }
-                if (conf->dir_layouts && conf->dir_layouts[i]) {
-                        gf_proc_dump_build_key(key, key_prefix,
-                                              "dir_layouts[%d]",i);
-                        dht_layout_dump(conf->dir_layouts[i], key);
-                }
-                if (conf->subvolume_status) {
-                        gf_proc_dump_build_key(key, key_prefix,
-                                                "subvolume_status[%d]", i);
-                        gf_proc_dump_write(key, "%d",
-                                               (int)conf->subvolume_status[i]);
-                }
-
-        }
-
-        gf_proc_dump_build_key(key, key_prefix,"default_dir_layout");
-        dht_layout_dump(conf->default_dir_layout, key);
-
-        gf_proc_dump_build_key(key, key_prefix, "search_unhashed");
-        gf_proc_dump_write(key, "%d", conf->search_unhashed);
-        gf_proc_dump_build_key(key, key_prefix, "gen");
-        gf_proc_dump_write(key, "%d", conf->gen);
-        gf_proc_dump_build_key(key, key_prefix, "min_free_disk");
-        gf_proc_dump_write(key, "%lu", conf->min_free_disk);
-        gf_proc_dump_build_key(key, key_prefix, "disk_unit");
-        gf_proc_dump_write(key, "%c", conf->disk_unit);
-        gf_proc_dump_build_key(key, key_prefix, "refresh_interval");
-        gf_proc_dump_write(key, "%d", conf->refresh_interval);
-        gf_proc_dump_build_key(key, key_prefix, "unhashed_sticky_bit");
-        gf_proc_dump_write(key, "%d", conf->unhashed_sticky_bit);
-        if (conf ->du_stats) {
-                gf_proc_dump_build_key(key, key_prefix,
-                                "du_stats.avail_percent");
-                gf_proc_dump_write(key, "%lf", conf->du_stats->avail_percent);
-                gf_proc_dump_build_key(key, key_prefix,
-                                "du_stats.avail_space");
-                gf_proc_dump_write(key, "%lu", conf->du_stats->avail_space);
-                gf_proc_dump_build_key(key, key_prefix,
-                                "du_stats.log");
-                gf_proc_dump_write(key, "%lu", conf->du_stats->log);
-        }
-        gf_proc_dump_build_key(key, key_prefix, "last_stat_fetch");
-        gf_proc_dump_write(key, "%s", ctime(&conf->last_stat_fetch.tv_sec));
-
-        UNLOCK(&conf->subvolume_lock);
-
-        return 0;
-}
-
-int32_t
-dht_inodectx_dump (xlator_t *this, inode_t *inode)
-{
-        int             ret = -1;
-        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
-        dht_layout_t    *layout = NULL;
-	uint64_t        tmp_layout = 0;
-
-        if (!inode)
-                return -1;
-
-	ret = inode_ctx_get (inode, this, &tmp_layout);
-
-        if (ret != 0)
-                return ret;
-
-        layout = (dht_layout_t *)(long)tmp_layout;
-
-        if (!layout)
-                return -1;
-
-        gf_proc_dump_build_key(key_prefix, "xlator.cluster.dht",
-                               "%s.inode.%ld", this->name, inode->ino);
-        dht_layout_dump(layout, key_prefix);
-
-        return 0;
-}
-
-int
-notify (xlator_t *this, int event, void *data, ...)
-{
-	int ret = -1;
-
-	ret = dht_notify (this, event, data);
-
-	return ret;
-}
-
-void
-fini (xlator_t *this)
-{
-        int         i = 0;
-        dht_conf_t *conf = NULL;
-
-	conf = this->private;
-        this->private = NULL;
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                if (conf->default_dir_layout)
-                        GF_FREE (conf->default_dir_layout);
-
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
-
-		if (conf->subvolume_status)
-			GF_FREE (conf->subvolume_status);
-
-                GF_FREE (conf);
-        }
-
-	return;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_dht_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
-
-        return ret;
-}
-int
-validate_options (xlator_t *this, dict_t *options, char **op_errstr)
-{
-        char            *temp_str = NULL;
-        gf_boolean_t     search_unhashed;
-        int              ret = 0;
-        
-
-
-
-
-        if (dict_get_str (options, "lookup-unhashed", &temp_str) == 0) {
-                if (strcasecmp (temp_str, "auto")) {
-                        if (!gf_string2boolean (temp_str, &search_unhashed)) {
-                                gf_log(this->name, GF_LOG_DEBUG, "Validated"
-                                                " lookup-unahashed (%s)",
-                                                                temp_str);
-                        }
-                        else {
-                                gf_log(this->name, GF_LOG_ERROR, "Validation:"
-                                                " lookup-unahashed should be boolean,"
-                                                                " not (%s)",
-                                                                temp_str);
-                                *op_errstr = gf_strdup ("Error, lookup-"
-                                                "unhashed be boolean");
-                                ret = -1;
-                                goto out;
-                        }
-                
-                }
-        }
-
-        
-
-
-
-out:
-                return ret;
-}
-
-int
-reconfigure (xlator_t *this, dict_t *options)
-{
-	dht_conf_t	*conf = NULL;
-	char		*temp_str = NULL;
-	gf_boolean_t     search_unhashed;
-	uint32_t         temp_free_disk = 0;
-	int		 ret = 0;
-
-
-	conf = this->private;
-        if (!conf)
-                return 0;
-
-	if (dict_get_str (options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean*/
-                if (strcasecmp (temp_str, "auto")) {
-                        if (!gf_string2boolean (temp_str, &search_unhashed)) {
-				gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
-				       " lookup-unahashed reconfigured (%s)",
-				       temp_str);
-				conf->search_unhashed = search_unhashed;
-			}
-			else {
-				gf_log(this->name, GF_LOG_ERROR, "Reconfigure:"
-				       " lookup-unahashed should be boolean,"
-				        " not (%s), defaulting to (%d)",
-				       temp_str, conf->search_unhashed);
-				//return -1;
-				ret = -1;
-				goto out;
-			}
-		
-		}
-                else {
-			gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
-			       " lookup-unahashed reconfigured auto ");
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-		}
-	}
-
-	if (dict_get_str (options, "min-free-disk", &temp_str) == 0) {
-		if (gf_string2percent (temp_str, &temp_free_disk) == 0) {
-                        if (temp_free_disk > 100) {
-                                gf_string2bytesize (temp_str, 
-                                                    &conf->min_free_disk);
-                                conf->disk_unit = 'b';
-                        } else {
-                                conf->min_free_disk = (uint64_t)temp_free_disk;
-                        }
-                } else {
-                        gf_string2bytesize (temp_str, &conf->min_free_disk);
-                        conf->disk_unit = 'b';
-                }
-
-		gf_log(this->name, GF_LOG_DEBUG, "Reconfigure:"
-			       " min-free-disk reconfigured to ",
-			       temp_str);
-	}
-
-out:
-	return ret;
-}
-
-int
-init (xlator_t *this)
-{
-        dht_conf_t    *conf = NULL;
-	char          *temp_str = NULL;
-        int            ret = -1;
-        int            i = 0;
-        uint32_t       temp_free_disk = 0;
-
-
-	if (!this->children) {
-		gf_log (this->name, GF_LOG_CRITICAL,
-			"Distribute needs more than one subvolume");
-		return -1;
-	}
-  
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile");
-	}
-
-        conf = GF_CALLOC (1, sizeof (*conf), gf_dht_mt_dht_conf_t);
-        if (!conf) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                goto err;
-        }
-
-        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
-	if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean */
-                if (strcasecmp (temp_str, "auto"))
-                        gf_string2boolean (temp_str, &conf->search_unhashed);
-                else
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-	}
-
-	conf->unhashed_sticky_bit = 0;
-
-	if (dict_get_str (this->options, "unhashed-sticky-bit", 
-                          &temp_str) == 0) {
-	        gf_string2boolean (temp_str, &conf->unhashed_sticky_bit);
-	}
-
-	conf->use_readdirp = 1;
-
-	if (dict_get_str (this->options, "use-readdirp",
-                          &temp_str) == 0) {
-	        gf_string2boolean (temp_str, &conf->use_readdirp);
-	}
-
-        conf->disk_unit = 'p';
-        conf->min_free_disk = 10;
-
-	if (dict_get_str (this->options, "min-free-disk", &temp_str) == 0) {
-		if (gf_string2percent (temp_str, &temp_free_disk) == 0) {
-                        if (temp_free_disk > 100) {
-                                gf_string2bytesize (temp_str, 
-                                                    &conf->min_free_disk);
-                                conf->disk_unit = 'b';
-                        } else {
-                                conf->min_free_disk = (uint64_t)temp_free_disk;
-                        }
-                } else {
-                        gf_string2bytesize (temp_str, &conf->min_free_disk);
-                        conf->disk_unit = 'b';
-                }
-	}
-
-
-        ret = dht_init_subvolumes (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-        ret = dht_layouts_init (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-        conf->du_stats = GF_CALLOC (conf->subvolume_cnt, sizeof (dht_du_t),
-                                    gf_dht_mt_dht_du_t);
-        if (!conf->du_stats) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                goto err;
-        }
-
-	LOCK_INIT (&conf->subvolume_lock);
-	LOCK_INIT (&conf->layout_lock);
-
-	conf->gen = 1;
-
-        this->private = conf;
-
-        return 0;
-
-err:
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                if (conf->default_dir_layout)
-                        GF_FREE (conf->default_dir_layout);
-
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
-
-		if (conf->subvolume_status)
-			GF_FREE (conf->subvolume_status);
-
-                if (conf->du_stats)
-                        GF_FREE (conf->du_stats);
-
-                GF_FREE (conf);
-        }
-
-        return -1;
-}
+#include "dht-common.h"
 
+class_methods_t class_methods = {
+        .init           = dht_init,
+        .fini           = dht_fini,
+        .reconfigure    = dht_reconfigure,
+        .notify         = dht_notify
+};
 
 struct xlator_fops fops = {
-	.lookup      = dht_lookup,
-	.mknod       = dht_mknod,
-	.create      = dht_create,
-
-	.stat        = dht_stat,
-	.fstat       = dht_fstat,
-	.truncate    = dht_truncate,
-	.ftruncate   = dht_ftruncate,
-	.access      = dht_access,
-	.readlink    = dht_readlink,
-	.setxattr    = dht_setxattr,
-	.getxattr    = dht_getxattr,
-	.removexattr = dht_removexattr,
-	.open        = dht_open,
-	.readv       = dht_readv,
-	.writev      = dht_writev,
-	.flush       = dht_flush,
-	.fsync       = dht_fsync,
-	.statfs      = dht_statfs,
-	.lk          = dht_lk,
-	.opendir     = dht_opendir,
-	.readdir     = dht_readdir,
-	.readdirp    = dht_readdirp,
-	.fsyncdir    = dht_fsyncdir,
-	.symlink     = dht_symlink,
-	.unlink      = dht_unlink,
-	.link        = dht_link,
-	.mkdir       = dht_mkdir,
-	.rmdir       = dht_rmdir,
-	.rename      = dht_rename,
-	.inodelk     = dht_inodelk,
-	.finodelk    = dht_finodelk,
-	.entrylk     = dht_entrylk,
-	.fentrylk    = dht_fentrylk,
-	.xattrop     = dht_xattrop,
-	.fxattrop    = dht_fxattrop,
-        .setattr     = dht_setattr, 
+        .lookup      = dht_lookup,
+        .mknod       = dht_mknod,
+        .create      = dht_create,
+
+        .open        = dht_open,
+        .statfs      = dht_statfs,
+        .opendir     = dht_opendir,
+        .readdir     = dht_readdir,
+        .readdirp    = dht_readdirp,
+        .fsyncdir    = dht_fsyncdir,
+        .symlink     = dht_symlink,
+        .unlink      = dht_unlink,
+        .link        = dht_link,
+        .mkdir       = dht_mkdir,
+        .rmdir       = dht_rmdir,
+        .rename      = dht_rename,
+        .entrylk     = dht_entrylk,
+        .fentrylk    = dht_fentrylk,
+
+        /* Inode read operations */
+        .stat        = dht_stat,
+        .fstat       = dht_fstat,
+        .access      = dht_access,
+        .readlink    = dht_readlink,
+        .getxattr    = dht_getxattr,
+        .fgetxattr    = dht_fgetxattr,
+        .readv       = dht_readv,
+        .flush       = dht_flush,
+        .fsync       = dht_fsync,
+        .inodelk     = dht_inodelk,
+        .finodelk    = dht_finodelk,
+        .lk          = dht_lk,
+
+        /* Inode write operations */
+        .fremovexattr = dht_fremovexattr,
+        .removexattr = dht_removexattr,
+        .setxattr    = dht_setxattr,
+        .fsetxattr   = dht_fsetxattr,
+        .truncate    = dht_truncate,
+        .ftruncate   = dht_ftruncate,
+        .writev      = dht_writev,
+        .xattrop     = dht_xattrop,
+        .fxattrop    = dht_fxattrop,
+        .setattr     = dht_setattr,
         .fsetattr    = dht_fsetattr,
+	.fallocate   = dht_fallocate,
+	.discard     = dht_discard,
+        .zerofill    = dht_zerofill,
 };
 
 struct xlator_dumpops dumpops = {
@@ -527,26 +82,8 @@ struct xlator_dumpops dumpops = {
 
 
 struct xlator_cbks cbks = {
-//	.release    = dht_release,
+//      .release    = dht_release,
 //      .releasedir = dht_releasedir,
-	.forget     = dht_forget
-};
-
-
-struct volume_options options[] = {
-        { .key  = {"lookup-unhashed"},
-          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
-                    "on", "off"},
-	  .type = GF_OPTION_TYPE_STR
-	},
-        { .key  = {"min-free-disk"},
-          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
-        },
-        { .key = {"unhashed-sticky-bit"},
-          .type = GF_OPTION_TYPE_BOOL
-        },
-        { .key = {"use-readdirp"},
-          .type = GF_OPTION_TYPE_BOOL
-        },
-	{ .key  = {NULL} },
+        .forget     = dht_forget
 };
+;
diff --git a/xlators/cluster/dht/src/nufa.c b/xlators/cluster/dht/src/nufa.c
index c13fdaebe..e934acdf0 100644
--- a/xlators/cluster/dht/src/nufa.c
+++ b/xlators/cluster/dht/src/nufa.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -23,13 +14,15 @@
 #include "config.h"
 #endif
 
-#include "dht-common.c"
+#include "dht-common.h"
 
 /* TODO: all 'TODO's in dht.c holds good */
 
+extern struct volume_options options[];
+
 int
 nufa_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int op_ret, int op_errno,
+                       int op_ret, int op_errno,
                        inode_t *inode, struct iatt *stbuf, dict_t *xattr,
                        struct iatt *postparent)
 {
@@ -41,67 +34,61 @@ nufa_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         loc_t        *loc         = NULL;
         int           i           = 0;
         call_frame_t *prev        = NULL;
-	int           call_cnt    = 0;
+        int           call_cnt    = 0;
         int           ret         = 0;
 
-
         conf  = this->private;
 
         prev  = cookie;
         local = frame->local;
         loc   = &local->loc;
 
-	if (ENTRY_MISSING (op_ret, op_errno)) {
-		if (conf->search_unhashed) {
-			local->op_errno = ENOENT;
-			dht_lookup_everywhere (frame, this, loc);
-			return 0;
-		}
-	}
+        if (ENTRY_MISSING (op_ret, op_errno)) {
+                if (conf->search_unhashed) {
+                        local->op_errno = ENOENT;
+                        dht_lookup_everywhere (frame, this, loc);
+                        return 0;
+                }
+        }
 
         if (op_ret == -1)
                 goto out;
 
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
         is_dir      = check_is_dir (inode, stbuf, xattr);
 
         if (!is_dir && !is_linkfile) {
                 /* non-directory and not a linkfile */
-
-		dht_itransform (this, prev->this, stbuf->ia_ino,
-				&stbuf->ia_ino);
-
-		ret = dht_layout_preset (this, prev->this, inode);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"could not set pre-set layout for subvol %s",
-				prev->this->name);
-			op_ret   = -1;
-			op_errno = EINVAL;
-			goto err;
-		}
+                ret = dht_layout_preset (this, prev->this, inode);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "could not set pre-set layout for subvol %s",
+                                prev->this->name);
+                        op_ret   = -1;
+                        op_errno = EINVAL;
+                        goto err;
+                }
 
                 goto out;
         }
 
         if (is_dir) {
                 call_cnt        = conf->subvolume_cnt;
-		local->call_cnt = call_cnt;
+                local->call_cnt = call_cnt;
 
                 local->inode = inode_ref (inode);
                 local->xattr = dict_ref (xattr);
 
-		local->op_ret = 0;
-		local->op_errno = 0;
+                local->op_ret = 0;
+                local->op_errno = 0;
 
-		local->layout = dht_layout_new (this, conf->subvolume_cnt);
-		if (!local->layout) {
-			op_ret   = -1;
-			op_errno = ENOMEM;
-			gf_log (this->name, GF_LOG_DEBUG,
-				"memory allocation failed :(");
-			goto err;
-		}
+                local->layout = dht_layout_new (this, conf->subvolume_cnt);
+                if (!local->layout) {
+                        op_ret   = -1;
+                        op_errno = ENOMEM;
+                        goto err;
+                }
 
                 for (i = 0; i < call_cnt; i++) {
                         STACK_WIND (frame, dht_lookup_dir_cbk,
@@ -118,53 +105,52 @@ nufa_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "linkfile not having link subvolume. path=%s",
                                 loc->path);
-			dht_lookup_everywhere (frame, this, loc);
-			return 0;
+                        dht_lookup_everywhere (frame, this, loc);
+                        return 0;
                 }
 
-		STACK_WIND (frame, dht_lookup_linkfile_cbk,
-			    subvol, subvol->fops->lookup,
-			    &local->loc, local->xattr_req);
+                STACK_WIND (frame, dht_lookup_linkfile_cbk,
+                            subvol, subvol->fops->lookup,
+                            &local->loc, local->xattr_req);
         }
 
         return 0;
 
 out:
-	if (!local->hashed_subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no subvolume in layout for path=%s",
-			local->loc.path);
+        if (!local->hashed_subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        local->loc.path);
                 local->op_errno = ENOENT;
                 dht_lookup_everywhere (frame, this, loc);
                 return 0;
-	}
+        }
 
-	STACK_WIND (frame, dht_lookup_cbk,
-		    local->hashed_subvol, local->hashed_subvol->fops->lookup,
-		    &local->loc, local->xattr_req);
+        STACK_WIND (frame, dht_lookup_cbk,
+                    local->hashed_subvol, local->hashed_subvol->fops->lookup,
+                    &local->loc, local->xattr_req);
 
-	return 0;
+        return 0;
 
- err:
+err:
         DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno,
-                          inode, stbuf, xattr, NULL);
+                          inode, stbuf, xattr, postparent);
         return 0;
 }
 
 int
 nufa_lookup (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, dict_t *xattr_req)
+             loc_t *loc, dict_t *xattr_req)
 {
         xlator_t     *hashed_subvol = NULL;
-        xlator_t     *cached_subvol = NULL;
         xlator_t     *subvol = NULL;
         dht_local_t  *local  = NULL;
-	dht_conf_t   *conf = NULL;
+        dht_conf_t   *conf = NULL;
         int           ret    = -1;
         int           op_errno = -1;
-	dht_layout_t *layout = NULL;
-	int           i = 0;
-	int           call_cnt = 0;
+        dht_layout_t *layout = NULL;
+        int           i = 0;
+        int           call_cnt = 0;
 
 
         VALIDATE_OR_GOTO (frame, err);
@@ -173,40 +159,26 @@ nufa_lookup (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (loc->inode, err);
         VALIDATE_OR_GOTO (loc->path, err);
 
-	conf = this->private;
+        conf = this->private;
 
-        local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
-
-        ret = loc_dup (loc, &local->loc);
-        if (ret == -1) {
-                op_errno = errno;
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "copying location failed for path=%s",
-                        loc->path);
+        local = dht_local_init (frame, loc, NULL, GF_FOP_LOOKUP);
+        if (!local) {
+                op_errno = ENOMEM;
                 goto err;
         }
 
-	if (xattr_req) {
-		local->xattr_req = dict_ref (xattr_req);
-	} else {
-		local->xattr_req = dict_new ();
-	}
+        if (xattr_req) {
+                local->xattr_req = dict_ref (xattr_req);
+        } else {
+                local->xattr_req = dict_new ();
+        }
 
-	hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
-	cached_subvol = dht_subvol_get_cached (this, local->loc.inode);
+        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
 
-	local->cached_subvol = cached_subvol;
-	local->hashed_subvol = hashed_subvol;
+        local->hashed_subvol = hashed_subvol;
 
         if (is_revalidate (loc)) {
-		local->layout = layout = dht_layout_get (this, loc->inode);
-
+                layout = local->layout;
                 if (!layout) {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "revalidate without cache. path=%s",
@@ -215,25 +187,24 @@ nufa_lookup (call_frame_t *frame, xlator_t *this,
                         goto err;
                 }
 
-		if (layout->gen && (layout->gen < conf->gen)) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"incomplete layout failure for path=%s",
-				loc->path);
+                if (layout->gen && (layout->gen < conf->gen)) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "incomplete layout failure for path=%s",
+                                loc->path);
                         dht_layout_unref (this, local->layout);
-			goto do_fresh_lookup;
-		}
+                        goto do_fresh_lookup;
+                }
 
-		local->inode    = inode_ref (loc->inode);
-		local->ia_ino   = loc->inode->ino;
+                local->inode = inode_ref (loc->inode);
 
-		local->call_cnt = layout->cnt;
-		call_cnt = local->call_cnt;
+                local->call_cnt = layout->cnt;
+                call_cnt = local->call_cnt;
 
-		/* NOTE: we don't require 'trusted.glusterfs.dht.linkto' attribute,
-		 *       revalidates directly go to the cached-subvolume.
-		 */
-		ret = dict_set_uint32 (local->xattr_req,
-				       "trusted.glusterfs.dht", 4 * 4);
+                /* NOTE: we don't require 'trusted.glusterfs.dht.linkto' attribute,
+                 *       revalidates directly go to the cached-subvolume.
+                 */
+                ret = dict_set_uint32 (local->xattr_req,
+                                       conf->xattr_name, 4 * 4);
                 if (ret < 0) {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "Failed to set dict value.");
@@ -241,20 +212,20 @@ nufa_lookup (call_frame_t *frame, xlator_t *this,
                         goto err;
                 }
 
-		for (i = 0; i < layout->cnt; i++) {
-			subvol = layout->list[i].xlator;
+                for (i = 0; i < layout->cnt; i++) {
+                        subvol = layout->list[i].xlator;
 
-			STACK_WIND (frame, dht_revalidate_cbk,
-				    subvol, subvol->fops->lookup,
-				    loc, local->xattr_req);
+                        STACK_WIND (frame, dht_revalidate_cbk,
+                                    subvol, subvol->fops->lookup,
+                                    loc, local->xattr_req);
 
-			if (!--call_cnt)
-				break;
-		}
-	} else {
+                        if (!--call_cnt)
+                                break;
+                }
+        } else {
         do_fresh_lookup:
-		ret = dict_set_uint32 (local->xattr_req,
-				       "trusted.glusterfs.dht", 4 * 4);
+                ret = dict_set_uint32 (local->xattr_req,
+                                       conf->xattr_name, 4 * 4);
                 if (ret < 0) {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "Failed to set dict value.");
@@ -262,8 +233,8 @@ nufa_lookup (call_frame_t *frame, xlator_t *this,
                         goto err;
                 }
 
-		ret = dict_set_uint32 (local->xattr_req,
-				       "trusted.glusterfs.dht.linkto", 256);
+                ret = dict_set_uint32 (local->xattr_req,
+                                       conf->link_xattr_name, 256);
                 if (ret < 0) {
                         gf_log (this->name, GF_LOG_ERROR,
                                 "Failed to set dict value.");
@@ -271,111 +242,100 @@ nufa_lookup (call_frame_t *frame, xlator_t *this,
                         goto err;
                 }
 
-		/* Send it to only local volume */
-		STACK_WIND (frame, nufa_local_lookup_cbk,
-			    (xlator_t *)conf->private,
-			    ((xlator_t *)conf->private)->fops->lookup,
-			    loc, local->xattr_req);
-	}
+                /* Send it to only local volume */
+                STACK_WIND (frame, nufa_local_lookup_cbk,
+                            (xlator_t *)conf->private,
+                            ((xlator_t *)conf->private)->fops->lookup,
+                            loc, local->xattr_req);
+        }
 
         return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
-	return 0;
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL,
+                          NULL);
+        return 0;
 }
 
 int
 nufa_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-				 xlator_t *this, int op_ret, int op_errno,
+                                 xlator_t *this, int op_ret, int op_errno,
                                  inode_t *inode, struct iatt *stbuf,
                                  struct iatt *preparent,
-                                 struct iatt *postparent)
+                                 struct iatt *postparent, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
+        dht_local_t  *local = NULL;
 
-	local = frame->local;
+        local = frame->local;
 
-	if (op_ret == -1)
-		goto err;
+        if (op_ret == -1)
+                goto err;
 
-	STACK_WIND (frame, dht_create_cbk,
-		    local->cached_subvol, local->cached_subvol->fops->create,
-		    &local->loc, local->flags, local->mode, local->fd,
-                    local->params);
+        STACK_WIND (frame, dht_create_cbk,
+                    local->cached_subvol, local->cached_subvol->fops->create,
+                    &local->loc, local->flags, local->mode, local->umask,
+                    local->fd, local->params);
 
-	return 0;
+        return 0;
 
- err:
-	DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
-	return 0;
+err:
+        DHT_STACK_UNWIND (create, frame, -1, op_errno,
+                          NULL, NULL, NULL, NULL, NULL, NULL);
+        return 0;
 }
 
 int
 nufa_create (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, int32_t flags, mode_t mode,
-             fd_t *fd, dict_t *params)
+             loc_t *loc, int32_t flags, mode_t mode,
+             mode_t umask, fd_t *fd, dict_t *params)
 {
-	dht_local_t *local = NULL;
-	dht_conf_t  *conf  = NULL;
-	xlator_t    *subvol = NULL;
+        dht_local_t *local = NULL;
+        dht_conf_t  *conf  = NULL;
+        xlator_t    *subvol = NULL;
         xlator_t    *avail_subvol = NULL;
-	int          op_errno = -1;
-	int          ret = -1;
+        int          op_errno = -1;
 
-	VALIDATE_OR_GOTO (frame, err);
-	VALIDATE_OR_GOTO (this, err);
-	VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
 
-	conf  = this->private;
+        conf  = this->private;
 
         dht_get_du_info (frame, this, loc);
 
-        local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
-
-	subvol = dht_subvol_get_hashed (this, loc);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no subvolume in layout for path=%s",
-			loc->path);
-		op_errno = ENOENT;
-		goto err;
-	}
+        local = dht_local_init (frame, loc, fd, GF_FOP_CREATE);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = dht_subvol_get_hashed (this, loc);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        loc->path);
+                op_errno = ENOENT;
+                goto err;
+        }
 
         avail_subvol = conf->private;
         if (dht_is_subvol_filled (this, (xlator_t *)conf->private)) {
                 avail_subvol =
                         dht_free_disk_available_subvol (this,
-                                                        (xlator_t *)conf->private);
+                                                        (xlator_t *)conf->private,
+                                                        local);
         }
 
         if (subvol != avail_subvol) {
                 /* create a link file instead of actual file */
-                ret = loc_copy (&local->loc, loc);
-                if (ret == -1) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Out of memory");
-                                op_errno = ENOMEM;
-                                goto err;
-                }
-
-                local->fd = fd_ref (fd);
                 local->params = dict_ref (params);
                 local->mode = mode;
                 local->flags = flags;
-
+                local->umask = umask;
                 local->cached_subvol = avail_subvol;
-                dht_linkfile_create (frame,
-                                     nufa_create_linkfile_create_cbk,
-                                     avail_subvol, subvol, loc);
+                dht_linkfile_create (frame, nufa_create_linkfile_create_cbk,
+                                     this, avail_subvol, subvol, loc);
                 return 0;
         }
 
@@ -384,379 +344,333 @@ nufa_create (call_frame_t *frame, xlator_t *this,
 
         STACK_WIND (frame, dht_create_cbk,
                     subvol, subvol->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, params);
 
         return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (create, frame, -1, op_errno,
+                          NULL, NULL, NULL, NULL, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 int
 nufa_mknod_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int op_ret, int op_errno, inode_t *inode,
                          struct iatt *stbuf, struct iatt *preparent,
-                         struct iatt *postparent)
+                         struct iatt *postparent, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
+        dht_local_t  *local = NULL;
 
-	local = frame->local;
-
-	if (op_ret >= 0) {
-		STACK_WIND (frame, dht_newfile_cbk,
-			    local->cached_subvol,
-			    local->cached_subvol->fops->mknod,
-			    &local->loc, local->mode, local->rdev,
-                            local->params);
+        local = frame->local;
+        if (!local || !local->cached_subvol) {
+                op_errno = EINVAL;
+                op_ret = -1;
+                goto err;
+        }
 
-		return 0;
-	}
+        if (op_ret >= 0) {
+                STACK_WIND_COOKIE (frame, dht_newfile_cbk,
+                            (void *)local->cached_subvol, local->cached_subvol,
+                            local->cached_subvol->fops->mknod,
+                            &local->loc, local->mode, local->rdev,
+                            local->umask, local->params);
 
+                return 0;
+        }
+err:
         WIPE (postparent);
         WIPE (preparent);
 
-	DHT_STACK_UNWIND (link, frame, op_ret, op_errno,
-                          inode, stbuf, preparent, postparent);
-	return 0;
+        DHT_STACK_UNWIND (link, frame, op_ret, op_errno,
+                          inode, stbuf, preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
 nufa_mknod (call_frame_t *frame, xlator_t *this,
-	    loc_t *loc, mode_t mode, dev_t rdev, dict_t *params)
+            loc_t *loc, mode_t mode, dev_t rdev, mode_t umask, dict_t *params)
 {
-	dht_local_t *local = NULL;
-	dht_conf_t  *conf  = NULL;
-	xlator_t    *subvol = NULL;
+        dht_local_t *local = NULL;
+        dht_conf_t  *conf  = NULL;
+        xlator_t    *subvol = NULL;
         xlator_t    *avail_subvol = NULL;
-	int          op_errno = -1;
-	int          ret = -1;
+        int          op_errno = -1;
 
-	VALIDATE_OR_GOTO (frame, err);
-	VALIDATE_OR_GOTO (this, err);
-	VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
 
-	conf  = this->private;
+        conf  = this->private;
 
         dht_get_du_info (frame, this, loc);
 
-        local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
-
-	subvol = dht_subvol_get_hashed (this, loc);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no subvolume in layout for path=%s",
-			loc->path);
-		op_errno = ENOENT;
-		goto err;
-	}
+        local = dht_local_init (frame, loc, NULL, GF_FOP_MKNOD);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = dht_subvol_get_hashed (this, loc);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        loc->path);
+                op_errno = ENOENT;
+                goto err;
+        }
 
         /* Consider the disksize in consideration */
         avail_subvol = conf->private;
         if (dht_is_subvol_filled (this, (xlator_t *)conf->private)) {
                 avail_subvol =
                         dht_free_disk_available_subvol (this,
-                                                        (xlator_t *)conf->private);
+                                                        (xlator_t *)conf->private,
+                                                        local);
         }
 
-	if (avail_subvol != subvol) {
-		/* Create linkfile first */
-		ret = loc_copy (&local->loc, loc);
-		if (ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"Out of memory");
-			op_errno = ENOMEM;
-			goto err;
-		}
+        if (avail_subvol != subvol) {
+                /* Create linkfile first */
 
                 local->params = dict_ref (params);
-		local->mode = mode;
-		local->rdev = rdev;
-		local->cached_subvol = avail_subvol;
+                local->mode = mode;
+                local->umask = umask;
+                local->rdev = rdev;
+                local->cached_subvol = avail_subvol;
 
-		dht_linkfile_create (frame, nufa_mknod_linkfile_cbk,
+                dht_linkfile_create (frame, nufa_mknod_linkfile_cbk, this,
                                      avail_subvol, subvol, loc);
-		return 0;
-	}
+                return 0;
+        }
 
-	gf_log (this->name, GF_LOG_TRACE,
-		"creating %s on %s", loc->path, subvol->name);
+        gf_log (this->name, GF_LOG_TRACE,
+                "creating %s on %s", loc->path, subvol->name);
 
-	STACK_WIND (frame, dht_newfile_cbk,
-		    subvol, subvol->fops->mknod,
-		    loc, mode, rdev, params);
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
+                           subvol->fops->mknod, loc, mode, rdev, umask,
+                           params);
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
+                          NULL, NULL, NULL, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 
-int
-notify (xlator_t *this, int event, void *data, ...)
+gf_boolean_t
+same_first_part (char *str1, char term1, char *str2, char term2)
 {
-	int ret = -1;
-
-	ret = dht_notify (this, event, data);
-
-	return ret;
+        gf_boolean_t    ended1;
+        gf_boolean_t    ended2;
+
+        for (;;) {
+                ended1 = ((*str1 == '\0') || (*str1 == term1));
+                ended2 = ((*str2 == '\0') || (*str2 == term2));
+                if (ended1 && ended2) {
+                        return _gf_true;
+                }
+                if (ended1 || ended2 || (*str1 != *str2)) {
+                        return _gf_false;
+                }
+                ++str1;
+                ++str2;
+        }
 }
 
-void
-fini (xlator_t *this)
+typedef struct nufa_args {
+        xlator_t    *this;
+        char        *volname;
+        gf_boolean_t addr_match;
+} nufa_args_t;
+
+static void
+nufa_find_local_brick (xlator_t *xl, void *data)
 {
-        int         i = 0;
-        dht_conf_t *conf = NULL;
+        nufa_args_t     *args = data;
+        xlator_t        *this = args->this;
+        char            *local_volname = args->volname;
+        gf_boolean_t    addr_match = args->addr_match;
+        char            *brick_host = NULL;
+        dht_conf_t      *conf = this->private;
+        int             ret = -1;
+
+        /*This means a local subvol was already found. We pick the first brick
+         * that is local*/
+        if (conf->private)
+                return;
+
+        if (strcmp (xl->name, local_volname) == 0) {
+                conf->private = xl;
+                gf_log (this->name, GF_LOG_INFO, "Using specified subvol %s",
+                        local_volname);
+                return;
+        }
 
-	conf = this->private;
+        if (!addr_match)
+                return;
+
+        ret = dict_get_str (xl->options, "remote-host", &brick_host);
+        if ((ret == 0) &&
+            (gf_is_same_address (local_volname, brick_host) ||
+             gf_is_local_addr (brick_host))) {
+                conf->private = xl;
+                gf_log (this->name, GF_LOG_INFO, "Using the first local "
+                        "subvol %s", xl->name);
+                return;
+        }
 
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
+}
 
-                if (conf->default_dir_layout)
-                        GF_FREE (conf->default_dir_layout);
+static void
+nufa_to_dht (xlator_t *this)
+{
+        GF_ASSERT (this);
+        GF_ASSERT (this->fops);
 
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
+        this->fops->lookup = dht_lookup;
+        this->fops->create = dht_create;
+        this->fops->mknod  = dht_mknod;
+}
 
-		if (conf->subvolume_status)
-			GF_FREE (conf->subvolume_status);
+int
+nufa_find_local_subvol (xlator_t *this,
+                        void (*fn) (xlator_t *each, void* data), void *data)
+{
+        int             ret = -1;
+        dht_conf_t      *conf = this->private;
+        xlator_list_t   *trav = NULL;
+        xlator_t        *parent = NULL;
+        xlator_t        *candidate = NULL;
+
+        xlator_foreach_depth_first (this, fn, data);
+        if (!conf->private) {
+                gf_log (this->name, GF_LOG_ERROR, "Couldn't find a local "
+                        "brick");
+                return -1;
+        }
+
+        candidate = conf->private;
+        trav = candidate->parents;
+        while (trav) {
+
+                parent = trav->xlator;
+                if (strcmp (parent->type, "cluster/nufa") == 0) {
+                        gf_log (this->name, GF_LOG_INFO, "Found local subvol, "
+                                "%s", candidate->name);
+                        ret = 0;
+                        conf->private = candidate;
+                        break;
+                }
 
-                GF_FREE (conf);
+                candidate = parent;
+                trav = parent->parents;
         }
 
-	return;
+        return ret;
 }
 
 int
-init (xlator_t *this)
+nufa_init (xlator_t *this)
 {
-        dht_conf_t    *conf = NULL;
-	xlator_list_t *trav = NULL;
-	data_t        *data = NULL;
-	char          *local_volname = NULL;
-	char          *temp_str = NULL;
+        data_t        *data = NULL;
+        char          *local_volname = NULL;
         int            ret = -1;
-        int            i = 0;
-	char           my_hostname[256];
-	uint32_t       temp_free_disk = 0;
-
-	if (!this->children) {
-		gf_log (this->name, GF_LOG_CRITICAL,
-			"NUFA needs more than one subvolume");
-		return -1;
-	}
-
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile");
-	}
-
-        conf = GF_CALLOC (1, sizeof (*conf),
-                          gf_dht_mt_dht_conf_t);
-        if (!conf) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                goto err;
+        char           my_hostname[256];
+        gf_boolean_t   addr_match = _gf_false;
+        nufa_args_t    args = {0, };
+
+        ret = dht_init(this);
+        if (ret) {
+                return ret;
         }
 
-        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
-	if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean */
-                if (strcasecmp (temp_str, "auto"))
-                        gf_string2boolean (temp_str, &conf->search_unhashed);
-                else
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-	}
+        if ((data = dict_get (this->options, "local-volume-name"))) {
+                local_volname = data->data;
 
-        ret = dht_init_subvolumes (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
+        } else {
+                addr_match = _gf_true;
+                local_volname = "localhost";
+                ret = gethostname (my_hostname, 256);
+                if (ret == 0)
+                        local_volname = my_hostname;
 
-        ret = dht_layouts_init (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
+                else
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "could not find hostname (%s)",
+                                strerror (errno));
 
-	LOCK_INIT (&conf->subvolume_lock);
-	LOCK_INIT (&conf->layout_lock);
-
-	conf->gen = 1;
-
-	local_volname = "localhost";
-	ret = gethostname (my_hostname, 256);
-	if (ret < 0) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"could not find hostname (%s)",
-			strerror (errno));
-	}
-
-	if (ret == 0)
-		local_volname = my_hostname;
-
-	data = dict_get (this->options, "local-volume-name");
-	if (data) {
-		local_volname = data->data;
-	}
-
-	trav = this->children;
-	while (trav) {
-		if (strcmp (trav->xlator->name, local_volname) == 0)
-			break;
-		trav = trav->next;
-	}
-
-	if (!trav) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Could not find subvolume named '%s'. "
-			"Please define volume with the name as the hostname "
-			"or override it with 'option local-volume-name'",
-			local_volname);
-		goto err;
-	}
-	/* The volume specified exists */
-	conf->private = trav->xlator;
-
-        conf->min_free_disk = 10;
-        conf->disk_unit = 'p';
-
-        if (dict_get_str (this->options, "min-free-disk",
-                          &temp_str) == 0) {
-                if (gf_string2percent (temp_str,
-                                       &temp_free_disk) == 0) {
-                        if (temp_free_disk > 100) {
-                                gf_string2bytesize (temp_str,
-                                                        &conf->min_free_disk);
-                                conf->disk_unit = 'b';
-                        } else {
-                                conf->min_free_disk = (uint64_t)temp_free_disk;
-                                conf->disk_unit = 'p';
-                        }
-                } else {
-                        gf_string2bytesize (temp_str,
-                                                &conf->min_free_disk);
-                        conf->disk_unit = 'b';
-                }
         }
 
-        conf->du_stats = GF_CALLOC (conf->subvolume_cnt, sizeof (dht_du_t),
-                                    gf_dht_mt_dht_du_t);
-        if (!conf->du_stats) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                goto err;
+        args.this = this;
+        args.volname = local_volname;
+        args.addr_match = addr_match;
+        ret = nufa_find_local_subvol (this, nufa_find_local_brick, &args);
+        if (ret) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Unable to find local subvolume, switching "
+                        "to dht mode");
+                nufa_to_dht (this);
         }
-
-        this->private = conf;
-
         return 0;
+}
 
-err:
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                if (conf->default_dir_layout)
-                        GF_FREE (conf->default_dir_layout);
-
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
-
-		if (conf->subvolume_status)
-			GF_FREE (conf->subvolume_status);
-
-                if (conf->du_stats)
-                        GF_FREE (conf->du_stats);
-
-                GF_FREE (conf);
-        }
 
-        return -1;
-}
+class_methods_t class_methods = {
+        .init           = nufa_init,
+        .fini           = dht_fini,
+        .reconfigure    = dht_reconfigure,
+        .notify         = dht_notify
+};
 
 
 struct xlator_fops fops = {
-	.lookup      = nufa_lookup,
-	.create      = nufa_create,
-	.mknod       = nufa_mknod,
-
-	.stat        = dht_stat,
-	.fstat       = dht_fstat,
-	.truncate    = dht_truncate,
-	.ftruncate   = dht_ftruncate,
-	.access      = dht_access,
-	.readlink    = dht_readlink,
-	.setxattr    = dht_setxattr,
-	.getxattr    = dht_getxattr,
-	.removexattr = dht_removexattr,
-	.open        = dht_open,
-	.readv       = dht_readv,
-	.writev      = dht_writev,
-	.flush       = dht_flush,
-	.fsync       = dht_fsync,
-	.statfs      = dht_statfs,
-	.lk          = dht_lk,
-	.opendir     = dht_opendir,
-	.readdir     = dht_readdir,
-	.readdirp    = dht_readdirp,
-	.fsyncdir    = dht_fsyncdir,
-	.symlink     = dht_symlink,
-	.unlink      = dht_unlink,
-	.link        = dht_link,
-	.mkdir       = dht_mkdir,
-	.rmdir       = dht_rmdir,
-	.rename      = dht_rename,
-	.inodelk     = dht_inodelk,
-	.finodelk    = dht_finodelk,
-	.entrylk     = dht_entrylk,
-	.fentrylk    = dht_fentrylk,
-	.xattrop     = dht_xattrop,
-	.fxattrop    = dht_fxattrop,
+        .lookup      = nufa_lookup,
+        .create      = nufa_create,
+        .mknod       = nufa_mknod,
+
+        .stat        = dht_stat,
+        .fstat       = dht_fstat,
+        .truncate    = dht_truncate,
+        .ftruncate   = dht_ftruncate,
+        .access      = dht_access,
+        .readlink    = dht_readlink,
+        .setxattr    = dht_setxattr,
+        .getxattr    = dht_getxattr,
+        .removexattr = dht_removexattr,
+        .open        = dht_open,
+        .readv       = dht_readv,
+        .writev      = dht_writev,
+        .flush       = dht_flush,
+        .fsync       = dht_fsync,
+        .statfs      = dht_statfs,
+        .lk          = dht_lk,
+        .opendir     = dht_opendir,
+        .readdir     = dht_readdir,
+        .readdirp    = dht_readdirp,
+        .fsyncdir    = dht_fsyncdir,
+        .symlink     = dht_symlink,
+        .unlink      = dht_unlink,
+        .link        = dht_link,
+        .mkdir       = dht_mkdir,
+        .rmdir       = dht_rmdir,
+        .rename      = dht_rename,
+        .inodelk     = dht_inodelk,
+        .finodelk    = dht_finodelk,
+        .entrylk     = dht_entrylk,
+        .fentrylk    = dht_fentrylk,
+        .xattrop     = dht_xattrop,
+        .fxattrop    = dht_fxattrop,
         .setattr     = dht_setattr,
 };
 
 
 struct xlator_cbks cbks = {
-	.forget     = dht_forget
-};
-
-
-struct volume_options options[] = {
-        { .key  = {"lookup-unhashed"},
-          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
-                    "on", "off"},
-	  .type = GF_OPTION_TYPE_STR
-	},
-	{ .key  = {"local-volume-name"},
-	  .type = GF_OPTION_TYPE_XLATOR
-	},
-        { .key  = {"min-free-disk"},
-          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
-        },
-	{ .key  = {NULL} },
+        .forget     = dht_forget
 };
diff --git a/xlators/cluster/dht/src/switch.c b/xlators/cluster/dht/src/switch.c
index dc56787eb..2717ce975 100644
--- a/xlators/cluster/dht/src/switch.c
+++ b/xlators/cluster/dht/src/switch.c
@@ -1,20 +1,11 @@
 /*
-   Copyright (c) 2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -23,7 +14,7 @@
 #include "config.h"
 #endif
 
-#include "dht-common.c"
+#include "dht-common.h"
 #include "dht-mem-types.h"
 
 #include <sys/time.h>
@@ -31,21 +22,23 @@
 #include <fnmatch.h>
 #include <string.h>
 
+extern struct volume_options options[];
+
 struct switch_sched_array {
-	xlator_t *xl;
-	int32_t   eligible;
-	int32_t   considered;
+        xlator_t *xl;
+        int32_t   eligible;
+        int32_t   considered;
 };
 
 /* Select one of this struct based on the path's pattern match */
 struct switch_struct {
-	struct switch_struct      *next;
+        struct switch_struct      *next;
         struct switch_sched_array *array;
-	int32_t                    node_index; /* Index of the node in
+        int32_t                    node_index; /* Index of the node in
                                                   this pattern. */
-	int32_t                    num_child;  /* Total num of child nodes
+        int32_t                    num_child;  /* Total num of child nodes
                                                   with this pattern. */
-	char                       path_pattern[256];
+        char                       path_pattern[256];
 };
 
 /* TODO: all 'TODO's in dht.c holds good */
@@ -74,31 +67,39 @@ get_switch_matching_subvol (const char *path, dht_conf_t *conf,
                             xlator_t *hashed_subvol)
 {
         struct switch_struct *cond      = NULL;
-	struct switch_struct *trav      = NULL;
-	char                 *pathname  = NULL;
-	int                   idx     = 0;
+        struct switch_struct *trav      = NULL;
+        char                 *pathname  = NULL;
+        int                   idx       = 0;
+        xlator_t             *subvol    = NULL;
 
         cond = conf->private;
+        subvol = hashed_subvol;
         if (!cond)
-                return hashed_subvol;
+                goto out;
 
-        trav = cond;
         pathname = gf_strdup (path);
-	while (trav) {
-		if (fnmatch (trav->path_pattern,
-			     pathname, FNM_NOESCAPE) == 0) {
+        if (!pathname)
+                goto out;
+
+        trav = cond;
+        while (trav) {
+                if (fnmatch (trav->path_pattern,
+                             pathname, FNM_NOESCAPE) == 0) {
                         for (idx = 0; idx < trav->num_child; idx++) {
                                 if (trav->array[idx].xl == hashed_subvol)
-                                        return hashed_subvol;
+                                        goto out;
                         }
                         idx = trav->node_index++;
                         trav->node_index %= trav->num_child;
-                        return trav->array[idx].xl;
-		}
-		trav = trav->next;
-	}
-	GF_FREE (pathname);
-	return hashed_subvol;
+                        subvol = trav->array[idx].xl;
+                        goto out;
+                }
+                trav = trav->next;
+        }
+out:
+        GF_FREE (pathname);
+
+        return subvol;
 }
 
 
@@ -116,7 +117,7 @@ switch_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         loc_t        *loc         = NULL;
         int           i           = 0;
         call_frame_t *prev        = NULL;
-	int           call_cnt    = 0;
+        int           call_cnt    = 0;
         int           ret         = 0;
 
         conf  = this->private;
@@ -125,57 +126,55 @@ switch_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local = frame->local;
         loc   = &local->loc;
 
-	if (ENTRY_MISSING (op_ret, op_errno)) {
-		if (conf->search_unhashed) {
-			local->op_errno = ENOENT;
-			dht_lookup_everywhere (frame, this, loc);
-			return 0;
-		}
-	}
+        if (ENTRY_MISSING (op_ret, op_errno)) {
+                if (conf->search_unhashed) {
+                        local->op_errno = ENOENT;
+                        dht_lookup_everywhere (frame, this, loc);
+                        return 0;
+                }
+        }
 
         if (op_ret == -1)
                 goto out;
 
-        is_linkfile = check_is_linkfile (inode, stbuf, xattr);
+        is_linkfile = check_is_linkfile (inode, stbuf, xattr,
+                                         conf->link_xattr_name);
         is_dir      = check_is_dir (inode, stbuf, xattr);
 
         if (!is_dir && !is_linkfile) {
                 /* non-directory and not a linkfile */
 
-		dht_itransform (this, prev->this, stbuf->ia_ino,
-				&stbuf->ia_ino);
-
-		ret = dht_layout_preset (this, prev->this, inode);
-		if (ret < 0) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"could not set pre-set layout for subvol %s",
-				prev->this->name);
-			op_ret   = -1;
-			op_errno = EINVAL;
-			goto err;
-		}
+                ret = dht_layout_preset (this, prev->this, inode);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "could not set pre-set layout for subvol %s",
+                                prev->this->name);
+                        op_ret   = -1;
+                        op_errno = EINVAL;
+                        goto err;
+                }
 
                 goto out;
         }
 
         if (is_dir) {
                 call_cnt        = conf->subvolume_cnt;
-		local->call_cnt = call_cnt;
+                local->call_cnt = call_cnt;
 
                 local->inode = inode_ref (inode);
                 local->xattr = dict_ref (xattr);
 
-		local->op_ret = 0;
-		local->op_errno = 0;
+                local->op_ret = 0;
+                local->op_errno = 0;
 
-		local->layout = dht_layout_new (this, conf->subvolume_cnt);
-		if (!local->layout) {
-			op_ret   = -1;
-			op_errno = ENOMEM;
-			gf_log (this->name, GF_LOG_DEBUG,
-				"memory allocation failed :(");
-			goto err;
-		}
+                local->layout = dht_layout_new (this, conf->subvolume_cnt);
+                if (!local->layout) {
+                        op_ret   = -1;
+                        op_errno = ENOMEM;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "memory allocation failed :(");
+                        goto err;
+                }
 
                 for (i = 0; i < call_cnt; i++) {
                         STACK_WIND (frame, dht_lookup_dir_cbk,
@@ -192,34 +191,34 @@ switch_local_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "linkfile not having link subvolume. path=%s",
                                 loc->path);
-			dht_lookup_everywhere (frame, this, loc);
-			return 0;
+                        dht_lookup_everywhere (frame, this, loc);
+                        return 0;
                 }
 
-		STACK_WIND (frame, dht_lookup_linkfile_cbk,
-			    subvol, subvol->fops->lookup,
-			    &local->loc, local->xattr_req);
+                STACK_WIND (frame, dht_lookup_linkfile_cbk,
+                            subvol, subvol->fops->lookup,
+                            &local->loc, local->xattr_req);
         }
 
         return 0;
 
 out:
-	if (!local->hashed_subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no subvolume in layout for path=%s",
-			local->loc.path);
+        if (!local->hashed_subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        local->loc.path);
                 local->op_errno = ENOENT;
                 dht_lookup_everywhere (frame, this, loc);
                 return 0;
-	}
+        }
 
-	STACK_WIND (frame, dht_lookup_cbk,
-		    local->hashed_subvol, local->hashed_subvol->fops->lookup,
-		    &local->loc, local->xattr_req);
+        STACK_WIND (frame, dht_lookup_cbk,
+                    local->hashed_subvol, local->hashed_subvol->fops->lookup,
+                    &local->loc, local->xattr_req);
 
-	return 0;
+        return 0;
 
- err:
+err:
         DHT_STACK_UNWIND (lookup, frame, op_ret, op_errno,
                           inode, stbuf, xattr, NULL);
         return 0;
@@ -227,18 +226,18 @@ out:
 
 int
 switch_lookup (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, dict_t *xattr_req)
+               loc_t *loc, dict_t *xattr_req)
 {
         xlator_t     *hashed_subvol = NULL;
         xlator_t     *cached_subvol = NULL;
         xlator_t     *subvol = NULL;
         dht_local_t  *local  = NULL;
-	dht_conf_t   *conf = NULL;
+        dht_conf_t   *conf = NULL;
         int           ret    = -1;
         int           op_errno = -1;
-	dht_layout_t *layout = NULL;
-	int           i = 0;
-	int           call_cnt = 0;
+        dht_layout_t *layout = NULL;
+        int           i = 0;
+        int           call_cnt = 0;
 
 
         VALIDATE_OR_GOTO (frame, err);
@@ -247,40 +246,27 @@ switch_lookup (call_frame_t *frame, xlator_t *this,
         VALIDATE_OR_GOTO (loc->inode, err);
         VALIDATE_OR_GOTO (loc->path, err);
 
-	conf = this->private;
+        conf = this->private;
 
-        local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
-
-        ret = loc_dup (loc, &local->loc);
-        if (ret == -1) {
-                op_errno = errno;
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "copying location failed for path=%s",
-                        loc->path);
+        local = dht_local_init (frame, loc, NULL, GF_FOP_LOOKUP);
+        if (!local) {
+                op_errno = ENOMEM;
                 goto err;
         }
 
-	if (xattr_req) {
-		local->xattr_req = dict_ref (xattr_req);
-	} else {
-		local->xattr_req = dict_new ();
-	}
+        if (xattr_req) {
+                local->xattr_req = dict_ref (xattr_req);
+        } else {
+                local->xattr_req = dict_new ();
+        }
 
-	hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
-	cached_subvol = dht_subvol_get_cached (this, local->loc.inode);
+        hashed_subvol = dht_subvol_get_hashed (this, &local->loc);
+        cached_subvol = local->cached_subvol;
 
-	local->cached_subvol = cached_subvol;
-	local->hashed_subvol = hashed_subvol;
+        local->hashed_subvol = hashed_subvol;
 
         if (is_revalidate (loc)) {
-		local->layout = layout = dht_layout_get (this, loc->inode);
-
+                layout = local->layout;
                 if (!layout) {
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "revalidate without cache. path=%s",
@@ -289,83 +275,80 @@ switch_lookup (call_frame_t *frame, xlator_t *this,
                         goto err;
                 }
 
-		if (layout->gen && (layout->gen < conf->gen)) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"incomplete layout failure for path=%s",
-				loc->path);
+                if (layout->gen && (layout->gen < conf->gen)) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "incomplete layout failure for path=%s",
+                                loc->path);
                         dht_layout_unref (this, local->layout);
-			goto do_fresh_lookup;
-		}
+                        goto do_fresh_lookup;
+                }
 
-		local->inode    = inode_ref (loc->inode);
-		local->ia_ino   = loc->inode->ino;
+                local->inode    = inode_ref (loc->inode);
 
-		local->call_cnt = layout->cnt;
-		call_cnt = local->call_cnt;
+                local->call_cnt = layout->cnt;
+                call_cnt = local->call_cnt;
 
-		/* NOTE: we don't require 'trusted.glusterfs.dht.linkto'
+                /* NOTE: we don't require 'trusted.glusterfs.dht.linkto'
                  * attribute, revalidates directly go to the cached-subvolume.
-		 */
-		ret = dict_set_uint32 (local->xattr_req,
-				       "trusted.glusterfs.dht", 4 * 4);
+                 */
+                ret = dict_set_uint32 (local->xattr_req,
+                                       conf->xattr_name, 4 * 4);
                 if (ret < 0)
                         gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set dict value for "
-                                "trusted.glusterfs.dht");
+                                "failed to set dict value for %s",
+                                conf->xattr_name);
 
-		for (i = 0; i < layout->cnt; i++) {
-			subvol = layout->list[i].xlator;
+                for (i = 0; i < layout->cnt; i++) {
+                        subvol = layout->list[i].xlator;
 
-			STACK_WIND (frame, dht_revalidate_cbk,
-				    subvol, subvol->fops->lookup,
-				    loc, local->xattr_req);
+                        STACK_WIND (frame, dht_revalidate_cbk,
+                                    subvol, subvol->fops->lookup,
+                                    loc, local->xattr_req);
 
-			if (!--call_cnt)
-				break;
-		}
-	} else {
+                        if (!--call_cnt)
+                                break;
+                }
+        } else {
         do_fresh_lookup:
-		ret = dict_set_uint32 (local->xattr_req,
-				       "trusted.glusterfs.dht", 4 * 4);
+                ret = dict_set_uint32 (local->xattr_req,
+                                       conf->xattr_name, 4 * 4);
                 if (ret < 0)
                         gf_log (this->name, GF_LOG_WARNING,
-                                "failed to set dict value for "
-                                "trusted.glusterfs.dht");
-                
-		ret = dict_set_uint32 (local->xattr_req,
-				       "trusted.glusterfs.dht.linkto", 256);
+                                "failed to set dict value for %s",
+                                conf->xattr_name);
+
+                ret = dict_set_uint32 (local->xattr_req,
+                                       conf->link_xattr_name, 256);
                 if (ret < 0)
-                        gf_log (this->name, GF_LOG_WARNING, 
-                                "failed to set dict value for " 
-                                "trusted.glusterfs.dht.linkto");
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to set dict value for %s",
+                                conf->link_xattr_name);
 
                 if (!hashed_subvol) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"no subvolume in layout for path=%s, "
-				"checking on all the subvols to see if "
-				"it is a directory", loc->path);
- 			call_cnt        = conf->subvolume_cnt;
- 			local->call_cnt = call_cnt;
-
- 			local->layout = dht_layout_new (this,
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "no subvolume in layout for path=%s, "
+                                "checking on all the subvols to see if "
+                                "it is a directory", loc->path);
+                        call_cnt        = conf->subvolume_cnt;
+                        local->call_cnt = call_cnt;
+
+                        local->layout = dht_layout_new (this,
                                                         conf->subvolume_cnt);
- 			if (!local->layout) {
- 				op_errno = ENOMEM;
- 				gf_log (this->name, GF_LOG_ERROR,
- 					"Out of memory");
- 				goto err;
- 			}
-
-			for (i = 0; i < call_cnt; i++) {
- 				STACK_WIND (frame, dht_lookup_dir_cbk,
- 					    conf->subvolumes[i],
- 					    conf->subvolumes[i]->fops->lookup,
- 					    &local->loc, local->xattr_req);
- 			}
- 			return 0;
+                        if (!local->layout) {
+                                op_errno = ENOMEM;
+                                goto err;
+                        }
+
+                        for (i = 0; i < call_cnt; i++) {
+                                STACK_WIND (frame, dht_lookup_dir_cbk,
+                                            conf->subvolumes[i],
+                                            conf->subvolumes[i]->fops->lookup,
+                                            &local->loc, local->xattr_req);
+                        }
+                        return 0;
                 }
 
-		/*  */
+                /*  */
                 cached_subvol = get_switch_matching_subvol (loc->path, conf,
                                                             hashed_subvol);
                 if (cached_subvol == hashed_subvol) {
@@ -384,99 +367,88 @@ switch_lookup (call_frame_t *frame, xlator_t *this,
         return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-        DHT_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
-	return 0;
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (lookup, frame, -1, op_errno,
+                          NULL, NULL, NULL, NULL);
+        return 0;
 }
 
 int
 switch_create_linkfile_create_cbk (call_frame_t *frame, void *cookie,
-				 xlator_t *this, int op_ret, int op_errno,
-                                 inode_t *inode, struct iatt *stbuf,
-                                 struct iatt *preparent,
-                                 struct iatt *postparent)
+                                   xlator_t *this, int op_ret, int op_errno,
+                                   inode_t *inode, struct iatt *stbuf,
+                                   struct iatt *preparent,
+                                   struct iatt *postparent, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
+        dht_local_t  *local = NULL;
 
-	local = frame->local;
+        local = frame->local;
 
-	if (op_ret == -1)
-		goto err;
+        if (op_ret == -1)
+                goto err;
 
-	STACK_WIND (frame, dht_create_cbk,
-		    local->cached_subvol, local->cached_subvol->fops->create,
-		    &local->loc, local->flags, local->mode, local->fd,
-                    local->params);
+        STACK_WIND (frame, dht_create_cbk,
+                    local->cached_subvol, local->cached_subvol->fops->create,
+                    &local->loc, local->flags, local->mode, local->umask,
+                    local->fd, local->params);
 
-	return 0;
+        return 0;
 
- err:
-	DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
-	return 0;
+err:
+        DHT_STACK_UNWIND (create, frame, -1, op_errno,
+                          NULL, NULL, NULL, NULL, NULL, NULL);
+        return 0;
 }
 
 int
 switch_create (call_frame_t *frame, xlator_t *this,
                loc_t *loc, int32_t flags, mode_t mode,
-               fd_t *fd, dict_t *params)
+               mode_t umask, fd_t *fd, dict_t *params)
 {
-	dht_local_t *local = NULL;
-	dht_conf_t  *conf  = NULL;
-	xlator_t    *subvol = NULL;
+        dht_local_t *local = NULL;
+        dht_conf_t  *conf  = NULL;
+        xlator_t    *subvol = NULL;
         xlator_t    *avail_subvol = NULL;
-	int          op_errno = -1;
-	int          ret = -1;
+        int          op_errno = -1;
 
-	VALIDATE_OR_GOTO (frame, err);
-	VALIDATE_OR_GOTO (this, err);
-	VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
 
-	conf  = this->private;
+        conf  = this->private;
 
         dht_get_du_info (frame, this, loc);
 
-        local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
-
-	subvol = dht_subvol_get_hashed (this, loc);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no subvolume in layout for path=%s",
-			loc->path);
-		op_errno = ENOENT;
-		goto err;
-	}
+        local = dht_local_init (frame, loc, fd, GF_FOP_CREATE);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = dht_subvol_get_hashed (this, loc);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        loc->path);
+                op_errno = ENOENT;
+                goto err;
+        }
 
         avail_subvol = get_switch_matching_subvol (loc->path, conf, subvol);
         if (dht_is_subvol_filled (this, avail_subvol)) {
                 avail_subvol =
-                        dht_free_disk_available_subvol (this, avail_subvol);
+                        dht_free_disk_available_subvol (this, avail_subvol,
+                                                        local);
         }
 
         if (subvol != avail_subvol) {
                 /* create a link file instead of actual file */
-                ret = loc_copy (&local->loc, loc);
-                if (ret == -1) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Out of memory");
-                                op_errno = ENOMEM;
-                                goto err;
-                }
-
-                local->fd = fd_ref (fd);
                 local->mode = mode;
                 local->flags = flags;
-
+                local->umask = umask;
                 local->cached_subvol = avail_subvol;
-                dht_linkfile_create (frame,
-                                     switch_create_linkfile_create_cbk,
-                                     avail_subvol, subvol, loc);
+                dht_linkfile_create (frame, switch_create_linkfile_create_cbk,
+                                     this, avail_subvol, subvol, loc);
                 return 0;
         }
 
@@ -485,176 +457,143 @@ switch_create (call_frame_t *frame, xlator_t *this,
 
         STACK_WIND (frame, dht_create_cbk,
                     subvol, subvol->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, params);
 
         return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (create, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (create, frame, -1, op_errno,
+                          NULL, NULL, NULL, NULL, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 int
 switch_mknod_linkfile_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int op_ret, int op_errno, inode_t *inode,
-                         struct iatt *stbuf, struct iatt *preparent,
-                         struct iatt *postparent)
+                           int op_ret, int op_errno, inode_t *inode,
+                           struct iatt *stbuf, struct iatt *preparent,
+                           struct iatt *postparent, dict_t *xdata)
 {
-	dht_local_t  *local = NULL;
-
-	local = frame->local;
+        dht_local_t  *local = NULL;
 
-	if (op_ret >= 0) {
-		STACK_WIND (frame, dht_newfile_cbk,
-			    local->cached_subvol,
-			    local->cached_subvol->fops->mknod,
-			    &local->loc, local->mode, local->rdev,
-                            local->params);
+        local = frame->local;
+        if (!local || !local->cached_subvol) {
+                op_errno = EINVAL;
+                op_ret = -1;
+                goto err;
+        }
 
-		return 0;
-	}
+        if (op_ret >= 0) {
+                STACK_WIND_COOKIE (frame, dht_newfile_cbk,
+                            (void *)local->cached_subvol, local->cached_subvol,
+                            local->cached_subvol->fops->mknod,
+                            &local->loc, local->mode, local->rdev,
+                            local->umask, local->params);
 
-	DHT_STACK_UNWIND (link, frame, op_ret, op_errno,
-                          inode, stbuf, preparent, postparent);
-	return 0;
+                return 0;
+        }
+err:
+        DHT_STACK_UNWIND (link, frame, op_ret, op_errno,
+                          inode, stbuf, preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
-switch_mknod (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, mode_t mode, dev_t rdev, dict_t *params)
+switch_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+              dev_t rdev, mode_t umask, dict_t *params)
 {
-	dht_local_t *local = NULL;
-	dht_conf_t  *conf  = NULL;
-	xlator_t    *subvol = NULL;
+        dht_local_t *local = NULL;
+        dht_conf_t  *conf  = NULL;
+        xlator_t    *subvol = NULL;
         xlator_t    *avail_subvol = NULL;
-	int          op_errno = -1;
-	int          ret = -1;
+        int          op_errno = -1;
 
-	VALIDATE_OR_GOTO (frame, err);
-	VALIDATE_OR_GOTO (this, err);
-	VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
 
-	conf  = this->private;
+        conf  = this->private;
 
         dht_get_du_info (frame, this, loc);
 
-        local = dht_local_init (frame);
-	if (!local) {
-		op_errno = ENOMEM;
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		goto err;
-	}
-
-	subvol = dht_subvol_get_hashed (this, loc);
-	if (!subvol) {
-		gf_log (this->name, GF_LOG_DEBUG,
-			"no subvolume in layout for path=%s",
-			loc->path);
-		op_errno = ENOENT;
-		goto err;
-	}
+        local = dht_local_init (frame, loc, NULL, GF_FOP_MKNOD);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        subvol = dht_subvol_get_hashed (this, loc);
+        if (!subvol) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "no subvolume in layout for path=%s",
+                        loc->path);
+                op_errno = ENOENT;
+                goto err;
+        }
 
         /* Consider the disksize in consideration */
         avail_subvol = get_switch_matching_subvol (loc->path, conf, subvol);
         if (dht_is_subvol_filled (this, avail_subvol)) {
                 avail_subvol =
-                        dht_free_disk_available_subvol (this, avail_subvol);
+                        dht_free_disk_available_subvol (this, avail_subvol,
+                                                        local);
         }
 
-	if (avail_subvol != subvol) {
-		/* Create linkfile first */
-		ret = loc_copy (&local->loc, loc);
-		if (ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"Out of memory");
-			op_errno = ENOMEM;
-			goto err;
-		}
+        if (avail_subvol != subvol) {
+                /* Create linkfile first */
 
                 local->params = dict_ref (params);
-		local->mode = mode;
-		local->rdev = rdev;
-		local->cached_subvol = avail_subvol;
+                local->mode = mode;
+                local->umask = umask;
+                local->rdev = rdev;
+                local->cached_subvol = avail_subvol;
 
-		dht_linkfile_create (frame, switch_mknod_linkfile_cbk,
-                                     avail_subvol, subvol, loc);
-		return 0;
-	}
+                dht_linkfile_create (frame, switch_mknod_linkfile_cbk,
+                                     this, avail_subvol, subvol, loc);
+                return 0;
+        }
 
-	gf_log (this->name, GF_LOG_TRACE,
-		"creating %s on %s", loc->path, subvol->name);
+        gf_log (this->name, GF_LOG_TRACE,
+                "creating %s on %s", loc->path, subvol->name);
 
-	STACK_WIND (frame, dht_newfile_cbk,
-		    subvol, subvol->fops->mknod,
-		    loc, mode, rdev, params);
+        STACK_WIND_COOKIE (frame, dht_newfile_cbk, (void *)subvol, subvol,
+                           subvol->fops->mknod, loc, mode, rdev, umask,
+                           params);
 
-	return 0;
+        return 0;
 
 err:
-	op_errno = (op_errno == -1) ? errno : op_errno;
-	DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
-                          NULL, NULL, NULL, NULL);
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        DHT_STACK_UNWIND (mknod, frame, -1, op_errno,
+                          NULL, NULL, NULL, NULL, NULL);
 
-	return 0;
+        return 0;
 }
 
 
-int
-notify (xlator_t *this, int event, void *data, ...)
-{
-	int ret = -1;
-
-	ret = dht_notify (this, event, data);
-
-	return ret;
-}
-
 void
-fini (xlator_t *this)
+switch_fini (xlator_t *this)
 {
-        int                   i = 0;
         dht_conf_t           *conf = NULL;
         struct switch_struct *trav = NULL;
         struct switch_struct *prev = NULL;
 
-	conf = this->private;
+        conf = this->private;
 
         if (conf) {
                 trav = (struct switch_struct *)conf->private;
                 conf->private = NULL;
                 while (trav) {
-                        if (trav->array)
-                                GF_FREE (trav->array);
+                        GF_FREE (trav->array);
                         prev = trav;
                         trav = trav->next;
                         GF_FREE (prev);
                 }
-
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                if (conf->default_dir_layout)
-                        GF_FREE (conf->default_dir_layout);
-
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
-
-		if (conf->subvolume_status)
-			GF_FREE (conf->subvolume_status);
-
-                GF_FREE (conf);
         }
 
-	return;
+        dht_fini(this);
 }
 
 int
@@ -679,32 +618,32 @@ set_switch_pattern (xlator_t *this, dht_conf_t *conf,
         struct switch_struct        *switch_buf = NULL;
         struct switch_struct        *switch_opt = NULL;
         struct switch_struct        *trav = NULL;
-	struct switch_sched_array  *switch_buf_array = NULL;
-	xlator_list_t              *trav_xl = NULL;
+        struct switch_sched_array  *switch_buf_array = NULL;
+        xlator_list_t              *trav_xl = NULL;
 
         trav_xl = this->children;
-	while (trav_xl) {
-		index++;
-		trav_xl = trav_xl->next;
-	}
-	child_count = index;
-	switch_buf_array = GF_CALLOC ((index + 1),
+        while (trav_xl) {
+                index++;
+                trav_xl = trav_xl->next;
+        }
+        child_count = index;
+        switch_buf_array = GF_CALLOC ((index + 1),
                                       sizeof (struct switch_sched_array),
                                       gf_switch_mt_switch_sched_array);
         if (!switch_buf_array)
                 goto err;
 
-	trav_xl = this->children;
-	index = 0;
+        trav_xl = this->children;
+        index = 0;
 
-	while (trav_xl) {
-		switch_buf_array[index].xl = trav_xl->xlator;
-		switch_buf_array[index].eligible = 1;
-		trav_xl = trav_xl->next;
-		index++;
-	}
+        while (trav_xl) {
+                switch_buf_array[index].xl = trav_xl->xlator;
+                switch_buf_array[index].eligible = 1;
+                trav_xl = trav_xl->next;
+                index++;
+        }
 
-	/*  *jpg:child1,child2;*mpg:child3;*:child4,child5,child6 */
+        /*  *jpg:child1,child2;*mpg:child3;*:child4,child5,child6 */
 
         /* Get the pattern for considering switch case.
            "option block-size *avi:10MB" etc */
@@ -714,20 +653,24 @@ set_switch_pattern (xlator_t *this, dht_conf_t *conf,
                 dup_str = gf_strdup (switch_str);
                 switch_opt = GF_CALLOC (1, sizeof (struct switch_struct),
                                         gf_switch_mt_switch_struct);
-                if (!switch_opt)
+                if (!switch_opt) {
+                        GF_FREE (dup_str);
                         goto err;
+                }
 
                 pattern = strtok_r (dup_str, ":", &tmp_str1);
                 childs = strtok_r (NULL, ":", &tmp_str1);
                 if (strncmp (pattern, "*", 2) == 0) {
-                        gf_log ("switch", GF_LOG_NORMAL,
+                        gf_log ("switch", GF_LOG_INFO,
                                 "'*' pattern will be taken by default "
                                 "for all the unconfigured child nodes,"
                                 " hence neglecting current option");
                         switch_str = strtok_r (NULL, ";", &tmp_str);
+                        GF_FREE (switch_opt);
                         GF_FREE (dup_str);
                         continue;
                 }
+                GF_FREE (dup_str);
                 memcpy (switch_opt->path_pattern, pattern, strlen (pattern));
                 if (childs) {
                         dup_childs = gf_strdup (childs);
@@ -749,7 +692,7 @@ set_switch_pattern (xlator_t *this, dht_conf_t *conf,
                         child = strtok_r (childs, ",", &tmp1);
                         switch_opt->num_child = idx;
                         switch_opt->array = GF_CALLOC (1, (idx *
-                                                       sizeof (struct switch_sched_array)),
+                                                           sizeof (struct switch_sched_array)),
                                                        gf_switch_mt_switch_sched_array);
                         if (!switch_opt->array)
                                 goto err;
@@ -784,8 +727,7 @@ set_switch_pattern (xlator_t *this, dht_conf_t *conf,
                                 "option in unify volume. Exiting");
                         goto err;
                 }
-                GF_FREE (dup_str);
-                
+
                 /* Link it to the main structure */
                 if (switch_buf) {
                         /* there are already few entries */
@@ -797,75 +739,77 @@ set_switch_pattern (xlator_t *this, dht_conf_t *conf,
                         /* First entry */
                         switch_buf = switch_opt;
                 }
+                switch_opt = NULL;
                 switch_str = strtok_r (NULL, ";", &tmp_str);
         }
 
-	/* Now, all the pattern based considerations done, so for all the
-	 * remaining pattern, '*' to all the remaining child nodes
-	 */
-	{
-		for (index=0; index < child_count; index++) {
-			/* check for considered flag */
-			if (switch_buf_array[index].considered)
-				continue;
-			flag++;
-		}
-		if (!flag) {
-			gf_log ("switch", GF_LOG_ERROR,
-				"No nodes left for pattern '*'. Exiting");
-			goto err;
-		}
-		switch_opt = GF_CALLOC (1, sizeof (struct switch_struct),
+        /* Now, all the pattern based considerations done, so for all the
+         * remaining pattern, '*' to all the remaining child nodes
+         */
+        {
+                for (index=0; index < child_count; index++) {
+                        /* check for considered flag */
+                        if (switch_buf_array[index].considered)
+                                continue;
+                        flag++;
+                }
+                if (!flag) {
+                        gf_log ("switch", GF_LOG_ERROR,
+                                "No nodes left for pattern '*'. Exiting");
+                        goto err;
+                }
+                switch_opt = GF_CALLOC (1, sizeof (struct switch_struct),
                                         gf_switch_mt_switch_struct);
                 if (!switch_opt)
                         goto err;
 
-		/* Add the '*' pattern to the array */
-		memcpy (switch_opt->path_pattern, "*", 2);
-		switch_opt->num_child = flag;
-		switch_opt->array =
-			GF_CALLOC (1, 
+                /* Add the '*' pattern to the array */
+                memcpy (switch_opt->path_pattern, "*", 2);
+                switch_opt->num_child = flag;
+                switch_opt->array =
+                        GF_CALLOC (1,
                                    flag * sizeof (struct switch_sched_array),
                                    gf_switch_mt_switch_sched_array);
                 if (!switch_opt->array)
                         goto err;
-		flag = 0;
-		for (index=0; index < child_count; index++) {
-			/* check for considered flag */
-			if (switch_buf_array[index].considered)
-				continue;
-			gf_log ("switch", GF_LOG_DEBUG,
-				"'%s' pattern will be scheduled to \"%s\"",
-				switch_opt->path_pattern,
-				switch_buf_array[index].xl->name);
-			switch_opt->array[flag].xl =
-				switch_buf_array[index].xl;
-			switch_buf_array[index].considered = 1;
-			flag++;
+                flag = 0;
+                for (index=0; index < child_count; index++) {
+                        /* check for considered flag */
+                        if (switch_buf_array[index].considered)
+                                continue;
+                        gf_log ("switch", GF_LOG_DEBUG,
+                                "'%s' pattern will be scheduled to \"%s\"",
+                                switch_opt->path_pattern,
+                                switch_buf_array[index].xl->name);
+                        switch_opt->array[flag].xl =
+                                switch_buf_array[index].xl;
+                        switch_buf_array[index].considered = 1;
+                        flag++;
                 }
-		if (switch_buf) {
-			/* there are already few entries */
-			trav = switch_buf;
-			while (trav->next)
-				trav = trav->next;
-			trav->next = switch_opt;
-		} else {
-			/* First entry */
-			switch_buf = switch_opt;
-		}
-	}
+                if (switch_buf) {
+                        /* there are already few entries */
+                        trav = switch_buf;
+                        while (trav->next)
+                                trav = trav->next;
+                        trav->next = switch_opt;
+                } else {
+                        /* First entry */
+                        switch_buf = switch_opt;
+                }
+                switch_opt = NULL;
+        }
         /* */
         conf->private = switch_buf;
 
         return 0;
 err:
+        GF_FREE (switch_buf_array);
+        GF_FREE (switch_opt);
+
         if (switch_buf) {
-                if (switch_buf_array)
-                        GF_FREE (switch_buf_array);
                 trav = switch_buf;
                 while (trav) {
-                        if (trav->array)
-                                GF_FREE (trav->array);
+                        GF_FREE (trav->array);
                         switch_opt = trav;
                         trav = trav->next;
                         GF_FREE (switch_opt);
@@ -875,192 +819,86 @@ err:
 }
 
 
-int
-init (xlator_t *this)
+int32_t
+switch_init (xlator_t *this)
 {
         dht_conf_t            *conf = NULL;
-	data_t                *data = NULL;
-	char                  *temp_str = NULL;
+        data_t                *data = NULL;
         int                    ret = -1;
-        int                    i = 0;
-	uint32_t               temp_free_disk = 0;
-
-	if (!this->children) {
-		gf_log (this->name, GF_LOG_CRITICAL,
-			"SWITCH needs more than one subvolume");
-		return -1;
-	}
-
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile");
-	}
-
-        conf = GF_CALLOC (1, sizeof (*conf), gf_switch_mt_dht_conf_t);
-        if (!conf) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                goto err;
-        }
 
-        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_ON;
-	if (dict_get_str (this->options, "lookup-unhashed", &temp_str) == 0) {
-                /* If option is not "auto", other options _should_ be boolean */
-                if (strcasecmp (temp_str, "auto"))
-                        gf_string2boolean (temp_str, &conf->search_unhashed);
-                else
-                        conf->search_unhashed = GF_DHT_LOOKUP_UNHASHED_AUTO;
-	}
-
-	conf->unhashed_sticky_bit = 0;
-	if (dict_get_str (this->options, "unhashed-sticky-bit",
-                          &temp_str) == 0) {
-	        gf_string2boolean (temp_str, &conf->unhashed_sticky_bit);
-	}
-
-        conf->min_free_disk = 10;
-        conf->disk_unit = 'p';
-
-        if (dict_get_str (this->options, "min-free-disk",
-                          &temp_str) == 0) {
-                if (gf_string2percent (temp_str,
-                                       &temp_free_disk) == 0) {
-                        if (temp_free_disk > 100) {
-                                gf_string2bytesize (temp_str,
-                                                        &conf->min_free_disk);
-                                conf->disk_unit = 'b';
-                        } else {
-                                conf->min_free_disk = (uint64_t)temp_free_disk;
-                                conf->disk_unit = 'p';
-                        }
-                } else {
-                        gf_string2bytesize (temp_str,
-                                                &conf->min_free_disk);
-                        conf->disk_unit = 'b';
-                }
+        ret = dht_init(this);
+        if (ret) {
+                return ret;
         }
+        conf = this->private;
 
-	data = dict_get (this->options, "pattern.switch.case");
-	if (data) {
+        data = dict_get (this->options, "pattern.switch.case");
+        if (data) {
                 /* TODO: */
                 ret = set_switch_pattern (this, conf, data->data);
                 if (ret) {
                         goto err;
                 }
-	}
-
-        ret = dht_init_subvolumes (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-        ret = dht_layouts_init (this, conf);
-        if (ret == -1) {
-                goto err;
-        }
-
-	LOCK_INIT (&conf->subvolume_lock);
-	LOCK_INIT (&conf->layout_lock);
-
-	conf->gen = 1;
-
-        conf->du_stats = GF_CALLOC (conf->subvolume_cnt, sizeof (dht_du_t),
-                                    gf_switch_mt_dht_du_t);
-        if (!conf->du_stats) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                goto err;
         }
 
         this->private = conf;
-
         return 0;
 
 err:
-        if (conf) {
-                if (conf->file_layouts) {
-                        for (i = 0; i < conf->subvolume_cnt; i++) {
-                                GF_FREE (conf->file_layouts[i]);
-                        }
-                        GF_FREE (conf->file_layouts);
-                }
-
-                if (conf->default_dir_layout)
-                        GF_FREE (conf->default_dir_layout);
-
-                if (conf->subvolumes)
-                        GF_FREE (conf->subvolumes);
-
-		if (conf->subvolume_status)
-			GF_FREE (conf->subvolume_status);
-
-                if (conf->du_stats)
-                        GF_FREE (conf->du_stats);
-
-                GF_FREE (conf);
-        }
-
+        dht_fini(this);
         return -1;
 }
 
 
-struct xlator_fops fops = {
-	.lookup      = switch_lookup,
-	.create      = switch_create,
-	.mknod       = switch_mknod,
-
-	.stat        = dht_stat,
-	.fstat       = dht_fstat,
-	.truncate    = dht_truncate,
-	.ftruncate   = dht_ftruncate,
-	.access      = dht_access,
-	.readlink    = dht_readlink,
-	.setxattr    = dht_setxattr,
-	.getxattr    = dht_getxattr,
-	.removexattr = dht_removexattr,
-	.open        = dht_open,
-	.readv       = dht_readv,
-	.writev      = dht_writev,
-	.flush       = dht_flush,
-	.fsync       = dht_fsync,
-	.statfs      = dht_statfs,
-	.lk          = dht_lk,
-	.opendir     = dht_opendir,
-	.readdir     = dht_readdir,
-	.readdirp    = dht_readdirp,
-	.fsyncdir    = dht_fsyncdir,
-	.symlink     = dht_symlink,
-	.unlink      = dht_unlink,
-	.link        = dht_link,
-	.mkdir       = dht_mkdir,
-	.rmdir       = dht_rmdir,
-	.rename      = dht_rename,
-	.inodelk     = dht_inodelk,
-	.finodelk    = dht_finodelk,
-	.entrylk     = dht_entrylk,
-	.fentrylk    = dht_fentrylk,
-	.xattrop     = dht_xattrop,
-	.fxattrop    = dht_fxattrop,
-        .setattr     = dht_setattr,
+class_methods_t class_methods = {
+        .init           = switch_init,
+        .fini           = switch_fini,
+        .reconfigure    = dht_reconfigure,
+        .notify         = dht_notify
 };
 
 
-struct xlator_cbks cbks = {
-	.forget     = dht_forget
+struct xlator_fops fops = {
+        .lookup      = switch_lookup,
+        .create      = switch_create,
+        .mknod       = switch_mknod,
+
+        .stat        = dht_stat,
+        .fstat       = dht_fstat,
+        .truncate    = dht_truncate,
+        .ftruncate   = dht_ftruncate,
+        .access      = dht_access,
+        .readlink    = dht_readlink,
+        .setxattr    = dht_setxattr,
+        .getxattr    = dht_getxattr,
+        .removexattr = dht_removexattr,
+        .open        = dht_open,
+        .readv       = dht_readv,
+        .writev      = dht_writev,
+        .flush       = dht_flush,
+        .fsync       = dht_fsync,
+        .statfs      = dht_statfs,
+        .lk          = dht_lk,
+        .opendir     = dht_opendir,
+        .readdir     = dht_readdir,
+        .readdirp    = dht_readdirp,
+        .fsyncdir    = dht_fsyncdir,
+        .symlink     = dht_symlink,
+        .unlink      = dht_unlink,
+        .link        = dht_link,
+        .mkdir       = dht_mkdir,
+        .rmdir       = dht_rmdir,
+        .rename      = dht_rename,
+        .inodelk     = dht_inodelk,
+        .finodelk    = dht_finodelk,
+        .entrylk     = dht_entrylk,
+        .fentrylk    = dht_fentrylk,
+        .xattrop     = dht_xattrop,
+        .fxattrop    = dht_fxattrop,
+        .setattr     = dht_setattr,
 };
 
 
-struct volume_options options[] = {
-        { .key  = {"lookup-unhashed"},
-          .value = {"auto", "yes", "no", "enable", "disable", "1", "0",
-                    "on", "off"},
-	  .type = GF_OPTION_TYPE_STR
-	},
-	{ .key  = {"pattern.switch.case"},
-	  .type = GF_OPTION_TYPE_ANY
-	},
-        { .key  = {"min-free-disk"},
-          .type = GF_OPTION_TYPE_PERCENT_OR_SIZET,
-        },
-	{ .key  = {NULL} },
+struct xlator_cbks cbks = {
+        .forget     = dht_forget
 };
diff --git a/xlators/cluster/dht/src/unittest/dht_layout_mock.c b/xlators/cluster/dht/src/unittest/dht_layout_mock.c
new file mode 100644
index 000000000..aa19ddc57
--- /dev/null
+++ b/xlators/cluster/dht/src/unittest/dht_layout_mock.c
@@ -0,0 +1,63 @@
+/*
+  Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "dht-common.h"
+#include "byte-order.h"
+
+int
+dht_hash_compute (xlator_t *this, int type, const char *name, uint32_t *hash_p)
+{
+    return 0;
+}
+
+int
+dht_inode_ctx_layout_get (inode_t *inode, xlator_t *this, dht_layout_t **layout)
+{
+    return 0;
+}
+
+int
+dht_inode_ctx_layout_set (inode_t *inode, xlator_t *this,
+                          dht_layout_t *layout_int)
+{
+    return 0;
+}
+
+int
+dict_get_ptr (dict_t *this, char *key, void **ptr)
+{
+    return 0;
+}
+
+int
+dict_get_ptr_and_len (dict_t *this, char *key, void **ptr, int *len)
+{
+    return 0;
+}
+
+int _gf_log (const char *domain, const char *file,
+             const char *function, int32_t line, gf_loglevel_t level,
+             const char *fmt, ...)
+{
+    return 0;
+}
+
+int _gf_log_callingfn (const char *domain, const char *file,
+                       const char *function, int32_t line, gf_loglevel_t level,
+                       const char *fmt, ...)
+{
+    return 0;
+}
diff --git a/xlators/cluster/dht/src/unittest/dht_layout_unittest.c b/xlators/cluster/dht/src/unittest/dht_layout_unittest.c
new file mode 100644
index 000000000..b5233d235
--- /dev/null
+++ b/xlators/cluster/dht/src/unittest/dht_layout_unittest.c
@@ -0,0 +1,124 @@
+/*
+  Copyright (c) 2008-2014 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include "dht-common.h"
+#include "logging.h"
+#include "xlator.h"
+
+#include <stdarg.h>
+#include <stddef.h>
+#include <setjmp.h>
+#include <inttypes.h>
+#include <cmockery/pbc.h>
+#include <cmockery/cmockery.h>
+
+/*
+ * Helper functions
+ */
+
+static xlator_t *
+helper_xlator_init(uint32_t num_types)
+{
+    xlator_t *xl;
+    int i, ret;
+
+    REQUIRE(num_types > 0);
+
+    xl = test_calloc(1, sizeof(xlator_t));
+    assert_non_null(xl);
+    xl->mem_acct.num_types = num_types;
+    xl->mem_acct.rec = test_calloc(num_types, sizeof(struct mem_acct_rec));
+    assert_non_null(xl->mem_acct.rec);
+
+    xl->ctx = test_calloc(1, sizeof(glusterfs_ctx_t));
+    assert_non_null(xl->ctx);
+
+    for (i = 0; i < num_types; i++) {
+            ret = LOCK_INIT(&(xl->mem_acct.rec[i].lock));
+            assert_false(ret);
+    }
+
+    ENSURE(num_types == xl->mem_acct.num_types);
+    ENSURE(NULL != xl);
+
+    return xl;
+}
+
+static int
+helper_xlator_destroy(xlator_t *xl)
+{
+    int i, ret;
+
+    for (i = 0; i < xl->mem_acct.num_types; i++) {
+            ret = LOCK_DESTROY(&(xl->mem_acct.rec[i].lock));
+            assert_int_equal(ret, 0);
+    }
+
+    free(xl->mem_acct.rec);
+    free(xl->ctx);
+    free(xl);
+    return 0;
+}
+
+/*
+ * Unit tests
+ */
+static void
+test_dht_layout_new(void **state)
+{
+    xlator_t *xl;
+    dht_layout_t *layout;
+    dht_conf_t   *conf;
+    int cnt;
+
+    expect_assert_failure(dht_layout_new(NULL, 0));
+    expect_assert_failure(dht_layout_new((xlator_t *)0x12345, -1));
+    xl = helper_xlator_init(10);
+
+    // xl->private is NULL
+    assert_null(xl->private);
+    cnt = 100;
+    layout = dht_layout_new(xl, cnt);
+    assert_non_null(layout);
+    assert_int_equal(layout->type, DHT_HASH_TYPE_DM);
+    assert_int_equal(layout->cnt, cnt);
+    assert_int_equal(layout->ref, 1);
+    assert_int_equal(layout->gen, 0);
+    assert_int_equal(layout->spread_cnt, 0);
+    free(layout);
+
+    // xl->private is not NULL
+    cnt = 110;
+    conf = (dht_conf_t *)test_calloc(1, sizeof(dht_conf_t));
+    assert_non_null(conf);
+    conf->dir_spread_cnt = 12345;
+    conf->gen = -123;
+    xl->private = conf;
+
+    layout = dht_layout_new(xl, cnt);
+    assert_non_null(layout);
+    assert_int_equal(layout->type, DHT_HASH_TYPE_DM);
+    assert_int_equal(layout->cnt, cnt);
+    assert_int_equal(layout->ref, 1);
+    assert_int_equal(layout->gen, conf->gen);
+    assert_int_equal(layout->spread_cnt, conf->dir_spread_cnt);
+    free(layout);
+
+    free(conf);
+    helper_xlator_destroy(xl);
+}
+
+int main(void) {
+    const UnitTest tests[] = {
+        unit_test(test_dht_layout_new),
+    };
+
+    return run_tests(tests, "xlator_dht_layout");
+}
diff --git a/xlators/cluster/ha/src/Makefile.am b/xlators/cluster/ha/src/Makefile.am
index 5f78a2965..5c1364b7f 100644
--- a/xlators/cluster/ha/src/Makefile.am
+++ b/xlators/cluster/ha/src/Makefile.am
@@ -1,15 +1,16 @@
 xlator_LTLIBRARIES = ha.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/cluster
 
-ha_la_LDFLAGS = -module -avoidversion 
+ha_la_LDFLAGS = -module -avoid-version 
 
 ha_la_SOURCES = ha-helpers.c ha.c
 ha_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = ha.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	    -I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/cluster/ha/src/ha-helpers.c b/xlators/cluster/ha/src/ha-helpers.c
index 759501f9f..19be1ed27 100644
--- a/xlators/cluster/ha/src/ha-helpers.c
+++ b/xlators/cluster/ha/src/ha-helpers.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include "xlator.h"
 #include "call-stub.h"
 #include "defaults.h"
diff --git a/xlators/cluster/ha/src/ha-mem-types.h b/xlators/cluster/ha/src/ha-mem-types.h
index 4d3b5747a..e5e97d237 100644
--- a/xlators/cluster/ha/src/ha-mem-types.h
+++ b/xlators/cluster/ha/src/ha-mem-types.h
@@ -1,24 +1,13 @@
-
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __HA_MEM_TYPES_H__
 #define __HA_MEM_TYPES_H__
 
diff --git a/xlators/cluster/ha/src/ha.c b/xlators/cluster/ha/src/ha.c
index 7310b5e95..3eccb516b 100644
--- a/xlators/cluster/ha/src/ha.c
+++ b/xlators/cluster/ha/src/ha.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 /* generate errors randomly, code is simple now, better alogorithm
  * can be written to decide what error to be returned and when
  */
@@ -1876,13 +1866,9 @@ err:
         }
 
         if (hafdp) {
-                if (hafdp->fdstate) {
-                        GF_FREE (hafdp->fdstate);
-                }
+                GF_FREE (hafdp->fdstate);
 
-                if (hafdp->path) {
-                        GF_FREE (hafdp->path);
-                }
+                GF_FREE (hafdp->path);
 
                 GF_FREE (hafdp);
         }
diff --git a/xlators/cluster/ha/src/ha.h b/xlators/cluster/ha/src/ha.h
index 5fb7b2585..e2ed7eaa6 100644
--- a/xlators/cluster/ha/src/ha.h
+++ b/xlators/cluster/ha/src/ha.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef __HA_H_
 #define __HA_H_
 
diff --git a/xlators/cluster/map/src/Makefile.am b/xlators/cluster/map/src/Makefile.am
index 26e19137a..a278b05e2 100644
--- a/xlators/cluster/map/src/Makefile.am
+++ b/xlators/cluster/map/src/Makefile.am
@@ -1,15 +1,16 @@
 xlator_LTLIBRARIES = map.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/cluster
 
-map_la_LDFLAGS = -module -avoidversion 
+map_la_LDFLAGS = -module -avoid-version 
 
 map_la_SOURCES = map.c map-helper.c
 map_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = map.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	    -I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/cluster/map/src/map-helper.c b/xlators/cluster/map/src/map-helper.c
index 41a81b617..851397b68 100644
--- a/xlators/cluster/map/src/map-helper.c
+++ b/xlators/cluster/map/src/map-helper.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2009-2009 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2009-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
diff --git a/xlators/cluster/map/src/map-mem-types.h b/xlators/cluster/map/src/map-mem-types.h
index 10f0dc60d..3e89f4736 100644
--- a/xlators/cluster/map/src/map-mem-types.h
+++ b/xlators/cluster/map/src/map-mem-types.h
@@ -1,24 +1,13 @@
-
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __MAP_MEM_TYPES_H__
 #define __MAP_MEM_TYPES_H__
 
diff --git a/xlators/cluster/map/src/map.c b/xlators/cluster/map/src/map.c
index 62987fc2c..6150a33ce 100644
--- a/xlators/cluster/map/src/map.c
+++ b/xlators/cluster/map/src/map.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2009 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2010-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -2375,8 +2365,7 @@ fini (xlator_t *this)
 	priv = this->private;
 
 	if (priv) {
-		if (priv->xlarray)
-			GF_FREE (priv->xlarray);
+		GF_FREE (priv->xlarray);
 
 		trav_map = priv->map;
 		while (trav_map) {
diff --git a/xlators/cluster/map/src/map.h b/xlators/cluster/map/src/map.h
index a4543776b..7703a543e 100644
--- a/xlators/cluster/map/src/map.h
+++ b/xlators/cluster/map/src/map.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __MAP_H__
 #define __MAP_H__
 
diff --git a/xlators/cluster/unify/Makefile.am b/xlators/cluster/nsr-client/Makefile.am
index d471a3f92..d471a3f92 100644
--- a/xlators/cluster/unify/Makefile.am
+++ b/xlators/cluster/nsr-client/Makefile.am
diff --git a/xlators/cluster/nsr-client/src/Makefile.am b/xlators/cluster/nsr-client/src/Makefile.am
new file mode 100644
index 000000000..4541ea01a
--- /dev/null
+++ b/xlators/cluster/nsr-client/src/Makefile.am
@@ -0,0 +1,33 @@
+noinst_PYTHON = gen-fops.py
+
+xlator_LTLIBRARIES = nsrc.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
+
+nsrc_la_LDFLAGS = -module -avoid-version
+nsrc_la_SOURCES = nsrc.c
+
+nsrc_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = fop-template.c \
+		 $(top_srcdir)/xlators/lib/src/libxlator.h \
+		 $(top_srcdir)/glusterfsd/src/glusterfsd.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+XLATOR_HEADER	= $(top_srcdir)/libglusterfs/src/xlator.h
+
+CLEANFILES = nsrc-cg.c
+
+CODEGEN_DIR = ../../nsr-server/src/codegen.py
+
+nsrc-cg.c: gen-fops.py $(CODEGEN) $(XLATOR_HEADER) fop-template.c
+	$(PYTHON) ./gen-fops.py $(XLATOR_HEADER) fop-template.c > $@
+
+nsrc.lo: nsrc-cg.c
+
+uninstall-local:
+	rm -f $(DESTDIR)$(xlatordir)/nsr.so
diff --git a/xlators/cluster/nsr-client/src/fop-template.c b/xlators/cluster/nsr-client/src/fop-template.c
new file mode 100644
index 000000000..699b07d40
--- /dev/null
+++ b/xlators/cluster/nsr-client/src/fop-template.c
@@ -0,0 +1,113 @@
+// template-name fop
+$TYPE$
+nsrc_$NAME$ (call_frame_t *frame, xlator_t *this,
+             $ARGS_LONG$)
+{
+        nsrc_local_t    *local          = NULL;
+        xlator_t        *target_xl      = ACTIVE_CHILD(this);
+
+        local = mem_get(this->local_pool);
+        if (!local) {
+                goto err;
+        }
+
+        local->stub = fop_$NAME$_stub (frame, nsrc_$NAME$_continue,
+                                       $ARGS_SHORT$);
+        if (!local->stub) {
+                goto err;
+        }
+        local->curr_xl = target_xl;
+        local->scars = 0;
+
+        frame->local = local;
+        STACK_WIND_COOKIE (frame, nsrc_$NAME$_cbk, target_xl,
+                    target_xl, target_xl->fops->$NAME$,
+                    $ARGS_SHORT$);
+        return 0;
+
+err:
+        if (local) {
+                mem_put(local);
+        }
+        STACK_UNWIND_STRICT ($NAME$, frame, -1, ENOMEM,
+                             $DEFAULTS$);
+        return 0;
+}
+
+// template-name cbk
+$TYPE$
+nsrc_$NAME$_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 $ARGS_LONG$)
+{
+        nsrc_local_t    *local          = frame->local;
+        xlator_t        *last_xl        = cookie;
+        xlator_t        *next_xl;
+        nsrc_private_t  *priv           = this->private;
+        struct timespec spec;
+
+        if (op_ret != (-1)) {
+                if (local->scars) {
+                        gf_log (this->name, GF_LOG_INFO,
+                                HILITE("retried %p OK"), frame->local);
+                }
+                priv->active = last_xl;
+                goto unwind;
+        }
+        if ((op_errno != EREMOTE) && (op_errno != ENOTCONN)) {
+                goto unwind;
+        }
+
+        /* TBD: get leader ID from xdata? */
+        next_xl = next_xlator(this,last_xl);
+        /*
+         * We can't just give up after we've tried all bricks, because it's
+         * quite likely that a new leader election just hasn't finished yet.
+         * We also shouldn't retry endlessly, and especially not at a high
+         * rate, but that's good enough while we work on other things.
+         * 
+         * TBD: implement slow/finite retry via a worker thread
+         */
+        if (!next_xl || (local->scars >= SCAR_LIMIT)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        HILITE("ran out of retries for %p"), frame->local);
+                goto unwind;
+        }
+
+        local->curr_xl = next_xl;
+        local->scars += 1;
+        spec.tv_sec = 1;
+        spec.tv_nsec = 0;
+        /*
+         * WARNING
+         *
+         * Just calling gf_timer_call_after like this leaves open the
+         * possibility that writes will get reordered, if a first write is
+         * rescheduled and then a second comes along to find an updated
+         * priv->active before the first actually executes.  We might need to
+         * implement a stricter (and more complicated) queuing mechanism to
+         * ensure absolute consistency in this case.
+         */
+        if (gf_timer_call_after(this->ctx,spec,nsrc_retry_cb,local)) {
+                return 0;
+        }
+
+unwind:
+        call_stub_destroy(local->stub);
+        STACK_UNWIND_STRICT ($NAME$, frame, op_ret, op_errno,
+                                     $ARGS_SHORT$);
+        return 0;
+}
+
+// template-name cont-func
+$TYPE$
+nsrc_$NAME$_continue (call_frame_t *frame, xlator_t *this,
+                      $ARGS_LONG$)
+{
+        nsrc_local_t    *local  = frame->local;
+
+        STACK_WIND_COOKIE (frame, nsrc_$NAME$_cbk, local->curr_xl,
+                           local->curr_xl, local->curr_xl->fops->$NAME$,
+                           $ARGS_SHORT$);
+        return 0;
+}
diff --git a/xlators/cluster/nsr-client/src/gen-fops.py b/xlators/cluster/nsr-client/src/gen-fops.py
new file mode 100755
index 000000000..b07b3c5b1
--- /dev/null
+++ b/xlators/cluster/nsr-client/src/gen-fops.py
@@ -0,0 +1,57 @@
+#!/usr/bin/python
+
+# This script generates the boilerplate versions of most fops in the client,
+# mostly so that we can use STACK_WIND instead of STACK_WIND_TAIL (see
+# fop-template.c for the details).  The problem we're solving is that we sit
+# under DHT, which makes assumptions about getting callbacks only from its
+# direct children.  If we didn't define our own versions of these fops, the
+# default versions would use STACK_WIND_TAIL and the callbacks would come from
+# DHT's grandchildren.  The code-generation approach allows us to handle this
+# with a minimum of code, and also keep up with any changes to the fop table.
+
+import sys
+sys.path.append("../../nsr-server/src") # Blech.
+import codegen
+
+type_re = "([a-z_0-9]+)"
+name_re = "\(\*fop_([a-z0-9]+)_t\)"
+full_re = type_re + " *" + name_re
+fop_cg = codegen.CodeGenerator()
+fop_cg.skip = 2
+fop_cg.parse_decls(sys.argv[1],full_re)
+fop_cg.load_templates(sys.argv[2])
+
+# Use the multi-template feature to generate multiple callbacks from the same
+# parsed declarations.
+type_re = "([a-z_0-9]+)"
+name_re = "\(\*fop_([a-z0-9]+)_cbk_t\)"
+full_re = type_re + " *" + name_re
+cbk_cg = codegen.CodeGenerator()
+cbk_cg.skip = 5
+cbk_cg.parse_decls(sys.argv[1],full_re)
+cbk_cg.load_templates(sys.argv[2])
+
+# This is a nasty little trick to handle the case where a generated fop needs
+# a set of default arguments for the corresponding callback.
+#
+# Yes, it's ironic that I'm copying and pasting the generator code.
+fop_cg.make_defaults = cbk_cg.make_defaults
+
+# Sorry, getspec, you're not a real fop until someone writes a stub function
+# for you.
+del fop_cg.decls["getspec"]
+del cbk_cg.decls["getspec"]
+
+# cbk is used by both fop and continue, so emit first
+for f_name in cbk_cg.decls.keys():
+	cbk_cg.emit(f_name,"cbk")
+	print("")
+
+# continue is used by fop, so emit next
+for f_name in fop_cg.decls.keys():
+	fop_cg.emit(f_name,"cont-func")
+	print("")
+
+for f_name in fop_cg.decls.keys():
+	fop_cg.emit(f_name,"fop")
+	print("")
diff --git a/xlators/cluster/nsr-client/src/nsrc.c b/xlators/cluster/nsr-client/src/nsrc.c
new file mode 100644
index 000000000..4551a1432
--- /dev/null
+++ b/xlators/cluster/nsr-client/src/nsrc.c
@@ -0,0 +1,243 @@
+/*
+  Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "call-stub.h"
+#include "defaults.h"
+#include "timer.h"
+#include "xlator.h"
+
+#define SCAR_LIMIT      20
+#define HILITE(x)       ("[1;33m"x"[0m")
+
+/*
+ * The fops are actually generated by gen-fops.py; the rest was mostly copied
+ * from defaults.c (commit cd253754 on 27 August 2013).
+ */
+
+enum gf_dht_mem_types_ {
+        gf_mt_nsrc_private_t = gf_common_mt_end + 1,
+        gf_mt_nsrc_end
+};
+
+typedef struct {
+        xlator_t        *active;
+} nsrc_private_t;
+
+typedef struct {
+        call_stub_t     *stub;
+        xlator_t        *curr_xl;
+        uint16_t        scars;
+} nsrc_local_t;
+
+char            *NSRC_XATTR     = "user.nsr.active";
+
+static inline
+xlator_t *
+ACTIVE_CHILD (xlator_t *parent)
+{
+        nsrc_private_t  *priv   = parent->private;
+
+        return priv ? priv->active : FIRST_CHILD(parent);
+}
+
+xlator_t *
+next_xlator (xlator_t *this, xlator_t *prev)
+{
+        xlator_list_t   *trav;
+
+        for (trav = this->children; trav; trav = trav->next) {
+                if (trav->xlator == prev) {
+                        return trav->next ? trav->next->xlator
+                                          : this->children->xlator;
+                }
+        }
+
+        return NULL;
+}
+
+void
+nsrc_retry_cb (void *cb_arg)
+{
+        nsrc_local_t    *local  = cb_arg;
+
+        gf_log (__func__, GF_LOG_INFO, HILITE("retrying %p"), local);
+        call_resume_wind(local->stub);
+}
+
+#include "nsrc-cg.c"
+
+int32_t
+nsrc_forget (xlator_t *this, inode_t *inode)
+{
+        gf_log_callingfn (this->name, GF_LOG_WARNING, "xlator does not "
+                          "implement forget_cbk");
+        return 0;
+}
+
+
+int32_t
+nsrc_releasedir (xlator_t *this, fd_t *fd)
+{
+        gf_log_callingfn (this->name, GF_LOG_WARNING, "xlator does not "
+                          "implement releasedir_cbk");
+        return 0;
+}
+
+int32_t
+nsrc_release (xlator_t *this, fd_t *fd)
+{
+        gf_log_callingfn (this->name, GF_LOG_WARNING, "xlator does not "
+                          "implement release_cbk");
+        return 0;
+}
+
+struct xlator_fops fops = {
+        .lookup         = nsrc_lookup,
+        .stat           = nsrc_stat,
+        .fstat          = nsrc_fstat,
+        .truncate       = nsrc_truncate,
+        .ftruncate      = nsrc_ftruncate,
+        .access         = nsrc_access,
+        .readlink       = nsrc_readlink,
+        .mknod          = nsrc_mknod,
+        .mkdir          = nsrc_mkdir,
+        .unlink         = nsrc_unlink,
+        .rmdir          = nsrc_rmdir,
+        .symlink        = nsrc_symlink,
+        .rename         = nsrc_rename,
+        .link           = nsrc_link,
+        .create         = nsrc_create,
+        .open           = nsrc_open,
+        .readv          = nsrc_readv,
+        .writev         = nsrc_writev,
+        .flush          = nsrc_flush,
+        .fsync          = nsrc_fsync,
+        .opendir        = nsrc_opendir,
+        .readdir        = nsrc_readdir,
+        .readdirp       = nsrc_readdirp,
+        .fsyncdir       = nsrc_fsyncdir,
+        .statfs         = nsrc_statfs,
+        .setxattr       = nsrc_setxattr,
+        .getxattr       = nsrc_getxattr,
+        .fsetxattr      = nsrc_fsetxattr,
+        .fgetxattr      = nsrc_fgetxattr,
+        .removexattr    = nsrc_removexattr,
+        .fremovexattr   = nsrc_fremovexattr,
+        .lk             = nsrc_lk,
+        .inodelk        = nsrc_inodelk,
+        .finodelk       = nsrc_finodelk,
+        .entrylk        = nsrc_entrylk,
+        .fentrylk       = nsrc_fentrylk,
+        .rchecksum      = nsrc_rchecksum,
+        .xattrop        = nsrc_xattrop,
+        .fxattrop       = nsrc_fxattrop,
+        .setattr        = nsrc_setattr,
+        .fsetattr       = nsrc_fsetattr,
+	.fallocate	= nsrc_fallocate,
+	.discard        = nsrc_discard,
+};
+
+struct xlator_cbks cbks = {
+};
+
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("nsrc", this, out);
+
+        ret = xlator_mem_acct_init (this, gf_mt_nsrc_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Memory accounting init" "failed");
+                return ret;
+        }
+out:
+        return ret;
+}
+
+
+int32_t
+nsrc_init (xlator_t *this)
+{
+        nsrc_private_t  *priv   = NULL;
+
+        this->local_pool = mem_pool_new (nsrc_local_t, 128);
+        if (!this->local_pool) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create nsrc_local_t pool");
+                goto err;
+        }
+
+        priv = GF_CALLOC (1, sizeof (*priv), gf_mt_nsrc_private_t);
+        if (!priv) {
+               goto err;
+        }
+
+        priv->active = FIRST_CHILD(this);
+        this->private = priv;
+        return 0;
+
+err:
+        if (priv) {
+                GF_FREE(priv);
+        }
+        return -1;
+}
+
+void
+nsrc_fini (xlator_t *this)
+{
+        GF_FREE(this->private);
+}
+
+int32_t
+nsrc_notify (xlator_t *this, int32_t event, void *data, ...)
+{
+        int32_t         ret     = 0;
+
+        switch (event) {
+        case GF_EVENT_CHILD_DOWN:
+                /*
+                 * TBD: handle this properly
+                 *
+                 * What we really should do is propagate this only if it caused
+                 * us to lose quorum, and likewise for GF_EVENT_CHILD_UP only
+                 * if it caused us to gain quorum.  However, that requires
+                 * tracking child states and for now it's easier to swallow
+                 * these unconditionally.  The consequence of failing to do
+                 * this is that DHT sees the first GF_EVENT_CHILD_DOWN and gets
+                 * confused, so it doesn't call us and doesn't get up-to-date
+                 * directory listings etc.
+                 */
+                break;
+        default:
+                ret = default_notify (this, event, data);
+        }
+
+        return ret;
+}
+
+class_methods_t class_methods = {
+        .init           = nsrc_init,
+        .fini           = nsrc_fini,
+        .notify         = nsrc_notify,
+};
+
+struct volume_options options[] = {
+	{ .key = {NULL} },
+};
diff --git a/xlators/protocol/legacy/client/Makefile.am b/xlators/cluster/nsr-recon/Makefile.am
index d471a3f92..d471a3f92 100644
--- a/xlators/protocol/legacy/client/Makefile.am
+++ b/xlators/cluster/nsr-recon/Makefile.am
diff --git a/xlators/cluster/nsr-recon/src/Makefile.am b/xlators/cluster/nsr-recon/src/Makefile.am
new file mode 100644
index 000000000..e639e4437
--- /dev/null
+++ b/xlators/cluster/nsr-recon/src/Makefile.am
@@ -0,0 +1,23 @@
+xlator_LTLIBRARIES = nsr_recon.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
+
+nsr_recon_la_LDFLAGS = -module -avoid-version
+nsr_recon_la_SOURCES = recon_driver.c recon_xlator.c
+
+nsr_recon_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+				$(top_builddir)/api/src/libgfapi.la
+
+noinst_HEADERS = recon_driver.h recon_xlator.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+XLATOR_HEADER	= $(top_srcdir)/libglusterfs/src/xlator.h
+
+CLEANFILES =
+
+uninstall-local:
+	rm -f $(DESTDIR)$(xlatordir)/nsr.so
diff --git a/xlators/cluster/nsr-recon/src/recon_driver.c b/xlators/cluster/nsr-recon/src/recon_driver.c
new file mode 100644
index 000000000..8c7622a02
--- /dev/null
+++ b/xlators/cluster/nsr-recon/src/recon_driver.c
@@ -0,0 +1,3130 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <sys/types.h>
+#include <fcntl.h>
+#include <string.h>
+#include <unistd.h>
+#include <fnmatch.h>
+
+
+#include "call-stub.h"
+#include "defaults.h"
+#include "xlator.h"
+
+
+#include "recon_driver.h"
+#include "recon_xlator.h"
+#include "api/src/glfs-internal.h"
+#include "api/src/glfs-handles.h"
+
+/* TBD: move declarations here and nsr.c into a common place */
+#define NSR_TERM_XATTR          "trusted.nsr.term"
+#define RECON_TERM_XATTR        "trusted.nsr.recon-term"
+#define RECON_INDEX_XATTR       "trusted.nsr.recon-index"
+
+/*
+ * Execution architecture for the NSR reconciliation driver.  The driver runs
+ * as a seperate process in each node where the brick is.  The main function of
+ * the driver is nsr_reconciliation_driver() (last function below) The driver
+ * just sits in a tight loop waiting for state changes. When a brick becomes a
+ * replica leader, it fences IO, contacts this process and waits for
+ * reconciliation to finish.
+ *
+ * The replica leader talks to other bricks in replica group which are alive
+ * and gets the last term info using which it decides which has the latest
+ * data.  That brick is referred to as the "reconciliator"; leader sends a
+ * message to reconciliator to freeze its data (by reading any incomplete data
+ * from other nodes from that term if required)
+ *
+ * Once that is done leader sends a message to all nodes except the
+ * reconciliator to sync themselves with the reconciliator. This process is
+ * referred to as "resolution".
+ *
+ * Hence the reconciliation processes need to talk to each other to get a given
+ * term info.  This is implemented using the recon translator IOs which
+ * implements a bare bone RPC by exposing a file interface to which
+ * reads/writes are done to pass control messages.  This is referred to as the
+ * "control plane". This implementation allows the control plane to be
+ * implemented as a bunch of threads for each of the nodes.
+ *
+ * The reconciliation process also needs to talk to the brick process on that
+ * node to actually write the data as part of reconciliation/resolution. This
+ * is referred to as the "data plane". Again there are a bunch of threads that
+ * do this work.
+ *
+ * The way the worker threads are organised is that main driver context has a
+ * pointer to contexts for each of these thread contexts. The thread context at
+ * index 0 always refers to talking with local recon process/brick. So the
+ * control worker at index 0 will get the local changelog info and data worker
+ * at index 0 will talk to local brick.
+ *
+ * All the ops from the control/data planes are implemented using the glfs
+ * APIs.
+ */
+
+#if defined(NSR_DEBUG)
+
+/* This lets us change on the fly even if NSR_DEBUG is defined. */
+int nsr_debug_level = GF_LOG_TRACE;
+
+FILE *
+recon_create_log (char *member, char *module)
+{
+        char    *dpath  = NULL;
+        char    *p;
+        char    *fpath  = NULL;
+        FILE    *fp     = NULL;
+        int     fd      = -1;
+
+        (void)mkdir(NSR_LOG_DIR,0777);
+        (void)asprintf(&dpath,NSR_LOG_DIR"/%s",member);
+        if (dpath) {
+                for (p = dpath + strlen(NSR_LOG_DIR) + 1; *p; ++p) {
+                        if (*p == '/') {
+                                *p = '-';
+                        }
+                }
+                (void)mkdir(dpath,0777);
+                (void)asprintf(&fpath,"%s/%s",dpath,module);
+                if (fpath) {
+                        fd = open(fpath,O_WRONLY|O_CREAT|O_APPEND|O_SYNC,0666);
+                        if (fd >= 0) {
+                                fp = fdopen(fd,"a");
+                                if (!fp) {
+                                        close(fd);
+                                }
+                        }
+                        if (fp) {
+                                if (setvbuf (fp, NULL, _IONBF, 0)) {
+                                        /*
+                                         * Might as well take advantage of it
+                                         * to log the error.
+                                         */
+                                        fprintf (fp,
+                                                 "setvbuf failed for log\n");
+                                        fprintf (fp,
+                                                 "log output may be async\n");
+                                        fflush(fp);
+                                }
+                        }
+                        free(fpath);
+                }
+                free(dpath);
+        }
+
+        return fp;
+}
+
+void
+_nsr_driver_log (const char *func, int line, char *member, FILE *fp,
+                 char *fmt, ...)
+{
+        va_list         ap;
+        char            *buf    = NULL;
+        int             retval;
+
+        if (!fp) {
+                fp = recon_create_log(member,"nsr-driver-log");
+                if (!fp) {
+                        return;
+                }
+        }
+
+        va_start(ap,fmt);
+        retval = vasprintf(&buf,fmt,ap);
+        if (buf) {
+                fprintf(fp,"[%s:%d] %.*s\n",func,line,retval,buf);
+                free(buf);
+        }
+        va_end(ap);
+}
+
+void
+_nsr_worker_log (const char *func, int line, char *member,
+                 char *type, uint32_t index, FILE *fp,
+                 char *fmt, ...)
+{
+        va_list         ap;
+        char            *buf    = NULL;
+        int             retval;
+
+        if (!fp) {
+                char *name;
+                if (asprintf(&name,"%s-%u",type,index) < 1) {
+                        return;
+                }
+                fp = recon_create_log (member, name);
+                if (!fp) {
+                        return;
+                }
+        }
+
+        va_start(ap,fmt);
+        retval = vasprintf(&buf,fmt,ap);
+        if (buf) {
+                fprintf(fp,"[%s:%d] %.*s\n",func,line,retval,buf);
+                free(buf);
+        }
+        va_end(ap);
+}
+
+#endif
+
+/*
+ * Recon Driver Calloc
+ *
+ * We need this because all of this messing about with gfapi from within a
+ * translator keeps scrambling THIS (only one reason it's a terrible idea) and
+ * we need THIS to have a value that represents our initialization with our
+ * memory types.
+ *
+ * Note that the macro requires "this" to be defined in the current scope.
+ */
+
+#define RD_CALLOC(x,y,z) ({THIS = this; GF_CALLOC(x,y,z); })
+
+/*
+ * This function gets the size of all the extended attributes for a file.
+ * This is used so that caller knows how much to allocate for key-value storage.
+ *
+ * Input Arguments:
+ * fd - the file opened using glfs API.
+ * dict - passed so that NSR translator can get this from the required brick
+ *
+ * Output Arguments:
+ * b - pointer to the buffer where the attributes are filled up.
+ * key_size - the size of all keys
+ * val_size - the size of all values
+ * num - number of key/values
+ */
+static int32_t
+get_xattr_total_size( struct glfs_fd *fd,
+                     char **b,
+                     uint32_t *key_size,
+                     uint32_t *val_size,
+                     uint32_t* num,
+                     dict_t *dict)
+{
+        int32_t s = -1, ret = -1;
+        char *c = NULL;
+
+        *key_size = 0;
+        *val_size = 0;
+        *num = 0;
+
+        // First get the size of the keys
+        s = glfs_flistxattr_with_xdata(fd, NULL,0, dict);
+        if (s == -1) {
+                goto out;
+        }
+        *key_size = s;
+
+        // TBD - use the regular calloc
+        (*b) = c = calloc(s+1,1);
+
+        // get the keys themselves
+        if (glfs_flistxattr_with_xdata(fd, c, s+1, dict) == -1) {
+                goto out;
+        }
+        do {
+                int32_t r;
+                uint32_t len = 0;
+                // for each key get the size of the value
+                r = glfs_fgetxattr_with_xdata(fd, c, NULL, 0, dict);
+                if (r == -1)
+                        goto out;
+                (*val_size) += r;
+                len = strlen(c) + 1;
+                c += len;
+                s -= len;
+                (*num)++;
+        } while(s);
+        ret = 0;
+out:
+        return ret;
+}
+
+/*
+ * This function gets bunch of xattr values given set of keys.
+ *
+ * Input Arguments:
+ * fd - the file opened using glfs API.
+ * keys - the bunch of keys
+ * size - size of values
+ * num - number of keys
+ * dict - passed so that NSR translator can get this from the required brick
+ *
+ * Output Arguments:
+ * buf - where the values are written one after the other (NULL seperated)
+ */
+static int32_t
+get_xattr(struct glfs_fd *fd,
+          char *keys,
+          char *buf,
+          uint32_t size,
+          uint32_t num,
+          dict_t *dict)
+{
+        while(num--) {
+                int32_t r;
+                uint32_t len = 0;
+
+                // copy the key
+                strcpy(buf, keys);
+                len = strlen(keys);
+                len++;
+                buf += len;
+
+                // get the value and copy the value after incrementing buf after the key
+                r = glfs_fgetxattr_with_xdata(fd, keys, buf, size, dict);
+
+                // TBD - handle error
+                if (r == -1)
+                        return -1;
+
+                // increment the key to next value
+                keys += len;
+
+                // increment buf to hold the next key
+                buf += strlen(buf) + 1;
+        }
+        return 0;
+}
+
+/*
+ * Function deletes a bunch of key values in extended attributes of a file.
+ * Input Arguments:
+ * fd - the file opened using glfs API.
+ * dict - passed so that NSR translator can do this from the required brick
+ * keys - bunch of NULL seperated key names
+ * num - number of keys
+ */
+static int32_t delete_xattr(struct glfs_fd *fd,
+                         dict_t *dict_t,
+                         char *keys,
+                         uint32_t num)
+{
+        while(num--) {
+                // get the value and copy the value
+                // TBD - handle failure cases when calling glfs_fremovexattr_with_xdata()
+                if (glfs_fremovexattr_with_xdata(fd, keys, dict_t) == -1)
+			return -1;
+                keys += strlen(keys) +1;
+        }
+        return 0;
+}
+
+/*
+ * Given a bunch of key value pairs, fill them as xattrs for a file
+ *
+ * Input Arguments:
+ * fd - the file opened using glfs API.
+ * dict - passed so that NSR translator can do this from the required brick
+ * buf - buffer containing the keys-values pairs. The key value are NULL seperated.
+ *       Each of the key-value is seperated by NULL in turn.
+ * num - Number of such key value pairs.
+ */
+static int32_t
+fill_xattr(struct glfs_fd *fd,
+           dict_t *dict,
+           char *buf,
+           uint32_t num)
+{
+        char *k = buf, *val = NULL;
+
+        while(num--) {
+                int32_t r;
+
+                val = k + strlen(k) + 1;
+
+                // TBD - handle failure cases when calling glfs_fsetxattr_with_xdata()
+                r = glfs_fsetxattr_with_xdata(fd, k, val, strlen(val), 0, dict);
+                if (r == -1)
+                        return -1;
+                k = val + strlen(val) + 1;
+        }
+        return 0;
+}
+
+/*
+ * This function gets a file that can be used for doing glfs_init later.
+ * The control file is used by control thread(function) to talk to peer reconciliation process.
+ * The data file is used by the data thread(function) to talk to the bricks.
+ * The control file is of name such as con:gfs1:-mnt-a1 where "gfs1" is name of host
+ * and the brick path is "/mnt/a1". 
+ * The data file is of name such as data:gfs1:-mnt-a1.
+ *
+ * Input Arguments:
+ * vol - name of the volume. This is used to build the full path of the control and data file
+ * such as /var/lib/glusterd/vols/test/bricks/gfs2:-mnt-test1-nsr-recon.vol.
+ * In above example the volume name is test and brick on gfs2 is on path /mnt/test1
+ *
+ * worker - The worker for a given node. This worker has 2 threads - one on the data plane
+ * and one on the control plane. The worker->name is already filled with hostname:brickname
+ * in the function nsr_reconciliation_driver(). Use that to build the volume file.
+ * So if worker->name has  gfs1:/mnt/a1, control file is con:gfs1:-mnt-a1 
+ * and data file is data:gfs1:-mnt-a1.
+ * All these files are under the bricks directory. TBD - move this to a NSR recon directory later.
+ */
+static void
+nsr_recon_get_file(char *vol, nsr_replica_worker_t *worker)
+{
+        char *ptr;
+        char tr[256];
+
+        // Replace the "/" to -
+        strcpy(tr, worker->name);
+        ptr = strchr (tr, '/');
+        while (ptr) {
+                *ptr = '-';
+                ptr = strchr (tr, '/');
+        }
+
+        // Build the base directory such as "/var/lib/glusterd/vols/test/bricks/"
+        sprintf(worker->control_worker->vol_file,
+                "/%s/%s/%s/%s/",
+                GLUSTERD_DEFAULT_WORKDIR,
+                GLUSTERD_VOLUME_DIR_PREFIX,
+                vol,
+                GLUSTERD_BRICK_INFO_DIR);
+
+        strcat(worker->control_worker->vol_file, "con:");
+        strcat(worker->control_worker->vol_file, tr);
+
+        sprintf(worker->data_worker->vol_file,
+                "/%s/%s/%s/%s/",
+                GLUSTERD_DEFAULT_WORKDIR,
+                GLUSTERD_VOLUME_DIR_PREFIX,
+                vol,
+                GLUSTERD_BRICK_INFO_DIR);
+        strcat(worker->data_worker->vol_file, "data:");
+        strcat(worker->data_worker->vol_file, tr);
+}
+
+/*
+ * This function does all the glfs initialisation 
+ * so that reconciliation process can talk to other recon processes/bricks
+ * for the control/data messages.
+ * This will be done everytime a worker needs to be kicked off to talk
+ * across any plane.
+ *
+ * Input arguments:
+ * ctx - The per worker based context
+ * control - set to true if this worker is for the control plane
+ */
+static int32_t
+nsr_recon_start_work(nsr_per_node_worker_t *ctx,
+           gf_boolean_t control)
+{
+        glfs_t    *fs = NULL;
+        xlator_t *this = ctx->driver_ctx->this;
+        int32_t ret = 0;
+        glfs_fd_t *aux_fd = NULL; // fd of auxilary log
+        char lf[256];
+        nsr_recon_private_t *priv = NULL;
+        char *my_name = NULL;
+        char *morph_name = NULL, *ptr = NULL;
+
+        priv = this->private;
+        my_name = RD_CALLOC (1,
+                             strlen (priv->replica_group_members[0]) + 1,
+                             gf_mt_recon_member_name_t);
+        strcpy (my_name, priv->replica_group_members[0]);
+
+        nsr_worker_log(this->name, GF_LOG_INFO,
+                       "starting work with volfile %s\n",
+                       ctx->vol_file);
+
+        fs = glfs_new(ctx->id);
+        if (!fs) {
+                glusterfs_this_set(this);
+                nsr_worker_log(this->name, GF_LOG_ERROR,
+                       "cannot create gfls context for thread %s\n",ctx->id);
+                return -1;
+        }
+
+        // For some vague reason, glfs init APIs seem to be clobbering "this". hence resetting it.
+        glusterfs_this_set(this);
+        nsr_worker_log(this->name, GF_LOG_INFO,
+                       "init done. setting volfile %s\n",
+                       ctx->vol_file);
+
+        ret = glfs_set_volfile(fs, ctx->vol_file);
+        if (ret != 0) {
+                glusterfs_this_set(this);
+                nsr_worker_log(this->name, GF_LOG_ERROR,
+                       "cannot set volfile %s for thread %s\n",ctx->vol_file, ctx->id);
+                return -1;
+        }
+
+        morph_name = RD_CALLOC (1, strlen (my_name) + 1,
+                                gf_mt_recon_member_name_t);
+        strcpy (morph_name, my_name);
+
+        ptr = strchr (morph_name, '/');
+        while (ptr)
+        {
+                *ptr = '-';
+                 ptr = strchr (morph_name, '/');
+        }
+        // TBD - convert this to right /usr/local/var/log based log files.
+
+        sprintf(lf, NSR_LOG_DIR"/%s/%s-%"PRIu32, morph_name,
+                (control == _gf_true)?"glfs-con":"glfs-data", ctx->index);
+        ret =  glfs_set_logging (fs, lf, 7);
+        if (ret) {
+                glusterfs_this_set(this);
+                gf_log (this->name, GF_LOG_ERROR, "glfs logging set failed (%s)",
+                        strerror (errno));
+                return -1;
+        }
+
+        ret = glfs_init (fs);
+        if (ret != 0) {
+                glusterfs_this_set(this);
+                nsr_worker_log(this->name, GF_LOG_ERROR, "cannot do init for thread %s with volfile %s\n",ctx->id, ctx->vol_file);
+                return -1;
+        }
+        glusterfs_this_set(this);
+        nsr_worker_log(this->name, GF_LOG_INFO,
+                       "setting volfile %s done\n",
+                       ctx->vol_file);
+
+        // If it is control thread, open the "/" as the aux_fd.
+        // All IOs happening via the fd will do the RPCs across the reconciliation
+        // processes. For some vague reason, the root seems to be open'able like a file.
+        // TBD - try to clean this up. (implement a virtual file???)
+        if (control == _gf_true) {
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "doing open for / \n");
+                aux_fd = glfs_open (fs, "/", O_RDWR);
+                // TBD - proper error handling. Stall reconciliation if such a thing happens?
+                if (aux_fd == NULL) {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                               "cannot open  aux log file for thread %s\n",ctx->id);
+			return -1;
+                } else {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                               "---opened aux log file for thread %s\n",ctx->id);
+		}
+                ctx->aux_fd = aux_fd;
+        }
+        glusterfs_this_set(this);
+        ctx->fs = fs;
+        return 0;
+}
+
+/*
+ *
+ * This function does the cleanup after reconciliation is done
+ * or before we start a new reconciliation.
+ *
+ * Input arguments:
+ * ctx - The per worker based context
+ * control - set to true if this worker is for the control plane
+ */
+static int32_t
+nsr_recon_end_work(nsr_per_node_worker_t *ctx,
+         gf_boolean_t control)
+{
+        int32_t ret = 0;
+        xlator_t *this = ctx->driver_ctx->this;
+
+        nsr_worker_log(this->name, GF_LOG_INFO,
+                       "doing fini for recon worker\n");
+
+        ret = glfs_fini(ctx->fs);
+        if (ret != 0) {
+                glusterfs_this_set(this);
+                nsr_worker_log(this->name, GF_LOG_ERROR, "cannot do fini for thread %s with volfile %s\n",ctx->id, ctx->vol_file);
+                return -1;
+        }
+        glusterfs_this_set(this);
+        ctx->fs = NULL;
+        if (control == _gf_true) {
+                glfs_close (ctx->aux_fd);
+                ctx->aux_fd = NULL;
+        }
+        return 0;
+}
+
+//called in case all worker functions run as sepeerate threads
+static void
+init_worker(nsr_per_node_worker_t *ctx, gf_boolean_t control)
+{
+        pthread_mutex_init(&(ctx->mutex), NULL);
+        pthread_cond_init(&(ctx->cv), NULL);
+        INIT_LIST_HEAD(&(ctx->head.list));
+}
+
+
+/*
+ * Control worker funct for getting changelog info on this node.
+ * calls directly functions to parse the changelog.
+ *
+ * Input arguments:
+ * ctx - The per worker based context
+ * control - set to true if this worker is for the control plane
+ */
+static void
+control_worker_func_0(nsr_per_node_worker_t *ctx,
+                      nsr_recon_work_t *work)
+{
+        unsigned int index = ctx->index;
+        nsr_replica_worker_t *rw = &(ctx->driver_ctx->workers[index]);
+        nsr_recon_driver_ctx_t *dr = ctx->driver_ctx;
+        xlator_t *this = dr->this;
+        nsr_recon_private_t *priv = this->private;
+
+        ctx->is_control = _gf_true;
+
+        switch (work->req_id) {
+        case NSR_WORK_ID_INI:
+            {
+                break;
+            }
+        case NSR_WORK_ID_FINI:
+            {
+                break;
+            }
+        case NSR_WORK_ID_GET_LAST_TERM_INFO:
+            {
+                nsr_recon_last_term_info_t lt;
+                nsr_reconciliator_info_t *recon_info = rw->recon_info;
+                // term is stuffed inside work->index. overloading.
+                int32_t term = work->index;
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "trying to get last term info for  node %d with current term %d\n",index, term);
+
+                // TBD - handle errors
+                // This is called by the leader after it gets the current term.
+                // Makes searching easier.
+                nsr_recon_libchangelog_get_last_term_info(this, priv->changelog_base_path, term, &lt);
+                recon_info->last_term = lt.last_term;
+                recon_info->commited_ops = lt.commited_ops;
+                recon_info->last_index = lt.last_index;
+                recon_info->first_index = lt.first_index;
+
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "out of get last term info with current term %d. got ops %d with first %d and last %d \n",
+                               recon_info->last_term, recon_info->commited_ops,
+                               recon_info->first_index, recon_info->last_index);
+                break;
+            }
+        case NSR_WORK_ID_GET_GIVEN_TERM_INFO:
+            {
+                nsr_recon_last_term_info_t lt;
+                nsr_reconciliator_info_t *recon_info = rw->recon_info;
+                // term is stuffed inside work->index. overloading.
+                int32_t term = work->index;
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "trying to get term info for  node %d for term %d\n",index, term);
+
+                // TBD - handle errors
+                nsr_recon_libchangelog_get_this_term_info(this,priv->changelog_base_path, term, &lt);
+
+                recon_info->last_term = lt.last_term;
+                recon_info->commited_ops = lt.commited_ops;
+                recon_info->last_index = lt.last_index;
+                recon_info->first_index = lt.first_index;
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "out of get term info for term %d. got ops %d with first %d and last %d \n",
+                               recon_info->last_term, recon_info->commited_ops,
+                               recon_info->first_index, recon_info->last_index);
+
+                break;
+            }
+        case NSR_WORK_ID_RECONCILIATOR_DO_WORK:
+            {
+                // For local resolution, the main driver thread does it.
+                // SO there is no way we can have this message for this node.
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "this message should not be sent \n");
+                break;
+            }
+        case NSR_WORK_ID_RESOLUTION_DO_WORK:
+            {
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "this message should not be sent \n");
+                ctx->result = -1;
+                break;
+            }
+        case NSR_WORK_ID_GET_RECONCILATION_WINDOW:
+            {
+                nsr_reconciliator_info_t *recon_info = rw->recon_info;
+                // first_index and last_index at 0 indicates empty log.
+                // For non empty log, the first_index always starts at 1.
+                uint32_t num = (dr->workers[index].recon_info->last_index -
+                               dr->workers[index].recon_info->first_index + 1);
+                nsr_recon_record_details_t *rd;
+                uint32_t i=0;
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "trying to get reconciliation window records for node %d for term %d with first %d last %d\n",
+                               index, recon_info->last_term, recon_info->first_index, recon_info->last_index);
+
+
+                // TBD - handle buffer allocation errors
+                rd = RD_CALLOC(num,
+                               sizeof(nsr_recon_record_details_t),
+                               gf_mt_recon_record_details_t);
+                if (rd == NULL) {
+                        ctx->result = -1;
+                        return;
+                }
+
+                recon_info->records = RD_CALLOC(num,
+                               sizeof(nsr_reconciliation_record_t),
+                               gf_mt_recon_record_t);
+                if (recon_info->records == NULL) {
+                        ctx->result = -1;
+                        return;
+                }
+
+                // TBD - handle errors
+                if (nsr_recon_libchangelog_get_records(this, priv->changelog_base_path,
+                                                   recon_info->last_term,
+                                                   recon_info->first_index,
+                                                   recon_info->last_index,
+                                                   rd) == _gf_false) {
+                        ctx->result = -1;
+                        return;
+                }
+
+                // The above function writes into rd from 0 to (num -1)
+                // We need to take care of this whenever we deal with records
+                for (i=0; i < num; i++) {
+                        ENDIAN_CONVERSION_RD(rd[i], _gf_true); //ntohl
+                        memcpy(&(recon_info->records[i].rec),
+                        &(rd[i]),
+                        sizeof(nsr_recon_record_details_t));
+                }
+
+                GF_FREE(rd);
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "got reconciliation window records for node %d for term %d \n",
+                               index, recon_info->last_term);
+                break;
+            }
+
+            default:
+                nsr_worker_log (this->name, GF_LOG_ERROR,
+                                "bad req id %u", work->req_id);
+        }
+
+        return;
+}
+
+// Control worker thread
+static void*
+control_worker_main_0(nsr_per_node_worker_t *ctx)
+{
+
+        ctx->is_control = _gf_true;
+        nsr_worker_log(this->name, GF_LOG_INFO,
+                       "starting control worker func 0\n");
+
+        init_worker(ctx, 1);
+
+        while(1)
+        {
+                nsr_recon_work_t *work = NULL;
+                nsr_recon_driver_ctx_t *dr = ctx->driver_ctx;
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "waiting for work\n");
+
+                pthread_mutex_lock(&ctx->mutex);
+                while (list_empty(&(ctx->head.list))) {
+                        pthread_cond_wait(&ctx->cv, &ctx->mutex);
+                }
+                pthread_mutex_unlock(&ctx->mutex);
+
+
+                list_for_each_entry(work, &(ctx->head.list), list) {
+                        nsr_worker_log(this->name, GF_LOG_INFO,
+                                       "got work with id %d\n", work->req_id);
+			work->in_use = _gf_false;
+
+                        // Call the main function.
+                        control_worker_func_0(ctx, work);
+
+                        atomic_dec(&(dr->outstanding));
+                        break;
+                }
+
+                nsr_worker_log(this->name, GF_LOG_INFO,"deleting work item\n");
+		list_del_init (&work->list);
+                GF_FREE(work);
+                nsr_worker_log(this->name, GF_LOG_INFO,"finished deleting work item\n");
+        }
+
+        return NULL;
+}
+
+static void
+control_worker_do_reconciliation (nsr_per_node_worker_t *ctx,
+                                  nsr_recon_work_t *work)
+{
+        unsigned int index = ctx->index;
+        nsr_recon_driver_ctx_t *dr = ctx->driver_ctx;
+        nsr_recon_role_t rr;
+        uint32_t i=0;
+        uint32_t num=0;
+        uint32_t idx = dr->reconciliator_index;
+        uint32_t term = dr->workers[idx].recon_info->last_term;
+
+        GF_ASSERT(idx == index);
+
+        nsr_worker_log(this->name, GF_LOG_INFO,
+                       "trying to make this index %d as reconciliator for term %d\n", index, term);
+
+        // TBD - error handling for all the glfs APIs
+        if (glfs_lseek(ctx->aux_fd,
+                   nsr_recon_xlator_sector_1,
+                   SEEK_SET) == -1) {
+                ctx->result = -1;
+                return;
+        }
+
+        // We have all the info for all other nodes.
+        // Fill all that info when sending data to that process.
+        for (i=0; i <  dr->replica_group_size; i++) {
+                if ( dr->workers[i].in_use &&
+                    (dr->workers[i].recon_info->last_term == term)) {
+                    rr.info[num].last_term =
+                        dr->workers[i].recon_info->last_term;
+                    rr.info[num].commited_ops =
+                        dr->workers[i].recon_info->commited_ops;
+                    rr.info[num].last_index =
+                        dr->workers[i].recon_info->last_index;
+                    rr.info[num].first_index =
+                        dr->workers[i].recon_info->first_index;
+                    strcpy(rr.info[num].name,
+                           dr->workers[i].name);
+                }
+                num++;
+        }
+        rr.num = num;
+        rr.role = reconciliator;
+        ENDIAN_CONVERSION_RR(rr, _gf_false); //htonl
+        if (glfs_write(ctx->aux_fd, &rr, sizeof(rr), 0) == -1) {
+                ctx->result = -1;
+                // Put the errno only for this case since we are bothered about
+                // retrying only for this case.  For rest of the cases we will
+                // just return EIO in errno.
+                ctx->op_errno = errno;
+                return;
+        }
+
+        nsr_worker_log(this->name, GF_LOG_INFO,
+                       "sent reconciliator info for term %d with node count as %d\n", term, num);
+}
+
+static void
+control_worker_do_resolution (nsr_per_node_worker_t *ctx,
+                              nsr_recon_work_t *work)
+{
+        unsigned int index = ctx->index;
+        nsr_recon_driver_ctx_t *dr = ctx->driver_ctx;
+        nsr_recon_role_t rr;
+        unsigned int i=0, j=0;
+        unsigned int rec = dr->reconciliator_index;
+
+        nsr_worker_log(this->name, GF_LOG_INFO,
+                       "trying to make this index %d as resolutor with reconciliator as %d\n",index, rec);
+
+        // TBD - error handling for all the glfs APIs
+        if (glfs_lseek(ctx->aux_fd,
+                   nsr_recon_xlator_sector_1,
+                   SEEK_SET) == -1) {
+                ctx->result = -1;
+                return;
+        }
+
+        rr.num = 2;
+
+        // Fill in info[0] as info for the node for which we are seeking
+        // resolution.  Fill in info[1] as info of the reconciliator node.  The
+        // function nsr_recon_driver_get_role() that will be called when this
+        // message reaches the node will look at index 1 for term information
+        // related to the reconciliator.
+        for (i=0; i <  2; i++) {
+                (i == 0) ? (j = index) : (j = rec);
+                rr.info[i].last_term =
+                        dr->workers[j].recon_info->last_term;
+                rr.info[i].commited_ops =
+                        dr->workers[j].recon_info->commited_ops;
+                rr.info[i].last_index =
+                        dr->workers[j].recon_info->last_index;
+                rr.info[i].first_index =
+                        dr->workers[j].recon_info->first_index;
+                // The name is used as the key to convert indices since the
+                // reconciliator index could be different across the nodes.
+                strcpy(rr.info[i].name,
+                        dr->workers[j].name);
+                if (i == 0) {
+                        nsr_worker_log(this->name, GF_LOG_INFO,
+                                       "this node info term=%d, ops=%d, first=%d, last=%d\n",
+                                       rr.info[i].last_term, rr.info[i].commited_ops,
+                                       rr.info[i].first_index,rr.info[i].last_index);
+                } else {
+                        nsr_worker_log(this->name, GF_LOG_INFO,
+                                       "reconciliator node info term=%d, ops=%d, first=%d, last=%d\n",
+                                       rr.info[i].last_term, rr.info[i].commited_ops,
+                                       rr.info[i].first_index,rr.info[i].last_index);
+                }
+        }
+        rr.role = resolutor;
+        ENDIAN_CONVERSION_RR(rr, _gf_false); //htonl
+        if (glfs_write(ctx->aux_fd, &rr, sizeof(rr), 0) == -1) {
+                ctx->result = -1;
+                // Put the errno only for this case since we are bothered about
+                // retrying only for this case.  For rest of the cases we will
+                // just return EIO in errno.
+                ctx->op_errno = errno;
+                return;
+        }
+
+        nsr_worker_log(this->name, GF_LOG_INFO,
+                       "sent message to this node %d resolutor with reconciliator as %d\n", index, rec);
+}
+
+static void
+control_worker_get_window (nsr_per_node_worker_t *ctx, nsr_recon_work_t *work)
+{
+        unsigned int index = ctx->index;
+        nsr_replica_worker_t *rw = &(ctx->driver_ctx->workers[index]);
+        nsr_recon_driver_ctx_t *dr = ctx->driver_ctx;
+        xlator_t *this = dr->this;
+        nsr_recon_log_info_t li;
+        nsr_reconciliator_info_t *recon_info = rw->recon_info;
+        uint32_t i = 0;
+        uint32_t num = (dr->workers[index].recon_info->last_index -
+                       dr->workers[index].recon_info->first_index +1);
+        nsr_recon_record_details_t *rd;
+
+        nsr_worker_log(this->name, GF_LOG_INFO,
+                       "trying to get reconciliation window records for node %d for term %d with first %d last %d\n",
+                       index, recon_info->last_term, recon_info->first_index, recon_info->last_index);
+
+        // TBD - error handling for all the glfs APIs
+        if (glfs_lseek(ctx->aux_fd, nsr_recon_xlator_sector_2, SEEK_SET) == -1) {
+                ctx->result = -1;
+                return;
+        }
+
+        // write to node what term & indices we are interested
+        li.term = recon_info->last_term;
+        li.first_index = recon_info->first_index;
+        li.last_index = recon_info->last_index;
+        ENDIAN_CONVERSION_LI(li, _gf_false); //htonl
+        if (glfs_write(ctx->aux_fd, &li, sizeof(li), 0) == -1) {
+                ctx->result = -1;
+                return;
+        }
+
+        // then read
+        rd = RD_CALLOC(num,
+                       sizeof(nsr_recon_record_details_t),
+                       gf_mt_recon_private_t);
+        if (rd == NULL) {
+                ctx->result = -1;
+                return;
+        }
+        recon_info->records = RD_CALLOC(num,
+                       sizeof(nsr_reconciliation_record_t),
+                       gf_mt_recon_private_t);
+        if (recon_info->records == NULL) {
+                ctx->result = -1;
+                goto err;
+        }
+
+        if (glfs_read(ctx->aux_fd, rd, num * sizeof(nsr_recon_record_details_t), 0) == -1) {
+                ctx->result = -1;
+                goto err;
+        }
+
+        for (i=0; i < num; i++) {
+                ENDIAN_CONVERSION_RD(rd[i], _gf_true); //ntohl
+                memcpy (&(recon_info->records[i].rec), &(rd[i]),
+                        sizeof(nsr_recon_record_details_t));
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "get_reconcilaition_window:Got %d at index %d\n",
+                               recon_info->records[i].rec.type,
+                               i + recon_info->first_index);
+        }
+
+        nsr_worker_log(this->name, GF_LOG_INFO,
+                       "got reconciliation window records for node %d for term %d \n",
+                       index, recon_info->last_term);
+
+err:
+        GF_FREE(rd);
+}
+
+/*
+ * Control worker funct for getting changelog info on some other node.
+ * calls glfs functions to seek/read/write on aux_fd.
+ *
+ * Input arguments:
+ * ctx - The per worker based context
+ * control - set to true if this worker is for the control plane
+ */
+static void
+control_worker_func(nsr_per_node_worker_t *ctx,
+                    nsr_recon_work_t *work)
+{
+        unsigned int index = ctx->index;
+        nsr_replica_worker_t *rw = &(ctx->driver_ctx->workers[index]);
+        nsr_recon_last_term_info_t lt;
+        nsr_reconciliator_info_t *recon_info = rw->recon_info;
+        int32_t term = htonl(work->index); // overloading it
+
+        ctx->is_control = _gf_true;
+
+        switch (work->req_id){
+
+        case NSR_WORK_ID_INI:
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "calling nsr_recon_start_work\n");
+
+                // TBD - handle error in case nsr_recon_start_work gives error
+                if (nsr_recon_start_work(ctx, _gf_true) != 0) {
+                        ctx->result = -1;
+                        return;
+                }
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "finished nsr_recon_start_work\n");
+                break;
+
+        case NSR_WORK_ID_FINI:
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "calling nsr_recon_end_work\n");
+
+                // TBD - handle error in case nsr_recon_end_work gives error
+                if (nsr_recon_end_work(ctx, _gf_true) != 0) {
+                        ctx->result = -1;
+                        return;
+                }
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "finished nsr_recon_end_work\n");
+                break;
+
+        case NSR_WORK_ID_GET_LAST_TERM_INFO:
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "trying to get last term info for  node %d with current term %d\n",index, work->index);
+
+                // first write the current term term number
+                // TBD - error handling for all the glfs APIs
+                if (glfs_lseek(ctx->aux_fd, nsr_recon_xlator_sector_4, SEEK_SET) == -1) {
+                        ctx->result = -1;
+                        return;
+                }
+                if (glfs_write(ctx->aux_fd, &term, sizeof(term), 0) == -1) {
+                        ctx->result = -1;
+                        return;
+                }
+                if (glfs_read(ctx->aux_fd, &lt, sizeof(lt), 0) == -1) {
+                        ctx->result = -1;
+                        return;
+                }
+                ENDIAN_CONVERSION_LT(lt, _gf_true); //ntohl
+                recon_info->last_term = lt.last_term;
+                recon_info->commited_ops = lt.commited_ops;
+                recon_info->last_index = lt.last_index;
+                recon_info->first_index = lt.first_index;
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "out of get last term info with current term %d. got ops %d with first %d and last %d \n",
+                               recon_info->last_term, recon_info->commited_ops,
+                               recon_info->first_index, recon_info->last_index);
+
+                break;
+
+        case NSR_WORK_ID_GET_GIVEN_TERM_INFO:
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "trying to get term info for  node %d for term %d\n",index, work->index);
+
+                // first write the term number
+                // TBD - error handling for all the glfs APIs
+                if (glfs_lseek(ctx->aux_fd, nsr_recon_xlator_sector_3, SEEK_SET) == -1) {
+                        ctx->result = -1;
+                        return;
+                }
+                if (glfs_write(ctx->aux_fd, &term, sizeof(term), 0) == -1) {
+                        ctx->result = -1;
+                        return;
+                }
+                if (glfs_read(ctx->aux_fd, &lt, sizeof(lt), 0) == -1) {
+                        ctx->result = -1;
+                        return;
+                }
+                ENDIAN_CONVERSION_LT(lt, _gf_true); //ntohl
+                recon_info->last_term = lt.last_term;
+                recon_info->commited_ops = lt.commited_ops;
+                recon_info->last_index = lt.last_index;
+                recon_info->first_index = lt.first_index;
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "out of get term info for term %d. got ops %d with first %d and last %d \n",
+                               recon_info->last_term, recon_info->commited_ops,
+                               recon_info->first_index, recon_info->last_index);
+
+                break;
+
+        case NSR_WORK_ID_RECONCILIATOR_DO_WORK:
+                control_worker_do_reconciliation(ctx,work);
+                break;
+
+        case NSR_WORK_ID_RESOLUTION_DO_WORK:
+                control_worker_do_resolution(ctx,work);
+                break;
+
+        case NSR_WORK_ID_GET_RECONCILATION_WINDOW:
+                control_worker_get_window(ctx,work);
+                break;
+
+        default:
+                nsr_worker_log (this->name, GF_LOG_ERROR,
+                                "bad work type %d", work->req_id);
+        }
+
+        return;
+}
+
+// Control worker thread
+static void*
+control_worker_main(nsr_per_node_worker_t *ctx)
+{
+        unsigned int index = ctx->index;
+
+        ctx->is_control = _gf_true;
+        nsr_worker_log(this->name, GF_LOG_INFO,
+                       "starting control worker func\n");
+
+        // if this is for local processing, call the changelog parsing calls directly
+        if (index == 0) {
+                control_worker_main_0(ctx);
+                return NULL;
+        }
+
+        init_worker(ctx, 1);
+
+
+        while(1)
+        {
+                nsr_recon_work_t *work = NULL;
+                nsr_recon_driver_ctx_t *dr = ctx->driver_ctx;
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "waiting for work\n");
+
+                pthread_mutex_lock(&ctx->mutex);
+                while (list_empty(&(ctx->head.list))) {
+                        pthread_cond_wait(&ctx->cv, &ctx->mutex);
+                }
+                pthread_mutex_unlock(&ctx->mutex);
+
+
+                list_for_each_entry(work, &(ctx->head.list), list) {
+                        nsr_worker_log(this->name, GF_LOG_INFO,
+                                       "got work with id %d\n", work->req_id);
+			work->in_use = _gf_false;
+                        control_worker_func(ctx,work);
+                        atomic_dec(&(dr->outstanding));
+			break;
+                }
+                nsr_worker_log(this->name, GF_LOG_INFO,"deleting work item\n");
+		list_del_init (&work->list);
+                GF_FREE(work);
+                nsr_worker_log(this->name, GF_LOG_INFO,"finished deleting work item\n");
+        }
+
+        return NULL;
+}
+
+/*
+ * This function gets called if this process is chosen as the reconciliator
+ * for this replica group. It would have already got the records for the last term
+ * for the indices that are required (from the first HOLE to last index) from
+ * all other nodes that also witnessed that term. COmpare all the records and
+ * compute the work required.
+ *
+ * Input arguments
+ * ctx - driver context. All recon work is stored in workers[0].recon_info
+ */
+static void
+compute_reconciliation_work(nsr_recon_driver_ctx_t *ctx)
+{
+        uint32_t i=0, j=0;
+        nsr_reconciliator_info_t *my_recon = ctx->workers[0].recon_info;
+        uint32_t num = (my_recon->last_index - my_recon->first_index + 1);
+
+        for (i=0; i < num; i++) {
+                nsr_log_type_t orig, new;
+                unsigned int src = 0;
+                orig = new = my_recon->records[i].rec.type;
+                nsr_recon_work_type_t tw = NSR_RECON_WORK_NONE;
+                // index 0 means this node. Look at all other nodes.
+                for (j=1; j < ctx->replica_group_size; j++) {
+                        if (ctx->workers[j].in_use) {
+                                nsr_log_type_t pr = ctx->workers[j].recon_info->records[i].work.type;
+                                if ((new != pr) && (pr > new)) {
+                                        src = j;
+                                        new = (new | pr);
+                                }
+                        }
+                }
+                // TBD - compare data if new and orig are all FILLs. (can detect changelog corruption)
+                // Right now we compare if both orig and new are psuedo holes since
+                // only that is of interest to us.
+                if (orig != new) {
+                        if ((orig == NSR_LOG_HOLE) && (new == NSR_LOG_PSEUDO_HOLE))
+                                tw = NSR_RECON_WORK_HOLE_TO_PSEUDO_HOLE;
+                        else if ((orig == NSR_LOG_HOLE) && (new == NSR_LOG_FILL))
+                                tw = NSR_RECON_WORK_HOLE_TO_FILL;
+                        else if ((orig == NSR_LOG_PSEUDO_HOLE) && (new == NSR_LOG_PSEUDO_HOLE))
+                                tw = NSR_RECON_WORK_COMPARE_PSEUDO_HOLE;
+                        else if ((orig == NSR_LOG_PSEUDO_HOLE) && (new == NSR_LOG_FILL))
+                                tw = NSR_RECON_WORK_HOLE_TO_FILL;
+                }
+                if (tw != NSR_RECON_WORK_NONE) {
+                        my_recon->records[i].work.type = tw;
+                        my_recon->records[i].work.source = src;
+                        // Overwrite the record
+                        memcpy(&(my_recon->records[i].rec),
+                               &(ctx->workers[src].recon_info->records[i].rec),
+                               sizeof(nsr_recon_record_details_t));
+               }
+       }
+       return;
+}
+
+static int32_t
+nsr_recon_in_use(nsr_recon_driver_ctx_t *ctx,
+                 uint32_t i,
+                 gf_boolean_t in_use);
+
+/*
+ * Write the role and associated information to the node.
+ * This gets called from recon xlator indicating node is either
+ * leader, reconciliator or should do resolution.
+ */
+gf_boolean_t
+nsr_recon_driver_set_role(nsr_recon_driver_ctx_t *ctx,
+                          nsr_recon_role_t *rr,
+                          uint32_t term)
+{
+        nsr_role_work_t *rw;
+        xlator_t *this = ctx->this;
+
+        nsr_driver_log(this->name, GF_LOG_INFO, "set role called \n");
+        rw = RD_CALLOC(1, sizeof (nsr_role_work_t), gf_mt_recon_role_work_t);
+        memcpy(&rw->role, rr, sizeof(nsr_recon_role_t));
+        rw->term = term;
+        INIT_LIST_HEAD(&(rw->list));
+        pthread_mutex_lock(&(ctx->mutex));
+        list_add_tail(&rw->list, &ctx->role_head.list);
+        pthread_cond_signal(&(ctx->cv));
+        pthread_mutex_unlock(&(ctx->mutex));
+        nsr_driver_log(this->name, GF_LOG_INFO, "set role returns \n");
+        return _gf_true;
+}
+
+/*
+ * First we undo the last role to make sure we clean up.
+ *
+ * Input arguments
+ * ctx - driver context.
+ * rr - Role information. 
+ *      If leader, the thread now sends the list of all nodes that are part of
+ *      the current replica group. Use that to find out the activate the
+ *      required worker threads.
+ *      If reconciliator, the leader node would have sent information  about
+ *      all nodes which saw last term as the reconciliator.
+ *      If resolution to be done, then rr.info[0] will have this node's info
+ *      which the leader would have got earlier. rr[1].info will have the
+ *      info regarding the reconciliator.
+ * term - leader's term that is causing this role
+ */
+nsr_recon_driver_state_t
+nsr_recon_driver_get_role(int32_t *status,
+			  nsr_recon_driver_ctx_t *ctx,
+                          nsr_role_work_t *rw)
+{
+        uint8_t i=0, j=0;
+        nsr_recon_role_t *rr = &(rw->role);
+        nsr_reconciliator_info_t *tmp;
+        xlator_t *this = ctx->this;
+
+        // First make all the threads uninitialise
+        for (i = 0; i < ctx->replica_group_size; i++) {
+                if (nsr_recon_in_use(ctx, i, _gf_false) == -1) {
+			*status = -1;
+			return 0;
+		}
+        }
+
+        switch (rr->role) {
+        case leader:
+        case joiner:
+
+                // First set info this node
+                tmp = RD_CALLOC (1, sizeof (nsr_reconciliator_info_t),
+                                 gf_mt_recon_reconciliator_info_t);
+                if (!tmp) {
+			*status = -1;
+                        return 0;
+                }
+                ctx->workers[0].recon_info = tmp;
+                if (nsr_recon_in_use(ctx, 0, _gf_true) == -1) {
+			*status = -1;
+			return 0;
+		}
+                ctx->current_term = rr->current_term;
+
+                // Find rest of the nodes
+                for (i=1; i < ctx->replica_group_size; i++) {
+                        for (j=0 ; /* nothing */; j++) {
+                                if (j >= rr->num) {
+                                        nsr_driver_log (this->name, GF_LOG_ERROR,
+                                                        "failed to find %s",
+                                                        ctx->workers[i].name);
+                                        break;
+                                }
+                                if (strcmp(ctx->workers[i].name,
+                                           rr->info[j].name)) {
+                                        continue;
+                                }
+                                nsr_driver_log (this->name, GF_LOG_INFO,
+                                               "nsr_recon_driver_get_role: this as %s. found other server %s\n",
+                                               (rr->role == leader) ? "leader"
+                                                                    : "joiner",
+                                               ctx->workers[i].name);
+
+                                // Allocate this here. This will get later
+                                // filled when the leader tries to get last term
+                                // information from all the nodes
+                                tmp = RD_CALLOC (1,
+                                        sizeof (nsr_reconciliator_info_t),
+                                        gf_mt_recon_reconciliator_info_t);
+                                if (!tmp) {
+                                        *status = -1;
+                                        return 0;
+                                }
+                                ctx->workers[i].recon_info = tmp;
+                                if (nsr_recon_in_use(ctx, i, _gf_true) == -1) {
+                                        *status = -1;
+                                        return 0;
+                                }
+                                break;
+                        }
+                }
+                // If leader, reconciliator has to be chosen.
+                // If joiner, we are the reconciliator.
+                if (rr->role == leader)
+                        ctx->reconciliator_index = -1;
+                else
+                        ctx->reconciliator_index = 0;
+                break;
+
+        case reconciliator:
+                ctx->reconciliator_index = 0;
+                // Copy information about all the other members which had the
+                // same term
+                for (i=0; i < rr->num; i++) {
+                        for (j=0; /* nothing */; j++) {
+                                if (j >= ctx->replica_group_size) {
+                                        nsr_driver_log (this->name, GF_LOG_ERROR,
+                                                        "failed to find %s",
+                                                        rr->info[i].name);
+                                        break;
+                                }
+                                if (strcmp(rr->info[i].name,
+                                           ctx->workers[j].name)) {
+                                        continue;
+                                }
+                                nsr_driver_log(this->name, GF_LOG_INFO,
+                                               "nsr_recon_driver_get_role: this as reconciliator. found other server %s\n",
+                                               ctx->workers[j].name);
+                                tmp = RD_CALLOC (1,
+                                        sizeof (nsr_reconciliator_info_t),
+                                        gf_mt_recon_reconciliator_info_t);
+                                if (!tmp) {
+                                        *status = -1;
+                                        return 0;
+                                }
+                                tmp->last_term = rr->info[i].last_term;
+                                tmp->commited_ops = rr->info[i].commited_ops;
+                                tmp->last_index = rr->info[i].last_index;
+                                tmp->first_index = rr->info[i].first_index;
+                                ctx->workers[j].recon_info = tmp;
+                                if (nsr_recon_in_use(ctx, j, _gf_true) == -1) {
+                                        *status = -1;
+                                        return 0;
+                                }
+                                break;
+                        }
+                 }
+                 break;
+
+        case resolutor:
+                for (j=0; /* nothing */; j++) {
+                        // info[1] has the information regarding the
+                        // reconciliator
+                        if (j >= ctx->replica_group_size) {
+                                nsr_driver_log (this->name, GF_LOG_ERROR,
+                                                "failed to find %s",
+                                                rr->info[1].name);
+                                break;
+                        }
+                        if (strcmp(rr->info[1].name,
+                                   ctx->workers[j].name)) {
+                                continue;
+                        }
+                        nsr_driver_log(this->name, GF_LOG_INFO,
+                                       "nsr_recon_driver_get_role: this as resolutor. found other server %s as reconciliator\n",
+                                       ctx->workers[j].name);
+                        tmp = RD_CALLOC (1,
+                                sizeof (nsr_reconciliator_info_t),
+                                gf_mt_recon_reconciliator_info_t);
+                        if (!tmp) {
+                                 *status = -1;
+                                 return 0;
+                        }
+                        tmp->last_term = rr->info[1].last_term;
+                        tmp->commited_ops = rr->info[1].commited_ops;
+                        tmp->last_index = rr->info[1].last_index;
+                        tmp->first_index = rr->info[1].first_index;
+                        ctx->reconciliator_index = j;
+                        ctx->workers[j].recon_info = tmp;
+                        if (nsr_recon_in_use(ctx, j, _gf_true) == -1) {
+                                 *status = -1;
+                                 return 0;
+                        }
+                        GF_ASSERT(ctx->reconciliator_index != 0);
+                        break;
+                 }
+                 tmp = RD_CALLOC (1,
+                         sizeof (nsr_reconciliator_info_t),
+                         gf_mt_recon_reconciliator_info_t);
+                 if (!tmp) {
+			 *status = -1;
+			 return 0;
+                 }
+                 // info[0] has all info for this node
+                 tmp->last_term = rr->info[0].last_term;
+                 tmp->commited_ops = rr->info[0].commited_ops;
+                 tmp->last_index = rr->info[0].last_index;
+                 tmp->first_index = rr->info[0].first_index;
+                 ctx->workers[0].recon_info = tmp;
+                 if (nsr_recon_in_use(ctx, 0, _gf_true) == -1) {
+			 *status = -1;
+			 return 0;
+		 }
+        }
+
+        ctx->term = rw->term;
+
+	*status = 0;
+        return rr->role;
+}
+
+
+/*
+ * This function gets called if this process is chosen to sync itself with 
+ * the reconciliator.
+ *
+ * Input arguments
+ * ctx - driver context.
+ * my_info - local changelog info that has all the local records for indices that require work
+ * his_info - reconciliator's info that has all the golden copies
+ * invalidate - if set to true, then do not consult local records
+ */
+
+static void
+compute_resolution_work(nsr_recon_driver_ctx_t *ctx,
+                        nsr_reconciliator_info_t *my_info,
+                        nsr_reconciliator_info_t *his_info,
+                        gf_boolean_t invalidate)
+{
+        uint32_t i=0;
+        uint32_t num = (my_info->last_index - my_info->first_index + 1);
+        xlator_t *this = ctx->this;
+
+        if (invalidate) {
+                if (my_info->records) {
+                        GF_FREE(my_info->records);
+		}
+		my_info->records = RD_CALLOC(num,
+					     sizeof(nsr_reconciliation_record_t),
+					     gf_mt_recon_record_t);
+        }
+
+        for (i=0; i < num; i++) {
+                nsr_log_type_t orig, new;
+                nsr_recon_work_type_t tw = NSR_RECON_WORK_NONE;
+                orig = my_info->records[i].rec.type;
+                if (invalidate)
+                        orig = NSR_LOG_HOLE;
+                new  = his_info->records[i].rec.type;
+                // TBD - we can never have PSUEDO_HOLE in reconciliator's info
+                // We should have taken care of that during reconciliation.
+                // Put an assert to validate that.
+                if (new != orig) {
+                        if ((orig != NSR_LOG_FILL) && (new == NSR_LOG_FILL))
+                                tw = NSR_RECON_WORK_HOLE_TO_FILL;
+                        else if ((orig != NSR_LOG_HOLE) && (new == NSR_LOG_HOLE))
+                                tw = NSR_RECON_WORK_UNDO_FILL;
+                }
+                // copy the records anyway
+                my_info->records[i].work.type = tw;
+                my_info->records[i].work.source = ctx->reconciliator_index;
+                memcpy(&(my_info->records[i].rec),
+                       &(his_info->records[i].rec),
+                       sizeof(nsr_recon_record_details_t));
+       }
+       return;
+}
+
+
+// Create an glfs object
+static struct glfs_object *
+create_obj(nsr_per_node_worker_t *ctx, char  *gfid_str)
+{
+       struct glfs_object *obj = NULL;
+       uuid_t gfid;
+
+       uuid_parse(gfid_str, gfid);
+
+       obj = glfs_h_create_from_handle(ctx->fs, gfid, GFAPI_HANDLE_LENGTH, NULL);
+       if (obj == NULL) {
+                GF_ASSERT(obj != NULL);
+                nsr_worker_log(this->name, GF_LOG_ERROR,
+                               "creating of handle failed\n");
+                return NULL;
+        }
+        return obj;
+}
+
+/*
+ * Function to apply the actual record onto the local brick.
+ * prior to this we should have read all the data from the
+ * brick that has the data.
+ *
+ * Input parameters:
+ * ctx - per node worker context that has the fs for communicating to brick
+ * ri - Reconciliation record that needs fixup
+ * dict - So that NSR server translator on brick applis fixup only on this brick
+ *        and the changelog translator consumes term and index.
+ */
+
+static gf_boolean_t
+apply_record(nsr_per_node_worker_t *ctx,
+             nsr_reconciliation_record_t *ri,
+             dict_t * dict)
+{
+        struct glfs_fd *fd = NULL;
+        struct glfs_object *obj = NULL;
+        struct glfs_object *to_obj = NULL;
+        gf_boolean_t retval = _gf_false;
+
+        if (ri->rec.op == GF_FOP_WRITE) {
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "DOing write for file %s  @offset %d for len %d\n",
+                               ri->rec.gfid, ri->rec.offset, ri->rec.len);
+
+                // The file has got deleted on the source. Hence just ignore
+                // this.
+                // TBD - get a way to just stuff the log entry without writing
+                // the data so that changelogs remain identical.
+                if (ri->work.data == NULL) {
+                        return _gf_true;
+                }
+
+                if ((obj = create_obj(ctx,ri->rec.gfid)) == NULL)
+		        goto err;
+
+                fd = glfs_h_open_with_xdata(ctx->fs, obj, O_RDWR, dict);
+                if (fd == NULL) {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "open for file %s failed\n",
+                                       ri->rec.gfid);
+                        goto err;
+                }
+                if (glfs_lseek_with_xdata(fd, ri->rec.offset, SEEK_SET, dict) != ri->rec.offset) {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "lseek for file %s failed at offset %d\n",
+                                       ri->rec.gfid, ri->rec.offset);
+                        goto err;
+                }
+                if (glfs_write_with_xdata(fd, ri->work.data, ri->rec.len, 0, dict) != ri->rec.len) {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "write for file %s failed for bytes %d\n",
+                                       ri->rec.gfid, ri->rec.len);
+                        goto err;
+                }
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "Finished DOing write for gfid %s  @offset %d for len %d\n",
+                               ri->rec.gfid, ri->rec.offset, ri->rec.len);
+
+        } else if (ri->rec.op == GF_FOP_FTRUNCATE) {
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "DOing truncate for file %s  @offset %d \n",
+                               ri->rec.gfid, ri->rec.offset);
+
+                if ((obj = create_obj(ctx, ri->rec.gfid)) == NULL) {
+                       goto err;
+                }
+
+                fd = glfs_h_open_with_xdata(ctx->fs, obj, O_RDWR, dict);
+                if (fd == NULL) {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "open for file %s failed\n",
+                                       ri->rec.gfid);
+                        goto err;
+                }
+                if (glfs_ftruncate_with_xdata(fd, ri->rec.offset, dict) == -1) {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "trunctae for file %s failed @offset %d\n",
+                                       ri->rec.gfid,ri->rec.offset );
+                        goto err;
+                }
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "Finished DOing truncate for gfid %s  @offset %d \n",
+                               ri->rec.gfid, ri->rec.offset);
+
+        } else if ((ri->rec.op == GF_FOP_FREMOVEXATTR) ||
+                   (ri->rec.op == GF_FOP_REMOVEXATTR) ||
+                   (ri->rec.op == GF_FOP_SETXATTR) ||
+                   (ri->rec.op == GF_FOP_FSETXATTR)) {
+
+                uint32_t k_s = 0, v_s = 0;
+                char *t_b= NULL;
+                uint32_t num = 0;
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "Doing set extended attr for file %s \n",
+                               ri->rec.gfid);
+
+                // The file has got deleted on the source. Hence just ignore
+                // this.  TBD - get a way to just stuff the log entry without
+                // writing the data so that changelogs remain identical.
+                if (ri->work.data == NULL) {
+                        return  _gf_true;
+                }
+
+                if ((obj = create_obj(ctx, ri->rec.gfid)) == NULL) {
+                       goto err;
+                }
+
+                if (obj->inode->ia_type  == IA_IFDIR)
+                        fd = glfs_h_opendir_with_xdata(ctx->fs, obj, dict);
+                else
+                        fd = glfs_h_open_with_xdata(ctx->fs, obj, O_RDWR, dict);
+                if (fd == NULL) {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "open for file %s failed\n",
+                                       ri->rec.gfid);
+                        goto err;
+                }
+
+                if(get_xattr_total_size(fd, &t_b, &k_s, &v_s, &num, dict) == -1) {
+                        if (t_b) free(t_b);
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "list of xattr of  %s failed\n", ri->rec.gfid);
+                        goto err;
+                }
+
+                if (delete_xattr(fd, dict, t_b, num) == -1) {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "deleting xattrs failed\n");
+                        goto err;
+		}
+
+                // Set one special dict flag to indicate the opcode so that
+                // the opcode gets set to this
+                if (dict_set_int32(dict,"recon-xattr-opcode",ri->rec.op)) {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "setting opcode to %d failed\n",ri->rec.op);
+                        goto err;
+                }
+
+                if (fill_xattr(fd, dict, ri->work.data,  ri->work.num) == -1) {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "filling xattrs failed\n");
+                        goto err;
+		}
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "Finsihed Doing set extended attr for  %s \n",
+                               ri->rec.gfid);
+
+        } else if (ri->rec.op == GF_FOP_CREATE) {
+
+                uuid_t gfid;
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "Doing create for file %s \n",
+                               ri->rec.gfid);
+
+                // TBD - add mode and flags later
+                uuid_parse(ri->rec.gfid, gfid);
+                if ((obj = create_obj(ctx, ri->rec.pargfid)) == NULL) {
+                       goto err;
+                }
+
+                nsr_worker_log (this->name, GF_LOG_INFO,
+                                "creating with mode 0%o", ri->rec.mode);
+                if (glfs_h_creat_with_xdata(ctx->fs, obj, ri->rec.entry, O_RDWR, ri->rec.mode,  NULL, gfid, dict) == NULL) {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "Failure for Doing create for file %s\n",
+                                       ri->rec.entry);
+                        goto err;
+                }
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "Finished Doing create for file %s \n",
+                               ri->rec.entry);
+
+        } else if (ri->rec.op == GF_FOP_MKNOD) {
+
+                uuid_t gfid;
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "Doing mknod for file %s \n",
+                               ri->rec.entry);
+
+                // TBD - add mode and flags later
+                uuid_parse(ri->rec.gfid, gfid);
+                if ((obj = create_obj(ctx, ri->rec.pargfid)) == NULL) {
+                       goto err;
+                }
+
+                if (glfs_h_mknod_with_xdata(ctx->fs, obj,  ri->rec.entry, O_RDWR, 0777, NULL, gfid, dict) == NULL) {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "Failure for Doing mknod for file %s\n",
+                                       ri->rec.entry);
+                        goto err;
+                }
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "Finished Doing mknod for file %s \n",
+                               ri->rec.entry);
+
+        } else if (ri->rec.op == GF_FOP_MKDIR) {
+
+                uuid_t gfid;
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "Doing mkdir for dir %s \n",
+                               ri->rec.gfid);
+
+                // TBD - add mode and flags later
+                uuid_parse(ri->rec.gfid, gfid);
+                if ((obj = create_obj(ctx, ri->rec.pargfid)) == NULL) {
+                       goto err;
+                }
+
+                if (glfs_h_mkdir_with_xdata(ctx->fs, obj, ri->rec.entry, 0777, NULL, gfid, dict) != 0) {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "Failure for Doing mkdir for file %s\n",
+                                       ri->rec.entry);
+                        goto err;
+                }
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "Finished Doing mkdir for file %s \n",
+                               ri->rec.entry);
+
+        } else if ((ri->rec.op == GF_FOP_RMDIR) || (ri->rec.op == GF_FOP_UNLINK)) {
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "Doing rmdir/ublink for dir %s \n",
+                               ri->rec.entry);
+
+                if ((obj = create_obj(ctx, ri->rec.pargfid)) == NULL) {
+                       goto err;
+                }
+                if (glfs_h_unlink_with_xdata(ctx->fs, obj, ri->rec.entry, dict) != 0) {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "Failure for Doing rmdir/unlink for file %s\n",
+                                       ri->rec.entry);
+                        goto err;
+                }
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "Finished Doing rmdir/unlink for file %s \n",
+                               ri->rec.entry);
+
+        } else if (ri->rec.op == GF_FOP_SYMLINK) {
+
+                uuid_t gfid;
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "Doing symlink for file %s to file  %s \n",
+                               ri->rec.entry, ri->rec.link_path);
+
+                if ((obj = create_obj(ctx, ri->rec.pargfid)) == NULL) {
+                       goto err;
+                }
+                uuid_parse(ri->rec.gfid, gfid);
+
+                if (glfs_h_symlink_with_xdata(ctx->fs, obj,  ri->rec.entry, ri->rec.link_path, NULL, gfid, dict) == NULL) {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "Failed to Doing symlink for file %s to file  %s \n",
+                                        ri->rec.entry, ri->rec.link_path);
+                        goto err;
+                }
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "Finished Doing symlink for file %s to file  %s \n",
+                               ri->rec.entry, ri->rec.link_path);
+
+        } else if (ri->rec.op == GF_FOP_LINK) {
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "Doing hard link for file %s to file  %s \n",
+                               ri->rec.entry, ri->rec.gfid);
+
+                if ((obj = create_obj(ctx, ri->rec.pargfid)) == NULL) {
+                       goto err;
+                }
+                if ((to_obj = create_obj(ctx, ri->rec.gfid)) == NULL) {
+                       goto err;
+                }
+
+                if (glfs_h_link_with_xdata(ctx->fs, to_obj, obj, ri->rec.entry, dict) == -1) {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "Failed to Doing hard link for file %s to file  %s \n",
+                                        ri->rec.entry, ri->rec.gfid);
+                        goto err;
+                }
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "Finsihed doing hard link for file %s to file  %s \n",
+                               ri->rec.entry, ri->rec.gfid);
+
+        } else if (ri->rec.op == GF_FOP_RENAME) {
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "Doing rename for file %s to file  %s \n",
+                               ri->rec.entry, ri->rec.newloc);
+
+                if ((obj = create_obj(ctx, ri->rec.pargfid)) == NULL) {
+                       goto err;
+                }
+                if ((to_obj = create_obj(ctx, ri->rec.gfid)) == NULL) {
+                       goto err;
+                }
+
+                if (glfs_h_rename_with_xdata(ctx->fs, obj, ri->rec.entry, to_obj, ri->rec.newloc, dict) == -1) {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "Failed to Doing rename for file %s to file  %s \n",
+                                       ri->rec.entry, ri->rec.newloc);
+                        goto err;
+                }
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "Finsihed doing renam for file %s to file  %s \n",
+                               ri->rec.entry, ri->rec.newloc);
+
+
+        } else if ((ri->rec.op == GF_FOP_SETATTR) || (ri->rec.op == GF_FOP_FSETATTR)) {
+
+                struct iatt      iatt = {0, };
+                int              valid = 0;
+                int              ret = -1;
+
+                // TBD - do the actual settings once we do that
+                // right now we just set the mode so that changelog gets filled
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "Doing attr for file %s \n",
+                               ri->rec.gfid);
+
+                if ((obj = create_obj(ctx, ri->rec.gfid)) == NULL) {
+                       goto err;
+                }
+
+                if (obj->inode->ia_type  == IA_IFDIR)
+                        fd = glfs_h_opendir_with_xdata(ctx->fs, obj, dict);
+                else
+                        fd = glfs_h_open_with_xdata(ctx->fs, obj, O_RDWR, dict);
+                if (fd == NULL) {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "open for file %s failed\n",
+                                       ri->rec.gfid);
+                        goto err;
+                }
+
+                iatt.ia_prot = ia_prot_from_st_mode(777);
+                valid = GF_SET_ATTR_MODE;
+
+
+                // Set one special dict flag to indicate the opcode so that
+                // the opcode gets set to this
+                if (dict_set_int32(dict,"recon-attr-opcode",ri->rec.op)) {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "setting opcode to %d failed\n",ri->rec.op);
+                        goto err;
+                }
+
+                ret = glfs_fsetattr_with_xdata(fd, &iatt, valid, dict);
+                if (ret == -1) {
+                        nsr_worker_log(this->name, GF_LOG_INFO,
+                                       "failed Doing attr for file %s \n",
+                                       ri->rec.gfid);
+                        goto err;
+                }
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                               "Doing attr for file %s \n",
+                               ri->rec.gfid);
+
+        }
+
+        retval = _gf_true;
+
+err:
+        if (fd) {
+                /*
+                 * It's not clear that we should be passing the same dict to
+                 * glfs_close that was passed to us for glfs_open, but that's
+                 * the prior behavior so let's preserve it for now.
+                 */
+                if (glfs_close_with_xdata(fd, dict) == -1) {
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "close failed\n");
+		}
+        }
+        if (obj) {
+                /*
+                 * AFAICT fd operations do not borrow this reference, so we
+                 * still need to drop it ourselves.
+                 */
+                glfs_h_close(obj);
+        }
+        if (to_obj) {
+                /*
+                 * AFAICT fd operations do not borrow this reference, so we
+                 * still need to drop it ourselves.
+                 */
+                glfs_h_close(to_obj);
+        }
+        return retval;
+}
+
+//return back opcodes that requires reading from source
+static gf_boolean_t
+recon_check_changelog(nsr_recon_record_details_t *rd)
+{
+        return((rd->op == GF_FOP_WRITE) ||
+               (rd->op == GF_FOP_FSETATTR) ||
+               (rd-> op == GF_FOP_SETATTR) ||
+               (rd->op == GF_FOP_FREMOVEXATTR) ||
+               (rd->op == GF_FOP_SETXATTR) ||
+               (rd->op == GF_FOP_FSETXATTR) ||
+               (rd->op == GF_FOP_SYMLINK));
+
+}
+
+// TBD
+static gf_boolean_t
+recon_compute_undo(nsr_recon_record_details_t *rd)
+{
+        return(_gf_false);
+}
+
+
+/*
+ * Function that talks to the brick for data tranfer.
+ *
+ * Input arguments:
+ * ctx - worker context
+ * work - pointer to work object
+ */
+static void
+data_worker_func(nsr_per_node_worker_t *ctx,
+                 nsr_recon_work_t *work)
+{
+        nsr_recon_driver_ctx_t *dr = ctx->driver_ctx;
+        xlator_t *this = dr->this;
+        nsr_reconciliation_record_t *ri = NULL;
+        nsr_recon_record_details_t *rd = NULL;
+        int wip = 0;
+        dict_t * dict = NULL;
+        struct glfs_fd *fd = NULL;
+        struct glfs_object *obj = NULL;
+        uuid_t gfid;
+        uint32_t k_s = 0, v_s = 0;
+        char *t_b= NULL;
+        uint32_t num=0;
+
+        switch (work->req_id){
+        case NSR_WORK_ID_INI:
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                                "started data ini \n");
+
+                if (nsr_recon_start_work(ctx, _gf_false) != 0) {
+			ctx->result = -1;
+			return;
+		}
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                                "finished data ini \n");
+                break;
+        case NSR_WORK_ID_FINI:
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                                "started data fini \n");
+
+                if (nsr_recon_end_work(ctx, _gf_false) != 0) {
+			ctx->result = -1;
+			return;
+		}
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                                "finished data fini \n");
+                break;
+        case NSR_WORK_ID_SINGLE_RECONCILIATION_READ:
+                // first_index always starts with 1 but records starts at 0.
+                wip = work->index - (dr->workers[0].recon_info->first_index);
+                ri = &(dr->workers[0].recon_info->records[wip]);
+                rd = &(ri->rec);
+
+                dict = dict_new ();
+                if (!dict) {
+			ctx->result = -1;
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "failed allocating for dictionary\n");
+                        break;
+                }
+                if (dict_set_int32(dict,RECON_TERM_XATTR,ri->work.term)) {
+			ctx->result = -1;
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "error setting term in dict\n");
+                        break;
+                }
+                if (dict_set_int32(dict,RECON_INDEX_XATTR,ri->work.index)) {
+			ctx->result = -1;
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "error setting term in dict\n");
+                        break;
+                }
+
+                switch (rd->op) {
+                case GF_FOP_WRITE:
+
+                        // record already copied.
+                        // copy data to this node's info.
+
+                        uuid_parse(ri->rec.gfid, gfid);
+
+                        nsr_worker_log(this->name, GF_LOG_INFO,
+                                       "started recon read for file %s at offset %d at len %d\n",
+                                       ri->rec.gfid, rd->offset, rd->len);
+
+                        obj = glfs_h_create_from_handle (ctx->fs, gfid,
+                                                         GFAPI_HANDLE_LENGTH,
+                                                         NULL);
+                        if (obj == NULL) {
+                                GF_ASSERT(obj != NULL);
+                                nsr_worker_log(this->name, GF_LOG_ERROR,
+                                               "creating of handle failed\n");
+                                break;
+                        }
+
+                        // The file has probably got deleted.
+                        fd = glfs_h_open_with_xdata (ctx->fs, obj, O_RDONLY,
+                                                     dict);
+                        if (fd  == NULL) {
+                                GF_ASSERT(fd != NULL);
+                                nsr_worker_log(this->name, GF_LOG_ERROR,
+                                               "opening of file failed\n");
+                                break;
+                        }
+
+                        if (glfs_lseek_with_xdata (fd, rd->offset, SEEK_SET,
+                                                   dict) != rd->offset) {
+                                nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "lseek of file failed to offset %d\n",
+                                       rd->offset);
+                                break;
+                        }
+
+                        ri->work.data = RD_CALLOC (rd->len , sizeof(char),
+                                                   gf_mt_recon_work_data_t);
+                        if (glfs_read_with_xdata (fd, ri->work.data, rd->len,
+                                                  0, dict) != rd->len) {
+                                nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "read of file failed to offset %d for bytes %d\n",
+                                       rd->offset, rd->len);
+                                break;
+                        }
+                        break;
+
+                case GF_FOP_FTRUNCATE:
+                case GF_FOP_SYMLINK:
+                case GF_FOP_RMDIR:
+                case GF_FOP_UNLINK:
+                case GF_FOP_MKNOD:
+                case GF_FOP_CREATE:
+                case GF_FOP_LINK:
+                case GF_FOP_MKDIR:
+                case GF_FOP_RENAME:
+                        nsr_worker_log (this->name, GF_LOG_ERROR,
+                                        "unimplemented fop %u\n", rd->op);
+                        break;
+
+                case GF_FOP_FREMOVEXATTR:
+                case GF_FOP_REMOVEXATTR:
+                case GF_FOP_SETXATTR:
+                case GF_FOP_FSETXATTR:
+
+                        uuid_parse(ri->rec.gfid, gfid);
+
+
+                        // This is for all the set attribute/extended
+                        // attributes commands.  Get all the attributes from
+                        // the source and fill it in the buffer as a NULL
+                        // seperated key and value which are in turn seperated
+                        // by NULL.
+
+                        nsr_worker_log(this->name, GF_LOG_INFO,
+                                       "doing getattr for gfid %s \n",
+                                       ri->rec.gfid);
+
+                        obj = glfs_h_create_from_handle (ctx->fs, gfid,
+                                                         GFAPI_HANDLE_LENGTH,
+                                                         NULL);
+                        if (obj == NULL) {
+                                GF_ASSERT(fd != NULL);
+                                nsr_worker_log(this->name, GF_LOG_ERROR,
+                                               "creating of handle failed\n");
+                                break;
+                        }
+
+                        if (obj->inode->ia_type  == IA_IFDIR)
+                                fd = glfs_h_opendir_with_xdata (ctx->fs, obj,
+                                                                dict);
+                        else
+                                fd = glfs_h_open_with_xdata (ctx->fs, obj,
+                                                             O_RDONLY, dict);
+
+                        if (fd  == NULL) {
+                                GF_ASSERT(fd != NULL);
+                                nsr_worker_log(this->name, GF_LOG_ERROR,
+                                               "opening of file failed\n");
+                                break;
+                        }
+
+                        if (get_xattr_total_size (fd, &t_b, &k_s, &v_s, &num,
+                                                  dict) == -1) {
+                                if (t_b) free(t_b);
+                                nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "list of xattr of gfid %s failed\n",
+                                       rd->gfid);
+                                break;
+                        }
+                        ri->work.data = RD_CALLOC ((k_s + v_s) , sizeof(char),
+                                                   gf_mt_recon_work_data_t);
+                        if (get_xattr(fd, t_b, ri->work.data, v_s, num, dict) == -1) {
+				nsr_worker_log(this->name, GF_LOG_ERROR,
+					       "get xattr of gfid %s failed\n", rd->gfid);
+				break;
+			}
+                        ri->work.num = num;
+                        nsr_worker_log(this->name, GF_LOG_INFO,
+                                       "finished getattr for gfid %s \n",
+                                       ri->rec.gfid);
+                        free(t_b);
+                        break;
+
+                case GF_FOP_FSETATTR:
+                case GF_FOP_SETATTR:
+                        //TBD - to get the actual attrbutes and fill
+                        // mode, uid, gid, size, atime, mtime
+                        nsr_worker_log (this->name, GF_LOG_ERROR,
+                                        "unimplemented fop %u\n", rd->op);
+                        break;
+                default:
+                        nsr_worker_log (this->name, GF_LOG_ERROR,
+                                        "unrecognized fop %u\n", rd->op);
+
+                }
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                       "finished recon read for gfid %s at offset %d for %d bytes \n",
+                       rd->gfid, rd->offset, rd->len);
+                break;
+
+        case NSR_WORK_ID_SINGLE_RECONCILIATION_COMMIT:
+                // first_index always starts with 1 but records starts at 0.
+                wip = work->index - (dr->workers[0].recon_info->first_index);
+                ri = &(dr->workers[0].recon_info->records[wip]);
+                rd = &(ri->rec);
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                        "got recon commit for index %d that has gfid %s \n",
+                        wip, rd->gfid);
+                dict = dict_new ();
+                if (!dict) {
+			ctx->result = -1;
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "failed allocating for dictionary\n");
+                        break;
+                }
+                if (dict_set_int32(dict,RECON_TERM_XATTR,ri->work.term)) {
+			ctx->result = -1;
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "error setting term in dict\n");
+                        break;
+                }
+                if (dict_set_int32(dict,RECON_INDEX_XATTR,ri->work.index)) {
+			ctx->result = -1;
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "error setting term in dict\n");
+                        break;
+                }
+                if (apply_record(ctx, ri, dict) == _gf_false) {
+			nsr_worker_log(this->name, GF_LOG_ERROR,
+				       "apply_record fails\n");
+		}
+
+                nsr_worker_log(this->name, GF_LOG_INFO,
+                        "finished recon commit for gfid %s \n",
+                        rd->gfid);
+                break;
+
+        case NSR_WORK_ID_SINGLE_RECONCILIATION_FLUSH:
+                dict = dict_new ();
+                if (!dict) {
+			ctx->result = -1;
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "failed allocating for dictionary\n");
+                        break;
+                }
+                if (dict_set_int32(dict,RECON_TERM_XATTR,ri->work.term)) {
+			ctx->result = -1;
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "error setting term in dict\n");
+                        break;
+                }
+                if (dict_set_int32(dict,RECON_INDEX_XATTR,ri->work.index)) {
+			ctx->result = -1;
+                        nsr_worker_log(this->name, GF_LOG_ERROR,
+                                       "error setting term in dict\n");
+                        break;
+                }
+
+                // Increment work index with the start index
+                wip = work->index - (dr->workers[0].recon_info->first_index);
+                ri = &(dr->workers[0].recon_info->records[wip]);
+                rd = &(ri->rec);
+
+                glfs_fsync_with_xdata(fd, dict);
+                break;
+
+        default:
+                nsr_worker_log (this->name, GF_LOG_ERROR,
+                                "unrecognized request id %u\n", work->req_id);
+        }
+
+        if (fd) {
+                glfs_close_with_xdata(fd, dict);
+        }
+        if (obj) {
+                glfs_h_close(obj);
+        }
+        if (dict) {
+                dict_unref(dict);
+        }
+}
+
+// thread for doing data work
+static void *
+data_worker_main(nsr_per_node_worker_t *ctx)
+{
+        nsr_worker_log(this->name, GF_LOG_INFO,
+                       "starting data worker func\n");
+        init_worker(ctx, 0);
+
+        while(1) {
+                  nsr_recon_work_t *work = NULL;
+                  nsr_recon_driver_ctx_t *dr = ctx->driver_ctx;
+
+                  nsr_worker_log(this->name, GF_LOG_INFO,
+                                 "waiting for work\n");
+
+                  pthread_mutex_lock(&(ctx->mutex));
+                  while (list_empty(&(ctx->head.list))) {
+                        pthread_cond_wait(&(ctx->cv), &(ctx->mutex));
+                  }
+                  pthread_mutex_unlock(&(ctx->mutex));
+                  list_for_each_entry(work, &(ctx->head.list), list) {
+                        nsr_worker_log(this->name, GF_LOG_INFO,
+                                       "got work with id %d\n",work->req_id);
+			work->in_use = _gf_false;
+                        data_worker_func(ctx, work);
+                        atomic_dec(&(dr->outstanding));
+			break;
+                }
+                nsr_worker_log(this->name, GF_LOG_INFO,"deleting work item\n");
+		list_del_init (&work->list);
+                GF_FREE(work);
+                nsr_worker_log(this->name, GF_LOG_INFO,"finished deleting work item\n");
+        }
+
+        return NULL;
+}
+
+
+//make recon work 
+static void
+recon_make_work(nsr_recon_driver_ctx_t *ctx,
+                nsr_recon_work_t **work,
+                nsr_recon_work_req_id_t req_id,
+                int32_t i)
+{
+        xlator_t *this = ctx->this;
+
+        // TBD - change this to get from a static pool
+        // This cannot fail
+        (*work) =  RD_CALLOC (1, sizeof (nsr_recon_work_t), gf_mt_recon_work_t);
+        (*work)->req_id = req_id;
+        (*work)->index = i;
+	(*work)->in_use = _gf_true;
+        INIT_LIST_HEAD(&((*work)->list));
+        return;
+}
+
+// Schedule a work object to a worker thread.
+static void
+recon_queue_to_worker(nsr_recon_driver_ctx_t *ctx,
+                      nsr_recon_work_t *work,
+                      unsigned int id,
+                      nsr_recon_queue_type_t type)
+{
+        nsr_per_node_worker_t *worker;
+        if (type == NSR_RECON_QUEUE_TO_CONTROL) {
+		worker = ctx->workers[id].control_worker;
+                nsr_driver_log(this->name, GF_LOG_INFO,
+                               "queueing work to control index %d\n",id);
+        } else {
+		 worker= ctx->workers[id].data_worker;
+                nsr_driver_log(this->name, GF_LOG_INFO,
+                               "queueing work to data index %d\n",id);
+        }
+        pthread_mutex_lock(&worker->mutex);
+        list_add_tail(&work->list, &worker->head.list);
+        pthread_cond_signal(&worker->cv);
+        pthread_mutex_unlock(&worker->mutex);
+        return;
+}
+
+typedef void * (*F_t) (void *);
+
+// In case  mode is set to NSR_USE_THREADS, create worker threads.
+static gf_boolean_t
+create_worker_threads(nsr_recon_private_t *priv,
+                       nsr_recon_driver_ctx_t *ctx,
+                       nsr_per_node_worker_t *w,
+                       gf_boolean_t control_or_data,
+                       F_t f,
+                       uint32_t num)
+{
+        uint32_t i;
+        nsr_per_node_worker_t *worker = w;
+        xlator_t *this = ctx->this;
+
+        for (i=0; i < num; i++) {
+                worker->id = RD_CALLOC(1, 10, gf_mt_recon_id_t);
+                if (!worker->id) {
+                        nsr_driver_log (priv->this->name, GF_LOG_ERROR, "memory allocation error \n");
+                        return _gf_false;
+                }
+                sprintf(worker->id,"recon_%d", i);
+                worker->driver_ctx = ctx ;
+
+                if (ctx->mode == NSR_USE_THREADS) {
+                        if (pthread_create(&worker->thread_id, NULL, f, worker)) {
+                                nsr_driver_log (ctx->this->name, GF_LOG_ERROR, "control work thread creation error \n");
+                                return  _gf_false;
+                        }
+                }
+		worker->index = i;
+                worker++;
+        }
+        return _gf_true;
+}
+
+/*
+ * In case of thread, send the work item; else call the function directly.
+ *
+ * Input arguments:
+ * bm - bitmap containing indices of nodes we want to send work
+ * num - number of such indices
+ * ctx - driver context from where we derive per worker context
+ * id - request ID
+ * q - control or data
+ * misc - used to overload such as index.
+ */
+static void
+send_and_wait(int32_t *result,
+	      int32_t *op_errno,
+	      int32_t bm,
+              uint32_t num,
+              nsr_recon_driver_ctx_t *ctx,
+              nsr_recon_work_req_id_t id,
+              nsr_recon_queue_type_t q,
+              int32_t misc)
+{
+        uint32_t i = 0;
+	nsr_recon_work_t *work;
+
+#define CONTROL_WORKER(i) ctx->workers[i].control_worker
+#define DATA_WORKER(i) ctx->workers[i].data_worker
+#define WORKER(i) ((q == NSR_RECON_QUEUE_TO_CONTROL) ? (CONTROL_WORKER(i)) : (DATA_WORKER(i)))
+
+	*result = *op_errno = 0;
+
+	for (i=0; i < num; i++) {
+                if ((bm & (1 << i)) && ctx->workers[i].in_use) {
+			WORKER(i)->result = 0;
+			WORKER(i)->op_errno = 0;
+		}
+	}
+        if (ctx->mode == NSR_SEQ) {
+                for (i=0; i < num; i++) {
+                        if ((bm & (1 << i)) && ctx->workers[i].in_use) {
+                                recon_make_work(ctx,&work, id, misc);
+                                if (q == NSR_RECON_QUEUE_TO_CONTROL) {
+                                        if (i == 0)
+                                                control_worker_func_0(ctx->workers[0].control_worker, work);
+                                        else
+                                                control_worker_func(ctx->workers[i].control_worker, work);
+                                } else {
+                                        data_worker_func(ctx->workers[i].data_worker, work);
+                                }
+                                GF_FREE(work);
+                        }
+                }
+		goto out;
+        }
+
+        for (i=0; i < num; i++) {
+                if ((bm & (1 << i)) && ctx->workers[i].in_use) {
+			recon_make_work(ctx,&work, id, misc);
+                        atomic_inc(&(ctx->outstanding));
+                        recon_queue_to_worker(ctx, work, i, q);
+                }
+        }
+
+        nsr_driver_log(this->name, GF_LOG_INFO, "send_and_wait: waiting\n");
+        while (ctx->outstanding) {
+                pthread_yield();
+        }
+out:
+	for (i=0; i < num; i++) {
+                if ((bm & (1 << i)) && ctx->workers[i].in_use) {
+			if (WORKER(i)->result == -1) {
+				*result = -1;
+			}
+		}
+	}
+	if (*result == -1) {
+		for (i=0; i < num; i++) {
+			if ((bm & (1 << i)) && ctx->workers[i].in_use) {
+				if (WORKER(i)->op_errno == EAGAIN) {
+					*op_errno = EAGAIN;
+					break;
+				} else {
+					*op_errno = EIO;
+				}
+			}
+		}
+	}
+
+        nsr_driver_log(this->name, GF_LOG_INFO, "send_and_wait: all workers have returned with result: %d errno:%d\n", *result, *op_errno);
+        return;
+}
+
+// send INI or FINI
+static int32_t
+nsr_recon_in_use(nsr_recon_driver_ctx_t *ctx,
+                 uint32_t i,
+                 gf_boolean_t in_use)
+{
+        uint32_t bm = 1 << i;
+	gf_boolean_t send = _gf_false;
+	int32_t status =0, op_errno = 0;
+
+        send = (ctx->workers[i].in_use != in_use);
+        ctx->workers[i].in_use = in_use;
+
+	if (!send) {
+                return 0;
+        }
+        nsr_driver_log (this->name, GF_LOG_INFO,
+                        "sending %s to index %d\n",in_use?"INI":"FINI",i);
+
+        send_and_wait(&status, &op_errno, bm, ctx->replica_group_size, ctx,
+              (in_use == _gf_true) ? NSR_WORK_ID_INI : NSR_WORK_ID_FINI,
+              NSR_RECON_QUEUE_TO_CONTROL, -1);
+        if (status == -1)
+                goto err;
+
+        send_and_wait(&status, &op_errno, bm, ctx->replica_group_size, ctx,
+              (in_use == _gf_true) ? NSR_WORK_ID_INI : NSR_WORK_ID_FINI,
+              NSR_RECON_QUEUE_TO_DATA, -1);
+        if (status == -1)
+                goto err;
+
+        /*
+         * We really need better error recovery.  To activate a worker, we
+         * allocate memory and send two messages.  If any of those actions
+         * fail, we should undo the others.  It would probably be good to
+         * collapse the two messages into one, because it's pretty trivial to
+         * allocate a temporary structure and either link it in or free it
+         * depending on the result here.
+         */
+
+        if (in_use == _gf_false) {
+                GF_FREE(ctx->workers[i].recon_info);
+        }
+
+	return 0;
+
+err:
+        GF_FREE(ctx->workers[i].recon_info);
+        ctx->workers[i].recon_info = NULL;
+        return -1;
+}
+
+gf_boolean_t
+nsr_recon_driver_reconciliator (nsr_recon_private_t *priv)
+{
+	uint32_t replica_group_size = priv->replica_group_size;
+        uint32_t i;
+        nsr_recon_driver_ctx_t *ctx = priv->driver_thread_context;
+        int32_t bm;
+	int32_t status = 0;
+	int32_t op_errno = 0;
+        gf_boolean_t do_recon = _gf_false;
+        uint32_t start_index = ctx->workers[0].recon_info->first_index;
+        uint32_t end_index = ctx->workers[0].recon_info->last_index;
+        uint32_t num = ((start_index == 0) && (end_index == 0)) ? 0 : (end_index - start_index + 1);
+
+        nsr_driver_log (this->name, GF_LOG_INFO,
+                        "starting reconciliation work as reconciliator \n");
+
+        // nothing to be done? signal back to the recon translator that this
+        // phase done.
+        bm = 1;
+        for (i=1; i < replica_group_size; i++) {
+                if (ctx->workers[i].in_use &&
+                   (ctx->workers[0].recon_info->last_term == ctx->workers[i].recon_info->last_term)) {
+                        ctx->workers[i].recon_info->last_index = end_index;
+                        ctx->workers[i].recon_info->first_index = start_index;
+                        bm |= (1 << i);
+                        do_recon = _gf_true;
+                }
+        }
+
+        if (!do_recon || !num) {
+                nsr_driver_log (this->name, GF_LOG_INFO,
+                                "nothing needs to be done as resolutor \n");
+                return _gf_true;
+        }
+
+        nsr_driver_log (this->name, GF_LOG_INFO,
+                        "getting reconciliation window for term %d from %dto %d  \n",
+                        ctx->workers[0].recon_info->last_term,
+                        start_index, end_index);
+        // We have set the bm in the above for loop where we go thru all nodes
+        // including this node that have seen the last term.
+        send_and_wait(&status, &op_errno, bm,
+                      replica_group_size,
+                      ctx,
+                      NSR_WORK_ID_GET_RECONCILATION_WINDOW,
+                      NSR_RECON_QUEUE_TO_CONTROL, -1);
+        if (status == -1)
+                return _gf_false;
+        nsr_driver_log (this->name, GF_LOG_INFO,
+                        "finished getting reconciliation window for term %d from %dto %d  \n",
+                        ctx->workers[0].recon_info->last_term,
+                        start_index, end_index);
+
+
+        // from the changelogs, calculate the entries that need action and the
+        // source for each of these entries
+        compute_reconciliation_work(ctx);
+
+        // for each of the entries that need fixup, issue IO
+        for (i=start_index; i < (start_index + num); i++) {
+                nsr_reconciliator_info_t *my_recon_info =
+                        ctx->workers[0].recon_info;
+                nsr_reconciliation_record_t *record =
+                        &(my_recon_info->records[i - start_index]);
+
+                record->work.term = ctx->workers[0].recon_info->last_term;
+                record->work.index = i;
+
+                nsr_driver_log (this->name, GF_LOG_INFO,
+                                "fixing index %d\n",i);
+                if ((record->work.type == NSR_RECON_WORK_HOLE_TO_PSEUDO_HOLE) ||
+                    (record->work.type  == NSR_RECON_WORK_HOLE_TO_FILL)) {
+                        // 1st case (RECON_WORK_HOLE_TO_PSEUDO_HOLE): If there
+                        // are only pseudo_holes in others, it is best effort.
+                        // Just pick from the first node that has it and
+                        // proceed.
+                        // 2nd case (RECON_WORK_HOLE_TO_FILL): this node has
+                        // either a HOLE or PSUEDO_HOLE and some one else has a
+                        // FILL(source).  analyse the changelog to check if data
+                        // needs to be read or if the log has all the data
+                        // required
+
+                        if (recon_check_changelog(&record->rec)) {
+                                bm = (1 << record->work.source);
+                                nsr_driver_log (this->name, GF_LOG_INFO,
+                                                "reading data from source %d\n",record->work.source);
+                                send_and_wait(&status, &op_errno, bm,
+                                              replica_group_size,
+                                              ctx,
+                                              NSR_WORK_ID_SINGLE_RECONCILIATION_READ,
+                                              NSR_RECON_QUEUE_TO_DATA,
+                                              i);
+                                if (status == -1)
+                                         return _gf_false;
+                                nsr_driver_log (this->name, GF_LOG_INFO,
+                                                "got data from source %d\n",record->work.source);
+                        }
+
+                        nsr_driver_log (this->name, GF_LOG_INFO,
+                                        "fixing local data as part of reconciliation\n");
+
+                        bm = 1;
+                        send_and_wait(&status, &op_errno, bm,
+                                      replica_group_size,
+                                      ctx,
+                                      NSR_WORK_ID_SINGLE_RECONCILIATION_COMMIT,
+                                      NSR_RECON_QUEUE_TO_DATA,
+                                      i);
+                        if (status == -1)
+                                 return _gf_false;
+
+                        nsr_driver_log (this->name, GF_LOG_INFO,
+                                        "finished fixing local data as part of reconciliation\n");
+
+                } else if (record->work.type == NSR_RECON_WORK_COMPARE_PSEUDO_HOLE) {
+                        // this node has a pseudo_hole and some others have just
+                        // that too. Just convert this to FILL.  let others
+                        // blindly pick it from here.
+                        nsr_driver_log (this->name, GF_LOG_INFO,
+                                        "fixing this record as a fill\n");
+                        bm = 1;
+                        send_and_wait(&status, &op_errno, bm,
+                                      replica_group_size,
+                                      ctx,
+                                      NSR_WORK_ID_SINGLE_RECONCILIATION_FLUSH,
+                                      NSR_RECON_QUEUE_TO_DATA,
+                                      i);
+                        if (status == -1)
+                                 return _gf_false;
+                        nsr_driver_log (this->name, GF_LOG_INFO,
+                                        "finished fixing this record as a fill\n");
+                }
+        }
+
+        nsr_driver_log (this->name, GF_LOG_INFO,
+                        "finished reconciliation work as reconciliator \n");
+
+        // tbd - mark this term golden in the reconciliator
+        return _gf_true;
+}
+
+gf_boolean_t
+nsr_recon_driver_resolutor (nsr_recon_private_t *priv)
+{
+	uint32_t replica_group_size = priv->replica_group_size;
+        uint32_t i;
+        nsr_recon_driver_ctx_t *ctx = priv->driver_thread_context;
+        int32_t bm;
+	int32_t status = 0;
+	int32_t op_errno = 0;
+        // This node's last term is filled when it gets a message from the
+        // leader to act as a reconciliator.
+        uint32_t recon_index = ctx->reconciliator_index;
+        nsr_reconciliator_info_t *my_info =
+                ctx->workers[0].recon_info;
+        nsr_reconciliator_info_t *his_info =
+                ctx->workers[recon_index].recon_info;
+        uint32_t my_last_term = my_info->last_term;
+        uint32_t to_do_term = his_info->last_term;
+        uint32_t my_start_index = 1, my_end_index = 1;
+        uint32_t his_start_index = 1, his_end_index = 1;
+        uint32_t num = 0;
+        gf_boolean_t fl = _gf_true;
+
+        nsr_driver_log (this->name, GF_LOG_INFO,
+                        "starting resolutor work with reconciliator as %d from term %d to term %d \n",
+                        recon_index, my_last_term, to_do_term);
+
+        do {
+
+                if (!fl) {
+                        (his_info->last_term)++;
+                        (my_info->last_term)++;
+                } else {
+                        his_info->last_term = my_last_term;
+                }
+
+                nsr_driver_log (this->name, GF_LOG_INFO, "resolving term %d \n", my_info->last_term);
+
+                // Get reconciliator's term information for that term
+                nsr_driver_log (this->name, GF_LOG_INFO,
+                                "getting info from reconciliator for term %d \n", my_info->last_term);
+                bm = (1 << recon_index);
+                send_and_wait(&status, &op_errno, bm,
+                              replica_group_size,
+                              ctx,
+                              NSR_WORK_ID_GET_GIVEN_TERM_INFO,
+                              NSR_RECON_QUEUE_TO_CONTROL, his_info->last_term);
+                if (status == -1)
+                         return _gf_false;
+                nsr_driver_log (this->name, GF_LOG_INFO,
+                                "finished getting info from reconciliator for term %d \n", my_info->last_term);
+
+
+                // empty term
+                if (!his_info->commited_ops) {
+                        nsr_driver_log (this->name, GF_LOG_INFO,
+                                "reconciliator for term %d is empty. moving to next term. \n", my_info->last_term);
+                        // TBD - mark the term golden
+                        fl = _gf_false;
+                        continue;
+                }
+
+                // calculate the resolution window boundary.  for the last term
+                // this node saw, we compare the resolution window of this and
+                // reconciliator.  for the rest of the nodes, we just accept the
+                // reconciliator info.
+                if (fl) {
+                        my_start_index = my_info->first_index;
+                        my_end_index = my_info->last_index;
+                        his_start_index = his_info->first_index;
+                        his_end_index = his_info->last_index;
+                        my_info->first_index = (my_start_index < his_start_index) ? my_start_index : his_start_index;
+                        my_info->last_index = (my_end_index >  his_end_index) ? my_end_index : his_end_index;
+                } else {
+                        my_info->first_index = his_info->first_index;
+                        my_info->last_index = his_info->last_index;
+                        my_info->commited_ops = his_info->commited_ops;
+                }
+                if (my_info->first_index == 0)
+                        my_info->first_index = 1;
+                num = (my_info->last_index - my_info->first_index) + 1;
+
+
+                // Get the logs from the reconciliator (and this node for this
+                // term)
+                if (fl)
+                        bm = ((1 << recon_index) | 1);
+                else
+                        bm = (1 << recon_index);
+
+                nsr_driver_log (this->name, GF_LOG_INFO,
+                                "getting reconciliation window for term %d from %d to %d  \n",
+                                my_info->last_term,
+                                my_info->first_index, my_info->last_index);
+                send_and_wait(&status, &op_errno, bm,
+                              replica_group_size,
+                              ctx,
+                              NSR_WORK_ID_GET_RECONCILATION_WINDOW,
+                              NSR_RECON_QUEUE_TO_CONTROL, -1);
+                if (status == -1)
+                         return _gf_false;
+                nsr_driver_log (this->name, GF_LOG_INFO,
+                                "finished getting reconciliation window for term %d from %d to %d  \n",
+                                my_info->last_term,
+                                my_info->first_index, my_info->last_index);
+
+                // from the changelogs, calculate the entries that need action
+                compute_resolution_work(ctx, my_info, his_info, !fl);
+
+
+                // for each of the entries that need fixup, issue IO
+                for (i=my_info->first_index; i < (my_info->first_index + num); i++) {
+                        nsr_reconciliation_record_t *record =
+                                &(my_info->records[i - my_info->first_index]);
+
+                        record->work.term = my_info->last_term;
+                        record->work.index = i;
+
+                        nsr_driver_log (this->name, GF_LOG_INFO,
+                                        "fixing index %d\n",i);
+                        if ((record->work.type == NSR_RECON_WORK_HOLE_TO_FILL) ||
+                            (record->work.type == NSR_RECON_WORK_UNDO_FILL)) {
+                                if (((record->work.type == NSR_RECON_WORK_HOLE_TO_FILL) &&
+                                      recon_check_changelog(&record->rec)) ||
+                                    ((record->work.type == NSR_RECON_WORK_UNDO_FILL) &&
+                                      recon_compute_undo(&record->rec))) {
+                                        nsr_driver_log (this->name, GF_LOG_INFO,
+                                                        "reading data from source %d\n",recon_index);
+                                        bm = (1 << recon_index);
+                                        send_and_wait(&status, &op_errno, bm,
+                                                      replica_group_size,
+                                                      ctx,
+                                                      NSR_WORK_ID_SINGLE_RECONCILIATION_READ,
+                                                      NSR_RECON_QUEUE_TO_DATA,
+                                                      i);
+                                        if (status == -1)
+                                                return _gf_false;
+                                        nsr_driver_log (this->name, GF_LOG_INFO,
+                                                        "finished reading data from source %d\n",recon_index);
+                                }
+
+                                nsr_driver_log (this->name, GF_LOG_INFO,
+                                                "fixing local data as part of resolutor\n");
+
+                                bm = 1;
+                                send_and_wait(&status, &op_errno, bm,
+                                              replica_group_size,
+                                              ctx,
+                                              NSR_WORK_ID_SINGLE_RECONCILIATION_COMMIT,
+                                              NSR_RECON_QUEUE_TO_DATA,
+                                              i);
+                                if (status == -1)
+                                        return _gf_false;
+
+                                nsr_driver_log (this->name, GF_LOG_INFO,
+                                                "finished fixing local data as part of resolutor\n");
+                        }
+                }
+                fl = _gf_false;
+
+                // tbd - mark this term golden in the reconciliator
+        } while (my_last_term++ != to_do_term);
+
+        nsr_driver_log (this->name, GF_LOG_INFO,
+                        "finished resolutor work \n");
+        return _gf_true;
+}
+
+gf_boolean_t
+nsr_recon_driver_leader (nsr_recon_private_t *priv)
+{
+	uint32_t replica_group_size = priv->replica_group_size;
+        uint32_t i;
+        nsr_recon_driver_ctx_t *ctx = priv->driver_thread_context;
+        int32_t bm;
+	int32_t status = 0;
+	int32_t op_errno = 0;
+        int32_t chosen = -1;
+        int32_t last_term = -1, last_ops = -1;
+
+        nsr_driver_log (this->name, GF_LOG_INFO,
+                        "getting last term info from all members of this group\n");
+        // Get last term info from all members for this group
+        send_and_wait(&status, &op_errno, -1,
+                      replica_group_size,
+                      ctx,
+                      NSR_WORK_ID_GET_LAST_TERM_INFO,
+                      NSR_RECON_QUEUE_TO_CONTROL, ctx->current_term);
+        if (status == -1)
+                return _gf_false;
+
+
+        // compare all the info received and choose the reconciliator First
+        // choose all with latest term
+        for (i=0; i < replica_group_size; i++) {
+                if (ctx->workers[i].in_use) {
+                        if (ctx->workers[i].recon_info->last_term > last_term) {
+                                last_term = ctx->workers[i].recon_info->last_term;
+                        }
+                }
+        }
+        // First choose all with latest term and highest ops
+        for (i=0; i < replica_group_size; i++) {
+                if ((ctx->workers[i].in_use) && (last_term == ctx->workers[i].recon_info->last_term)) {
+                        if (ctx->workers[i].recon_info->commited_ops > last_ops) {
+                                last_ops = ctx->workers[i].recon_info->commited_ops;
+                        }
+                }
+        }
+        // choose the first among the lot
+        for (i=0; i < replica_group_size; i++) {
+                if ((ctx->workers[i].in_use) &&
+                    (last_term == ctx->workers[i].recon_info->last_term) &&
+                    (last_ops == ctx->workers[i].recon_info->commited_ops)) {
+                    chosen = i;
+                    break;
+                }
+        }
+
+        nsr_driver_log (this->name, GF_LOG_INFO,
+                        "reconciliator chosen is %d\n", chosen);
+        ctx->reconciliator_index = chosen;
+        GF_ASSERT(chosen != -1);
+        if (chosen == -1) {
+                nsr_driver_log (this->name, GF_LOG_INFO,
+                                "no reconciliatior chosen\n");
+                return _gf_false;
+        }
+
+        // send the message to reconciliator to do reconciliation with list of
+        // nodes that are part of this quorum
+        if (chosen != 0) {
+                nsr_driver_log (this->name, GF_LOG_INFO,
+                                "sending reconciliation work to %d\n", chosen);
+                 bm = 1 << ctx->reconciliator_index;
+                 send_and_wait(&status, &op_errno, bm,
+                      replica_group_size,
+                      ctx,
+                      NSR_WORK_ID_RECONCILIATOR_DO_WORK,
+                      NSR_RECON_QUEUE_TO_CONTROL, -1);
+                if (status == -1)
+                      return _gf_false;
+                nsr_driver_log (this->name, GF_LOG_INFO,
+                                "finished reconciliation work to %d\n", chosen);
+        } else {
+                nsr_driver_log (this->name, GF_LOG_INFO,
+                                "local node is reconciliator. before set jmp\n");
+                nsr_recon_driver_reconciliator(priv);
+        }
+
+        // send message to all other nodes to sync up with the reconciliator
+        // including itself if required
+        // requires optimisation - TBD
+        if (chosen != 0) {
+                nsr_driver_log (this->name, GF_LOG_INFO,
+                                "local node resolution needs to be done. before set jmp\n");
+                nsr_recon_driver_resolutor(priv);
+        }
+
+        nsr_driver_log (this->name, GF_LOG_INFO,
+                        "sending resolution work to all nodes except this node and reconciliator\n");
+        bm = ~((1 << ctx->reconciliator_index) || 1);
+        send_and_wait(&status, &op_errno, bm,
+                      replica_group_size,
+                      ctx,
+                      NSR_WORK_ID_RESOLUTION_DO_WORK,
+                      NSR_RECON_QUEUE_TO_CONTROL, -1);
+        if (status == -1)
+                return _gf_false;
+
+        nsr_driver_log (this->name, GF_LOG_INFO,
+                        "finished reconciliation work as leader \n");
+        return _gf_true;
+}
+
+// main recon driver thread
+void *
+nsr_reconciliation_driver(void *arg)
+{
+        nsr_recon_private_t *priv = (nsr_recon_private_t *) arg;
+	uint32_t replica_group_size = priv->replica_group_size;
+        uint32_t i;
+        nsr_per_node_worker_t *control_s, *data_s;
+        nsr_recon_driver_ctx_t **driver_ctx, *ctx;
+        int32_t bm;
+        xlator_t *this = priv->this;
+        char *con_name, *data_name;
+	int32_t status = 0;
+	int32_t op_errno = 0;
+
+        driver_ctx = &priv->driver_thread_context;
+        (*driver_ctx) = GF_CALLOC (1,
+                                   sizeof (nsr_recon_driver_ctx_t),
+                                   gf_mt_recon_driver_ctx_t);
+        if (!driver_ctx) {
+                gf_log (this->name, GF_LOG_ERROR, "memory allocation error \n");
+                return NULL;
+        }
+        ctx = *driver_ctx;
+        ctx->this = priv->this;
+        ctx->replica_group_size = replica_group_size;
+
+        ctx->fp = recon_create_log (priv->replica_group_members[0], "nsr-driver-log");
+        if (!ctx->fp)
+                return NULL;
+
+        if ((pthread_mutex_init(&(ctx->mutex), NULL)) ||
+            (pthread_cond_init(&(ctx->cv), NULL))){
+                nsr_driver_log (this->name, GF_LOG_ERROR, "mutex init error \n");
+                return NULL;
+        }
+        INIT_LIST_HEAD(&(ctx->role_head.list));
+
+        ctx->workers = RD_CALLOC (replica_group_size,
+                                  sizeof(nsr_replica_worker_t),
+                                  gf_mt_recon_worker_t);
+        if (!ctx->workers) {
+                nsr_driver_log (this->name, GF_LOG_ERROR, "memory allocation error \n");
+                return NULL;
+        }
+        for (i=0; i < replica_group_size; i++) {
+		strcpy(ctx->workers[i].name, priv->replica_group_members[i]);
+        }
+
+        control_s = RD_CALLOC (replica_group_size,
+                               sizeof(nsr_per_node_worker_t),
+                               gf_mt_recon_per_node_worker_t);
+        if (!control_s) {
+                nsr_driver_log (this->name, GF_LOG_ERROR, "memory allocation error \n");
+                return NULL;
+        }
+
+        data_s = RD_CALLOC (replica_group_size,
+                            sizeof(nsr_per_node_worker_t),
+                            gf_mt_recon_per_node_worker_t);
+        if (!data_s) {
+                nsr_driver_log (this->name, GF_LOG_ERROR, "memory allocation error \n");
+                return NULL;
+        }
+        for (i=0; i < replica_group_size; i++) {
+                ctx->workers[i].control_worker = &control_s[i];
+                if (asprintf(&con_name,"recon-con-%u",i) < 1) {
+                        return NULL;
+                }
+                ctx->workers[i].control_worker->fp = recon_create_log
+                                (priv->replica_group_members[0], con_name);
+                if (!ctx->workers[i].control_worker->fp)
+                        return NULL;
+                ctx->workers[i].data_worker = &data_s[i];
+                if (asprintf (&data_name,"recon-data-%u",i) <1) {
+                        return NULL;
+                }
+                ctx->workers[i].data_worker->fp = recon_create_log
+                                (priv->replica_group_members[0], data_name);
+                if (!ctx->workers[i].data_worker->fp)
+                        return NULL;
+       }
+
+         nsr_driver_log (this->name, GF_LOG_INFO, "creating threads \n");
+        // Create the worker threads
+        // For every brick including itself there will be 2 worker threads:
+        // one for data and one for control
+        if (!create_worker_threads(priv, ctx, control_s, _gf_true,
+                                   (F_t) control_worker_main, replica_group_size) ||
+            !create_worker_threads(priv, ctx, data_s, _gf_false,
+                                   (F_t) data_worker_main, replica_group_size)) {
+                return NULL;
+        }
+
+        for (i=0; i < replica_group_size; i++) {
+                nsr_recon_get_file(priv->volname, &(ctx->workers[i]));
+        }
+
+        while (1) {
+
+            nsr_role_work_t *rr;
+
+            nsr_driver_log (this->name, GF_LOG_INFO, "waiting for role to be queued \n");
+            pthread_mutex_lock(&(ctx->mutex));
+            while (list_empty(&(ctx->role_head.list))) {
+                  pthread_cond_wait(&(ctx->cv), &(ctx->mutex));
+            }
+            pthread_mutex_unlock(&(ctx->mutex));
+
+            list_for_each_entry(rr, &(ctx->role_head.list), list) {
+                nsr_recon_driver_state_t state;
+                state = nsr_recon_driver_get_role(&status, ctx, rr);
+
+		if (status == -1) {
+			op_errno = EIO;
+			goto out;
+		}
+
+                switch (state) {
+
+                case leader:
+                        if (!nsr_recon_driver_leader(priv)) {
+                                goto out;
+                        }
+                        break;
+                case reconciliator:
+                        if (!nsr_recon_driver_reconciliator(priv)) {
+                                goto out;
+                        }
+                        break;
+                case resolutor:
+                        if (!nsr_recon_driver_resolutor(priv)) {
+                                goto out;
+                        }
+                        break;
+
+                case joiner:
+
+                        nsr_driver_log (this->name, GF_LOG_INFO, "getting last term info from all members of this group\n");
+                        // Get last term info from all members for this group
+                        // which will be the leader(this node) and the node that wants to join.
+                        send_and_wait(&status, &op_errno, -1,
+                                      replica_group_size,
+                                      ctx,
+                                      NSR_WORK_ID_GET_LAST_TERM_INFO,
+                                      NSR_RECON_QUEUE_TO_CONTROL, ctx->current_term);
+			if (status == -1)
+				goto out;
+
+
+                        // send message to other node that just joined to sync up with this node which is also the leader
+                        nsr_driver_log (this->name, GF_LOG_INFO, "sending resolution work to all nodes except this\n");
+                        bm = ~(1);
+                        send_and_wait(&status, &op_errno, bm,
+                                      replica_group_size,
+                                      ctx,
+                                      NSR_WORK_ID_RESOLUTION_DO_WORK,
+                                      NSR_RECON_QUEUE_TO_CONTROL, -1);
+			if (status == -1)
+				goto out;
+
+                        nsr_driver_log (this->name, GF_LOG_INFO,
+                                        "finished recon work as joiner \n");
+                        break;
+
+                default:
+                        nsr_driver_log (this->name, GF_LOG_ERROR,
+                                        "bad state %d", state);
+                }
+
+
+                // free the asasociated recon_info contexts created as part of this role
+
+out:
+                nsr_driver_log (this->name, GF_LOG_INFO,
+                               "sending end of reconciliation message \n");
+                nsr_recon_return_back(priv, ctx->term, status, op_errno);
+                nsr_driver_log (this->name, GF_LOG_INFO,
+                               "finished sending end of reconciliation message \n");
+           }
+           list_del_init (&rr->list);
+        }
+
+        return NULL;
+}
diff --git a/xlators/cluster/nsr-recon/src/recon_driver.h b/xlators/cluster/nsr-recon/src/recon_driver.h
new file mode 100644
index 000000000..3efb26269
--- /dev/null
+++ b/xlators/cluster/nsr-recon/src/recon_driver.h
@@ -0,0 +1,325 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __RECON_DRIVER_H__
+#define __RECON_DRIVER_H__
+
+
+#include "api/src/glfs.h"
+
+#define MAX_HOSTNAME_LEN 32
+#define MAXIMUM_REPLICA_STRENGTH 8
+#define MAX_RECONCILIATION_WINDOW_SIZE 10000
+
+#define GLUSTERD_DEFAULT_WORKDIR "/var/lib/glusterd"
+#define GLUSTERD_VOLUME_DIR_PREFIX "vols"
+#define GLUSTERD_BRICK_INFO_DIR "bricks"
+
+/*
+ * Even with the names fixed, the non-NSR_DEBUG definitions of nsr_*_log don't
+ * work because many callers don't have "this" defined.
+ *
+ * TBD: use gf_log, fix "this" problem, eliminate extra fields and newlines.
+ */
+#define NSR_DEBUG
+
+typedef enum nsr_recon_work_req_id_t {
+       NSR_WORK_ID_GET_NONE = 0,
+       NSR_WORK_ID_GET_LAST_TERM_INFO = NSR_WORK_ID_GET_NONE + 1,
+       NSR_WORK_ID_GET_GIVEN_TERM_INFO = NSR_WORK_ID_GET_LAST_TERM_INFO + 1,
+       NSR_WORK_ID_RECONCILIATOR_DO_WORK = NSR_WORK_ID_GET_GIVEN_TERM_INFO + 1,
+       NSR_WORK_ID_RESOLUTION_DO_WORK = NSR_WORK_ID_RECONCILIATOR_DO_WORK + 1,
+       NSR_WORK_ID_GET_RECONCILATION_WINDOW = NSR_WORK_ID_RESOLUTION_DO_WORK + 1,
+       NSR_WORK_ID_SINGLE_RECONCILIATION_READ = NSR_WORK_ID_GET_RECONCILATION_WINDOW + 1,
+       NSR_WORK_ID_SINGLE_RECONCILIATION_COMMIT = NSR_WORK_ID_SINGLE_RECONCILIATION_READ + 1,
+       NSR_WORK_ID_SINGLE_RECONCILIATION_FLUSH = NSR_WORK_ID_SINGLE_RECONCILIATION_COMMIT + 1,
+       NSR_WORK_ID_GET_RESOLUTION_WINDOW = NSR_WORK_ID_SINGLE_RECONCILIATION_FLUSH + 1,
+       NSR_WORK_ID_END_RECONCILIATION = NSR_WORK_ID_GET_RESOLUTION_WINDOW + 1,
+       NSR_WORK_ID_INI = NSR_WORK_ID_END_RECONCILIATION + 1,
+       NSR_WORK_ID_FINI = NSR_WORK_ID_INI + 1
+} nsr_recon_work_req_id_t;
+
+typedef enum nsr_recon_queue_type_t {
+       NSR_RECON_QUEUE_TO_CONTROL = 0,
+       NSR_RECON_QUEUE_TO_DATA =NSR_RECON_QUEUE_TO_CONTROL + 1,
+} nsr_recon_queue_type_t;
+
+typedef enum nsr_log_type_t {
+       NSR_LOG_HOLE = 0b0,
+       NSR_LOG_PSEUDO_HOLE = 0b1,
+       NSR_LOG_FILL = 0b11
+} nsr_log_type_t;
+
+typedef enum nsr_mode_t {
+        NSR_SEQ = 0,
+        NSR_USE_THREADS = 1,
+        NSR_ASYNC = 2
+} nsr_mode_t;
+
+typedef enum nsr_recon_work_type_t {
+       NSR_RECON_WORK_NONE = 0,
+       NSR_RECON_WORK_HOLE_TO_NOOP = NSR_RECON_WORK_NONE + 1,
+       NSR_RECON_WORK_HOLE_TO_PSEUDO_HOLE = NSR_RECON_WORK_HOLE_TO_NOOP + 1,
+       NSR_RECON_WORK_COMPARE_PSEUDO_HOLE = NSR_RECON_WORK_HOLE_TO_PSEUDO_HOLE + 1,
+       NSR_RECON_WORK_HOLE_TO_FILL = NSR_RECON_WORK_COMPARE_PSEUDO_HOLE + 1,
+       NSR_RECON_WORK_UNDO_FILL = NSR_RECON_WORK_HOLE_TO_FILL + 1,
+} nsr_recon_work_type_t;
+
+typedef enum nsr_recon_driver_state_t {
+        none = 0,
+        leader = 1,
+        reconciliator = 2,
+        resolutor = 3,
+        joiner = 4,
+} nsr_recon_driver_state_t;
+
+// role structure
+#pragma pack(push, 1)
+typedef struct _nsr_recon_role_s {
+         uint32_t role; // leader, reconciliator, resolutor
+         uint32_t num; // required in case state is reconciliator
+         uint32_t current_term; // current term used in case of leader
+         // In case this is reconciliator, num is set to nodes that were part
+         // of previous term.
+         // In case this is resolutor, num is set to 2.
+         // info[0] - information for this node.
+         // info[1] - information of the reconciliator.
+         // In case this is leader, num is set to this term's membership list
+         // set info.name to all members including the leader
+         struct {
+                int32_t last_term;
+                int32_t commited_ops;
+                uint32_t last_index;
+                uint32_t first_index;
+                char name[MAX_HOSTNAME_LEN];
+        } info[MAXIMUM_REPLICA_STRENGTH];
+} nsr_recon_role_t;
+#pragma pack(pop)
+
+#define ENDIAN_CONVERSION_RR(rr, is_true) \
+{ \
+        uint32_t i=0; \
+        uint32_t (*f)(uint32_t) = ((is_true == _gf_true) ? ntohl : htonl); \
+        if (is_true == _gf_true) rr.num = f(rr.num); \
+        rr.current_term = f(rr.current_term); \
+        for (i=0; i < rr.num; i++) { \
+                rr.info[i].last_term = f(rr.info[i].last_term); \
+                rr.info[i].commited_ops = f(rr.info[i].commited_ops); \
+                rr.info[i].last_index = f(rr.info[i].last_index); \
+                rr.info[i].first_index = f(rr.info[i].first_index); \
+        } \
+        if (is_true == _gf_false) rr.num = f(rr.num); \
+}
+
+//  last term info structure
+#pragma pack(push, 1)
+typedef struct _nsr_recon_last_term_info_s {
+        int32_t last_term;
+        int32_t commited_ops;
+        uint32_t last_index;
+        uint32_t first_index;
+} nsr_recon_last_term_info_t;
+#pragma pack(pop)
+
+#define ENDIAN_CONVERSION_LT(lt, is_true) \
+{ \
+        uint32_t (*f)(uint32_t) = ((is_true == _gf_true) ? ntohl : htonl); \
+        lt.last_term = f(lt.last_term); \
+        lt.commited_ops = f(lt.commited_ops); \
+        lt.last_index = f(lt.last_index); \
+        lt.first_index = f(lt.first_index); \
+}
+
+// log information
+#pragma pack(push, 1)
+typedef struct _nsr_recon_log_info_s {
+        uint32_t term;
+        uint32_t first_index;
+        uint32_t last_index;
+} nsr_recon_log_info_t;
+#pragma pack(pop)
+
+#define ENDIAN_CONVERSION_LI(li, is_true) \
+{ \
+        uint32_t (*f)(uint32_t) = ((is_true == _gf_true) ? ntohl : htonl); \
+        li.term = f(li.term); \
+        li.first_index = f(li.first_index); \
+        li.last_index = f(li.last_index); \
+}
+
+#pragma pack(push, 1)
+typedef struct nsr_recon_record_details_s {
+       uint32_t type;
+       uint32_t op;
+       char gfid[36+1];
+       char pargfid[36+1];
+       char link_path[256]; // should it be PATH_MAX?
+       uint32_t offset;
+       uint32_t len;
+       char entry[128];
+       char newloc[128]; // for rename. can you overload link_path for this? TBD
+       mode_t mode;
+} nsr_recon_record_details_t;
+#pragma pack(pop)
+
+#define ENDIAN_CONVERSION_RD(rd, is_true) \
+{ \
+        uint32_t (*f)(uint32_t) = ((is_true == _gf_true) ? ntohl : htonl); \
+        rd.type = f(rd.type); \
+        rd.op = f(rd.op); \
+        rd.offset = f(rd.offset); \
+        rd.len = f(rd.len); \
+}
+
+typedef struct _nsr_role_work_s {
+        nsr_recon_role_t role;
+        uint32_t term;
+        struct list_head list;
+} nsr_role_work_t;
+        
+typedef struct _nsr_recon_work_s {
+	gf_boolean_t in_use;
+        uint32_t index;
+        uint32_t req_id;
+        struct list_head list;
+} nsr_recon_work_t;
+
+typedef struct _nsr_reconciliation_work_s {
+       uint32_t term;
+       uint32_t index;
+       uint32_t type;
+       uint32_t source;
+       void *data;
+
+       uint32_t num; // used for xattr
+
+} nsr_reconciliation_work_t;
+
+typedef struct _nsr_reconciliation_record_s {
+        nsr_reconciliation_work_t work; // will store the computed work
+        nsr_recon_record_details_t rec;
+} nsr_reconciliation_record_t;
+
+typedef struct _nsr_reconciliator_info {
+        uint32_t reconcilator_index;
+        int32_t last_term;
+        int32_t commited_ops;
+        uint32_t last_index;
+        uint32_t first_index;
+        //nsr_reconciliation_record_t records[MAX_RECONCILIATION_WINDOW_SIZE];
+        nsr_reconciliation_record_t *records;
+} nsr_reconciliator_info_t;
+
+typedef struct _nsr_per_node_worker_s {
+       char *id; // identifier
+       char vol_file[256]; //volfile that will be used by this thread
+       glfs_t *fs;
+       glfs_fd_t *aux_fd;
+       uint32_t index; // index into array of workers
+       pthread_t thread_id; // thread id
+       void * context; // thread context
+       struct _nsr_recon_driver_ctxt *driver_ctx;
+       char local; // local data worker
+       //struct list_head  list; //list of work items
+       nsr_recon_work_t head;
+       pthread_mutex_t mutex; //mutex to guard the state
+       pthread_cond_t cv; //condition variable for signaling the worker thread
+       gf_boolean_t is_control;
+#if defined(NSR_DEBUG)
+       FILE *fp;
+#endif
+       int32_t result; // result of latest work
+       int32_t op_errno; // errno
+} nsr_per_node_worker_t;
+
+typedef struct _nsr_replica_worker_s {
+       char name[256];
+       nsr_per_node_worker_t *control_worker;
+       nsr_per_node_worker_t *data_worker;
+       gf_boolean_t in_use;
+       nsr_reconciliator_info_t *recon_info; // Bunch of infos kept for this reconciliation
+} nsr_replica_worker_t;
+
+typedef struct _nsr_recon_driver_ctxt {
+       xlator_t *this;
+       uint32_t replica_group_size; // number of static members of replica group
+       nsr_replica_worker_t *workers; // worker info
+       int32_t reconciliator;
+       pthread_mutex_t mutex; 
+       pthread_cond_t cv;
+       nsr_role_work_t role_head;
+       volatile int32_t outstanding;
+       uint32_t reconciliator_index;
+       uint32_t term;
+       uint32_t current_term;
+       nsr_mode_t mode; // default set to seq
+#if defined(NSR_DEBUG)
+       FILE *fp;
+#endif
+} nsr_recon_driver_ctx_t;
+
+void *
+nsr_reconciliation_driver(void *);
+
+gf_boolean_t
+nsr_recon_driver_set_role(nsr_recon_driver_ctx_t *ctx, nsr_recon_role_t *rr, uint32_t term);
+
+#define atomic_inc(ptr)        ((void) __sync_fetch_and_add(ptr, 1))
+#define atomic_dec(ptr)        ((void) __sync_fetch_and_add(ptr, -1))
+#define atomic_fetch_and       __sync_fetch_and_and
+#define atomic_fetch_or        __sync_fetch_and_or
+
+#if defined(NSR_DEBUG)
+
+#define NSR_LOG_DIR "/var/log/nsr-logs"
+
+extern int      nsr_debug_level;
+extern FILE     *recon_create_log (char *member, char *module);
+
+extern void
+_nsr_driver_log (const char *func, int line, char *member, FILE *fp,
+                 char *fmt, ...);
+
+#define nsr_driver_log(dom, levl, fmt...) do {                          \
+        FMT_WARN (fmt);                                                 \
+        if (levl <= nsr_debug_level) {                                  \
+                nsr_recon_private_t     *priv   = ctx->this->private;   \
+                _nsr_driver_log (__FUNCTION__, __LINE__,                \
+                                   priv->replica_group_members[0],      \
+                                   ctx->fp,                             \
+                                   ##fmt);                              \
+        }                                                               \
+} while (0)
+
+extern void
+_nsr_worker_log (const char *func, int line, char *member,
+                 char *type, uint32_t index, FILE *fp,
+                 char *fmt, ...);
+
+#define nsr_worker_log(dom, levl, fmt...) do {                          \
+        FMT_WARN (fmt);                                                 \
+        if (levl <= nsr_debug_level) {                                  \
+                nsr_recon_private_t *priv;                              \
+                priv = ctx->driver_ctx->this->private;                  \
+                _nsr_worker_log (__FUNCTION__, __LINE__,                \
+                                 priv->replica_group_members[0],        \
+                                 ctx->is_control ? "recon-con" :        \
+                                 "recon-data",                          \
+                                 ctx->index, ctx->fp,                   \
+                                 ##fmt);                                \
+        }                                                               \
+} while (0)
+
+#else
+#define nsr_driver_log(dom, levl, fmt...) gf_log(dom, levl, fmt)
+#define nsr_worker_log(dom, levl, fmt...) gf_log(dom, levl, fmt)
+#endif
+
+#endif /* #ifndef __RECON_DRIVER_H__ */
diff --git a/xlators/cluster/nsr-recon/src/recon_xlator.c b/xlators/cluster/nsr-recon/src/recon_xlator.c
new file mode 100644
index 000000000..272c35dc2
--- /dev/null
+++ b/xlators/cluster/nsr-recon/src/recon_xlator.c
@@ -0,0 +1,1010 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <sys/types.h>
+#include <fcntl.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "call-stub.h"
+#include "defaults.h"
+#include "xlator.h"
+
+#include "recon_driver.h"
+#include "recon_xlator.h"
+
+typedef struct _nsr_recon_fd_s {
+        int32_t term;
+        nsr_recon_driver_state_t state;
+        uint32_t first_index;
+        uint32_t last_index;
+        call_frame_t *frame;
+} nsr_recon_fd_t;
+
+#if defined(NSR_DEBUG)
+
+void
+_recon_main_log (const char *func, int line, char *member, FILE *fp,
+                 char *fmt, ...)
+{
+        va_list         ap;
+        char            *buf    = NULL;
+        int             retval;
+
+        if (!fp) {
+                fp = recon_create_log(member,"recon-main-log");
+                if (!fp) {
+                        return;
+                }
+        }
+
+        va_start(ap,fmt);
+        retval = vasprintf(&buf,fmt,ap);
+        if (buf) {
+                fprintf(fp,"[%s:%d] %.*s\n",func,line,retval,buf);
+                free(buf);
+        }
+        va_end(ap);
+}
+
+#endif
+
+// Given fd, get back the NSR based fd context.
+static int32_t this_fd_ctx_get(fd_t *fd, xlator_t *this, nsr_recon_fd_t **rfd)
+{
+        uint64_t tmp = 0;
+        int32_t ret = -1;
+
+        if ((ret = fd_ctx_get(fd, this, &tmp)) != 0) {
+                return ret;
+        } else {
+                *rfd = (nsr_recon_fd_t *)tmp;
+                return 0;
+        }
+}
+
+// Add the frame in q after associating with term
+// term usage tbd
+static void put_frame(nsr_recon_private_t *priv,
+                      call_frame_t *frame,
+                      uint32_t term)
+{
+	xlator_t *this = priv->this;
+        recon_main_log (this->name, GF_LOG_INFO, "adding frame for term %d \n", term);
+	priv->frame = frame;
+	return;
+}
+
+// get the frame from the queue given the term
+// term usage tbd
+static void get_frame(nsr_recon_private_t *priv,
+                      call_frame_t **frame,
+                      uint32_t term)
+{
+	if (frame != NULL)
+		*frame = priv->frame;
+	priv->frame = NULL;
+	return;
+}
+
+// check if there are outstanding frames
+static gf_boolean_t is_frame(nsr_recon_private_t *priv)
+{
+	return((priv->frame != NULL) ? _gf_true : _gf_false);
+}
+
+#define ENTRY_SIZE 128
+
+long
+get_entry_count (char *path)
+{
+        int             fd;
+        struct stat     buf;
+        unsigned long   entries         = -1;
+        long            min;            /* last entry not known to be empty */
+        long            max;            /* first entry known to be empty */
+        long            curr;
+        char            entry[ENTRY_SIZE];
+        void            *err_label      = &&done;
+
+        fd = open(path,O_RDONLY);
+        if (fd < 0) {
+                goto *err_label;
+        }
+        err_label = &&close_fd;
+
+        if (fstat(fd,&buf) < 0) {
+                goto *err_label;
+        }
+
+        min = 0;
+        max = buf.st_size / ENTRY_SIZE;
+        printf("max = %ld\n",max);
+
+        while ((min+1) < max) {
+                curr = (min + max) / 2;
+                printf("trying entry %ld\n",curr);
+                if (lseek(fd,curr*ENTRY_SIZE,SEEK_SET) < 0) {
+                        goto *err_label;
+                }
+                if (read(fd,entry,sizeof(entry)) != sizeof(entry)) {
+                        goto *err_label;
+                }
+                if ((entry[0] == '_') && (entry[1] == 'P')) {
+                        min = curr;
+                }
+                else {
+                        max = curr;
+                }
+        }
+
+        entries = max;
+
+close_fd:
+        close(fd);
+done:
+        return entries;
+}
+
+// Get the term info for the term number specified
+void nsr_recon_libchangelog_get_this_term_info(xlator_t *this, char *bp, int32_t term, nsr_recon_last_term_info_t *lt)
+{
+        char path[PATH_MAX];
+        long entries;
+
+        bzero(lt, sizeof(nsr_recon_last_term_info_t));
+        lt->last_term = term;
+        sprintf(path,"%s/%s%d",bp,"TERM.",term);
+        entries = get_entry_count(path);
+        if (entries > 1) {
+                /* The first entry is actually a header. */
+                lt->first_index = 1;
+                /*
+                 * This seems wrong, because it means that last_index*128 will
+                 * be exactly at EOF and commited_ops will be one greater than
+                 * it should be.  Maybe some other code makes the exact
+                 * opposite mistake to compensate.
+                 */
+                lt->last_index = lt->commited_ops = (int)entries;
+        }
+        recon_main_log (this->name, GF_LOG_INFO, "for term=%d got first_index=%d last_index=%d commited_ops=%d\n",
+                        term, lt->first_index, lt->last_index, lt->commited_ops);
+        return;
+}
+
+// Given the term number, find the last term in the changelogs
+void nsr_recon_libchangelog_get_last_term_info(xlator_t *this, char *bp, int32_t term, nsr_recon_last_term_info_t *lt)
+{
+        uint32_t t = term;
+        struct stat buf;
+        char path[PATH_MAX];
+        bzero(lt, sizeof(nsr_recon_last_term_info_t));
+        while(t) {
+                // journal file is of type TERM-1.jnl
+                sprintf(path,"%s/%s%d",bp,"TERM.",t);
+                if (!stat(path, &buf)) {
+                        nsr_recon_libchangelog_get_this_term_info(this, bp, t, lt);
+                        recon_main_log (this->name, GF_LOG_INFO, "got last term given current term %d as %d\n", term, t);
+                        return;
+                }
+                t--;
+        }
+        recon_main_log (this->name, GF_LOG_INFO, "got no last term given current term %d \n", term);
+
+        return;
+}
+
+// Return back the frame stored against the term
+void nsr_recon_return_back(nsr_recon_private_t *priv, uint32_t term, int32_t status, int32_t op_errno)
+{
+        call_frame_t *old_frame = NULL;
+        xlator_t *this = priv->this;
+
+        get_frame(priv, &old_frame, term);
+        if (old_frame) {
+                recon_main_log (this->name, GF_LOG_INFO, "nsr_recon_writev returns old frame \n");
+                                // first return the original write for which this ack was sent
+                STACK_UNWIND_STRICT (writev, old_frame, status, op_errno, NULL, NULL, NULL);
+        } else {
+                recon_main_log (this->name, GF_LOG_ERROR, "EIII---nsr_recon_writev cnnot return old frame \n");
+        }
+}
+
+typedef enum records_type_t {
+        fop_gfid_pgfid_oldloc_newloc = 1,
+        fop_gfid_pgfid_entry = fop_gfid_pgfid_oldloc_newloc + 1,
+        fop_gfid = fop_gfid_pgfid_entry + 1 ,
+        fop_gfid_offset = fop_gfid + 1,
+        fop_gfid_offset_len = fop_gfid_offset + 1,
+} records_type_t;
+
+// Get the backend ./glusterfs/xx/xx/<...> path
+static void
+get_gfid_path(nsr_recon_private_t *priv, char *gfid, char *path)
+{
+        strcpy(path, priv->base_dir);
+        strcat(path, "/.glusterfs/");
+        strncat(path,gfid,2);
+        strcat(path,"/");
+        strncat(path,gfid+2,2);
+        strcat(path,"/");
+        strcat(path,gfid);
+}
+
+
+// Get the link to which backend points to
+static gf_boolean_t
+get_link_using_gfid(nsr_recon_private_t *priv, char *gfid, char *path)
+{
+        char lp[PATH_MAX];
+        xlator_t *this = priv->this;
+        get_gfid_path(priv,gfid, lp);
+        if (readlink(lp, path, 255) == -1) {
+                GF_ASSERT(0);
+                recon_main_log(priv->this, GF_LOG_ERROR,
+                               "cannot get readlink  for %s\n",lp);
+                return _gf_false;
+        }
+        return _gf_true;
+}
+
+// Get the list of changelog records given a term , first and last index.
+//
+// TBD: rewrite this hideous ball of mud in at least the following ways:
+//
+//      (1) Break out the code for handling a single record into a separate
+//          function, to make error handling easier and reduce "indentation
+//          creep" so the code's readable.
+//
+//      (2) Change all of the fop_xxx_yyy nonsense to OR together values
+//          like FOP_HAS_FIELD_XXX and FOP_HAS_FIELD_YYY, to reduce code
+//          duplication and facilitate the addition of new fields.
+//
+//      (3) Stop making so many assumptions about the underlying formats.
+//          The code as it is won't even work for the existing binary format,
+//          let alone as changelog evolves over time.
+//
+// Really, 90% of this code should just GO AWAY in favor of using
+// libgfchangelog, enhanced as necessary to support our needs.
+
+/*
+ * Use this macro to skip over a field we're not using yet.
+ * NB: the body is a null statement on purpose
+ * TBD: all instances of this should be removed eventually!
+ */
+#define SKIP_FIELD do /* nothing */ ; while (*(start++) != '\0')
+
+#define SKIP_OVER
+gf_boolean_t nsr_recon_libchangelog_get_records(xlator_t *this, char *bp, int32_t term, uint32_t first, uint32_t last, void *buf)
+{
+        // do a mmap; seek into the first and read all records till last.
+        // TBD - right now all records are pseudo holes but mark them as fills.
+        // TBD - pseudo hole to be implemented when actual fsync gets done on data.
+        char *rb = NULL, *orig = NULL;
+        char path[PATH_MAX];
+        int fd;
+        uint32_t index = 0;
+
+        recon_main_log (this->name, GF_LOG_INFO,
+                        "libchangelog_get_records called for term %d  index from %d to %d \n",
+                        term, first, last );
+
+        orig = rb = GF_CALLOC(128, ((last - first) + 1),
+                              gf_mt_recon_changelog_buf_t);
+
+        sprintf(path,"%s/%s%d",bp,"TERM.",term);
+        fd = open(path, O_RDONLY);
+        if (fd == -1) {
+		return _gf_false;
+	} else {
+                char *start = NULL;
+                nsr_recon_record_details_t * rec = (nsr_recon_record_details_t *)buf;
+
+                if (first == 0)
+                        lseek(fd, 128, SEEK_SET);
+                else
+                        lseek(fd, first * 128, SEEK_SET);
+                if (read(fd, rb, (last - first  + 1) * 128) == -1) {
+			return _gf_false;
+		}
+                start = rb;
+                index = first;
+                do {
+                        recon_main_log (this->name, GF_LOG_INFO,
+                                        "libchangelog_get_records start inspecting records at index %d \n",
+                                        index );
+                        if (!strncmp(start, "_PRE_", 5)) {
+                                uint32_t i;
+                                uint32_t opcode = 0;
+                                records_type_t type;
+
+                                start += 5;
+                                // increment by the NULLs after the PRE
+                                start += 4;
+                                SKIP_FIELD;     // real index
+                                // now we have the opcode
+                                while (*start != '\0') {
+                                        opcode *= 10;
+                                        opcode += (*(start++) - '0');
+                                }
+                                ++start;
+                                recon_main_log (this->name, GF_LOG_ERROR,
+                                                "libchangelog_get_records: got opcode %d @index %d\n", opcode, index);
+                                if ((opcode == GF_FOP_RENAME)) {
+                                        type = fop_gfid_pgfid_oldloc_newloc;
+                                } else if ((opcode == GF_FOP_UNLINK) ||
+                                    (opcode == GF_FOP_RMDIR) ||
+                                    (opcode == GF_FOP_LINK) ||
+                                    (opcode == GF_FOP_MKDIR) ||
+                                    (opcode == GF_FOP_SYMLINK) ||
+                                    (opcode == GF_FOP_MKNOD) ||
+                                    (opcode == GF_FOP_CREATE)) {
+                                        type = fop_gfid_pgfid_entry;
+                                } else if ((opcode == GF_FOP_FSETATTR) ||
+                                    (opcode == GF_FOP_SETATTR) ||
+                                    (opcode == GF_FOP_FREMOVEXATTR) ||
+                                    (opcode == GF_FOP_REMOVEXATTR) ||
+                                    (opcode == GF_FOP_SETXATTR) ||
+                                    (opcode == GF_FOP_FSETXATTR)) {
+                                        type = fop_gfid;
+                                } else if ((opcode == GF_FOP_TRUNCATE) ||
+                                    (opcode == GF_FOP_FTRUNCATE)) {
+                                        type = fop_gfid_offset;
+                                } else if (opcode == GF_FOP_WRITE) {
+                                        type = fop_gfid_offset_len;
+                                } else {
+                                        recon_main_log (this->name,
+                                                        GF_LOG_ERROR,
+                                                        "libchangelog_get_records:got no proper opcode %d @index %d\n",
+                                                        opcode, index);
+                                        //GF_ASSERT(0);
+                                        // make this as a hole.
+                                        // TBD - check this logic later. maybe we should raise alarm here because
+                                        // this means that changelog is corrupted. We are not handling changelog
+                                        // corruptions as of now.
+                                        rec->type = NSR_LOG_HOLE;
+                                        goto finish;
+                                }
+                                // TBD - handle psuedo holes once that logic is in.
+                                rec->type = NSR_LOG_FILL;
+                                recon_main_log (this->name, GF_LOG_ERROR,
+                                               "libchangelog_get_records:got type %d at index %d \n",
+                                                rec->type, index);
+                                rec->op = opcode;
+
+                                // Now get the gfid and parse it
+                                // before that increment the pointer
+                                for (i=0; i < 36; i++) {
+                                        rec->gfid[i] = (*start);
+                                        start++;
+                                }
+                                rec->gfid[i] = '\0';
+
+                                GF_ASSERT(*start == 0);
+                                start ++;
+
+                                if (opcode == GF_FOP_SYMLINK) {
+                                        i = 0;
+                                        do {
+                                                if (i >= 256) {
+                                                        goto finish;
+                                                }
+                                                rec->link_path[i++] = *start;
+                                        } while (*(start++) != '\0');
+                                }
+
+                                i = 0;
+                                // If type is fop_gfid_offset+_len, get offset
+                                if ((type == fop_gfid_offset) || (type == fop_gfid_offset_len)) {
+                                        char offset_str[128];
+                                        while(*start != 0) {
+                                                offset_str[i++] = *start;
+                                                start ++;
+                                        }
+                                        offset_str[i] = '\0';
+                                        // get over the 0
+                                        start++;
+                                        rec->offset = strtoul(offset_str, NULL, 10);
+                                        recon_main_log (this->name,
+                                                        GF_LOG_ERROR,
+                                                        "libchangelog_get_records:got offset %d @index %d \n", rec->offset, index);
+
+                                }
+                                i = 0;
+                                if (type == fop_gfid_offset_len) {
+                                        char len_str[128];
+                                        while(*start != 0) {
+                                                len_str[i++] = *start;
+                                                start ++;
+                                        }
+                                        len_str[i] = '\0';
+                                        // get over the 0
+                                        start++;
+                                        rec->len = strtoul(len_str, NULL, 10);
+                                        recon_main_log (this->name,
+                                                        GF_LOG_ERROR,
+                                                        "libchangelog_get_records:got length %d @index %d \n", rec->len, index);
+                                }
+                                i = 0;
+                                if (type == fop_gfid_pgfid_entry) {
+                                        switch (opcode) {
+                                        case GF_FOP_CREATE:
+                                        case GF_FOP_MKDIR:
+                                        case GF_FOP_MKNOD:
+                                                SKIP_FIELD;     // mode
+                                                break;
+                                        /* TBD: handle GF_FOP_SYMLINK target */
+                                        default:
+                                                ;
+                                        }
+                                        SKIP_FIELD;             // uid
+                                        SKIP_FIELD;             // gid
+                                        if (opcode == GF_FOP_MKNOD) {
+                                                SKIP_FIELD;     // dev
+                                        }
+                                        // first get the gfid and then the path
+                                        for (i=0; i < 36; i++) {
+                                                rec->pargfid[i] = (*start);
+                                                start++;
+                                        }
+                                        rec->pargfid[i] = '\0';
+                                        GF_ASSERT(*start == '/');
+                                        start ++;
+
+                                        i = 0;
+                                        while(*start != 0) {
+                                                rec->entry[i++] = *start;
+                                                start ++;
+                                        }
+                                        rec->entry[i] = '\0';
+                                        // get over the 0
+                                        start++;
+                                        /*
+                                         * Having to add this as a special case
+                                         * is awful.  See the function header
+                                         * comment for the real solution.
+                                         */
+                                        if (opcode == GF_FOP_CREATE) {
+                                                rec->mode = 0;
+                                                while (*start != '\0') {
+                                                        rec->mode *= 10;
+                                                        rec->mode += *start
+                                                                   - '0';
+                                                        ++start;
+                                                }
+                                                ++start;
+                                        }
+                                        recon_main_log (this->name,
+                                                        GF_LOG_ERROR,
+                                                        "libchangelog_get_records:got entry %s @index %d \n", rec->entry, index);
+
+                                }
+                                i = 0;
+                                if (type == fop_gfid_pgfid_oldloc_newloc) {
+
+                                        // first get the source and then the destination
+                                        // source stuff gets stored in pargfid/entry
+                                        for (i=0; i < 36; i++) {
+                                                rec->pargfid[i] = (*start);
+                                                start++;
+                                        }
+                                        rec->pargfid[i] = '\0';
+                                        GF_ASSERT(*start == '/');
+                                        start ++;
+
+                                        i=0;
+                                        while(*start != 0) {
+                                                rec->entry[i++] = *start;
+                                                start ++;
+                                        }
+                                        rec->entry[i] = '\0';
+                                        // get over the 0
+                                        start++;
+
+                                        // dst stuff gets stored in gfid/newloc
+                                        for (i=0; i < 36; i++) {
+                                                rec->gfid[i] = (*start);
+                                                start++;
+                                        }
+                                        rec->gfid[i] = '\0';
+                                        GF_ASSERT(*start == '/');
+                                        start ++;
+                                        i = 0;
+                                        while(*start != 0) {
+                                                rec->newloc[i++] = *start;
+                                                start ++;
+                                        }
+                                        rec->newloc[i] = '\0';
+                                        // get over the 0
+                                        start++;
+
+                                }
+                                ENDIAN_CONVERSION_RD((*rec), _gf_false); //htonl
+                        }
+finish:
+                        if (index == last)
+                                break;
+                        index++;
+                        rb += 128;
+                        start = rb;
+                        rec++;
+                } while(1);
+        }
+        GF_FREE(orig);
+        close(fd);
+
+        recon_main_log (this->name, GF_LOG_INFO,
+                        "libchangelog_get_records finsihed inspecting records for  term %d \n",
+                        term);
+        return _gf_true;
+}
+
+int32_t
+nsr_recon_open (call_frame_t *frame, xlator_t *this,
+                loc_t *loc, int32_t flags, fd_t *fd, dict_t *xdata)
+{
+        int32_t op_ret       = 0;
+        int32_t op_errno     = 0;
+        nsr_recon_fd_t *rfd = NULL;
+
+        recon_main_log (this->name, GF_LOG_INFO, "nsr_recon_open called for path %s \n",loc->path );
+        rfd = GF_CALLOC (1, sizeof (*rfd), gf_mt_recon_fd_t);
+        if (!rfd) {
+                op_ret = -1;
+                op_errno = ENOMEM;
+        }
+
+        op_ret = fd_ctx_set (fd, this, (uint64_t)(long)rfd);
+        if (op_ret) {
+                op_ret = -1;
+                op_errno = EINVAL;
+        }
+        recon_main_log (this->name, GF_LOG_INFO, "nsr_recon_open returns with %d for path %s \n",op_ret,loc->path );
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, NULL);
+        return 0;
+}
+
+int32_t
+nsr_recon_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  struct iovec *vector, int32_t count, off_t offset,
+                  uint32_t flags, struct iobref *iobref, dict_t *xdata)
+{
+        nsr_recon_fd_t *rfd = NULL;
+        nsr_recon_private_t *priv = NULL;
+        int32_t op_ret = 0;
+        int32_t op_errno = 0;
+        int32_t ret = 0;
+
+        ret = this_fd_ctx_get (fd, this, &rfd);
+        if (ret < 0) {
+                return -1;
+        }
+        priv = (nsr_recon_private_t *)this->private;
+
+        recon_main_log (this->name, GF_LOG_INFO, "nsr_recon_writev called for offset %d \n",(unsigned int)offset );
+        GF_ASSERT(count == 1);
+        switch (offset) {
+                // client(brick, leader) writes the role of the node
+                case nsr_recon_xlator_sector_1 :
+                {
+                        nsr_recon_role_t rr;
+                        memcpy((void *)&rr, (void *)vector[0].iov_base, sizeof(rr));
+                        ENDIAN_CONVERSION_RR(rr, _gf_true); //ntohl
+
+                        recon_main_log (this->name, GF_LOG_INFO, "nsr_recon_writev called to set role %d\n", rr.role);
+                        if ((rr.role != leader) &&
+                            (rr.role != reconciliator) &&
+                            (rr.role != resolutor) &&
+                            (rr.role != joiner)) {
+                                recon_main_log (this->name, GF_LOG_ERROR,
+                                                "EIII---nsr_recon_writev cannot set state \n");
+                                STACK_UNWIND_STRICT (writev, frame, -1, op_errno,
+                                                     NULL, NULL, NULL);
+                        }
+
+                        GF_ASSERT(rr.num <= MAXIMUM_REPLICA_STRENGTH);
+
+			// Check if already a role play is going on. If yes return with EAGAIN.
+			// Ideally we should check if we have got a higher term number while
+			// servicing a lower term number; if so abort the older one.
+			// However the abort infrastructure needs to be sketched properly; TBD.
+			if (is_frame(priv) == _gf_true) {
+                                recon_main_log (this->name, GF_LOG_ERROR,
+                                                "nsr_recon_writev set_role - already role play \n");
+                                STACK_UNWIND_STRICT (writev, frame, -1, EAGAIN,
+                                                     NULL, NULL, NULL);
+			} else {
+
+				// Store the stack frame so that when the actual job gets finished
+				// we send the response back to the brick.
+				put_frame(priv, frame, rr.current_term);
+				if (nsr_recon_driver_set_role(priv->driver_thread_context,
+					                      &rr,
+							       rr.current_term) == _gf_false) {
+					 get_frame(priv, NULL, rr.current_term);
+					 recon_main_log (this->name, GF_LOG_ERROR,
+                                                         "nsr_recon_writev set_role - cannot seem to set role \n");
+					 STACK_UNWIND_STRICT (writev, frame, -1, op_errno,
+							       NULL, NULL, NULL);
+				} else {
+					 recon_main_log (this->name, GF_LOG_INFO,
+                                                         "nsr_recon_writev set_role - set role succesfully \n");
+				}
+			}
+                        break;
+                }
+                // client(reconciliator) writes how much it needs for the read
+                case nsr_recon_xlator_sector_2 :
+                {
+                        nsr_recon_log_info_t li;
+                        memcpy((void *)&li, (void *)vector[0].iov_base, sizeof(li));
+                        ENDIAN_CONVERSION_LI(li, _gf_true); //ntohl
+
+                        recon_main_log (this->name, GF_LOG_INFO,
+                                        "nsr_recon_writev - setting term info for reconcilation info. term=%d, first_index=%d,start_index=%d \n",
+                                        li.term, li.first_index, li.last_index);
+                        rfd->term = li.term;
+                        rfd->last_index = li.last_index;
+                        rfd->first_index = li.first_index;
+                        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno,
+                                             NULL, NULL, NULL);
+                        break;
+                }
+                // client(reconciliator) writes term for which it needs info
+                case nsr_recon_xlator_sector_3 :
+                {
+                        int32_t term;
+
+                        memcpy((void *)&term, (void *)vector[0].iov_base, sizeof(term));
+                        term = ntohl(term); //ntohl
+                        recon_main_log (this->name, GF_LOG_INFO,
+                                        "nsr_recon_writev - setting term info for term info. term=%d\n",
+                                        term);
+                        rfd->term = term;
+                        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno,
+                                             NULL, NULL, NULL);
+                        break;
+                }
+                // client(reconciliator) writes current term so that it gets last term info later
+                case nsr_recon_xlator_sector_4 :
+                {
+                        int32_t term;
+
+                        memcpy((void *)&term, (void *)vector[0].iov_base, sizeof(term));
+                        term = ntohl(term); //ntohl
+                        recon_main_log (this->name, GF_LOG_INFO,
+                                        "nsr_recon_writev - setting term info for last term info  given current term=%d\n",
+                                        term);
+                        rfd->term = term;
+                        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno,
+                                             NULL, NULL, NULL);
+                        break;
+                }
+		default:
+		{
+                        recon_main_log (this->name, GF_LOG_ERROR,
+                                       "nsr_recon_writev called with wrong offset\n");
+                        STACK_UNWIND_STRICT (writev, frame, -1, op_errno,
+                                             NULL, NULL, NULL);
+			break;
+		}
+        }
+
+        return 0;
+}
+
+int
+nsr_recon_readv (call_frame_t *frame, xlator_t *this,
+                 fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+{
+        nsr_recon_fd_t *rfd = NULL;
+        int32_t op_errno = 0;
+        // copied stuff from quick-read.c and posix.c
+        struct iobuf *iobuf = NULL;
+        struct iobref    *iobref = NULL;
+        struct iovec      iov = {0, };
+        int32_t ret = -1;
+        nsr_recon_private_t *priv = NULL;
+
+        iobuf = iobuf_get2 (this->ctx->iobuf_pool, size);
+        if (!iobuf) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        iobref = iobref_new ();
+        if (!iobref) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        iobref_add (iobref, iobuf);
+
+        ret = this_fd_ctx_get (fd, this, &rfd);
+        if (ret < 0) {
+                op_errno = -ret;
+                goto out;
+        }
+        priv = (nsr_recon_private_t *)this->private;
+
+        recon_main_log (this->name, GF_LOG_INFO, "nsr_recon_readv called for offset %d \n",(unsigned int)offset );
+        switch (offset) {
+                // client(leader) reads from here to get info for this term on this node
+                // invole libchagelog to get the information
+                case nsr_recon_xlator_sector_3 :
+                {
+                        nsr_recon_last_term_info_t lt;
+                        GF_ASSERT(size == sizeof(lt));
+                        nsr_recon_libchangelog_get_this_term_info(this,priv->changelog_base_path, rfd->term, &lt);
+                        recon_main_log (this->name, GF_LOG_INFO,
+                                        "nsr_recon_readv - getting term info for term=%d, ops=%d, first=%d, last=%d\n",
+                                        rfd->term, lt.commited_ops, lt.first_index, lt.last_index);
+                        ENDIAN_CONVERSION_LT(lt, _gf_false); //htonl
+                        memcpy(iobuf->ptr, &lt, size);
+                        goto out;
+                }
+                // client(reconciliator) reads individual record information
+                case nsr_recon_xlator_sector_2 :
+                {
+                        uint32_t num = (rfd->last_index - rfd->first_index + 1);
+                        recon_main_log (this->name, GF_LOG_INFO,
+                                        "nsr_recon_readv - expected size %lu got size %lu\n",
+                                        (num * sizeof(nsr_recon_record_details_t)), size);
+
+                        GF_ASSERT(size == (num * sizeof(nsr_recon_record_details_t)));
+                        bzero(iobuf->ptr, size);
+                        recon_main_log (this->name, GF_LOG_INFO,
+                                        "nsr_recon_readv - getting records for term=%d from %d to %d\n",
+                                        rfd->term, rfd->first_index, rfd->last_index);
+                        nsr_recon_libchangelog_get_records(this, priv->changelog_base_path,
+                                                           rfd->term, rfd->first_index, rfd->last_index, iobuf->ptr);
+                        goto out;
+                }
+                // read last term info
+                case nsr_recon_xlator_sector_4 :
+                {
+                        nsr_recon_last_term_info_t lt;
+                        GF_ASSERT(size == sizeof(lt));
+                        nsr_recon_libchangelog_get_last_term_info(this, priv->changelog_base_path, rfd->term, &lt);
+                        recon_main_log (this->name, GF_LOG_INFO,
+                                        "nsr_recon_readv - getting last term info given current term=%d. last term = %d ops=%d, first=%d, last=%d\n",
+                                        rfd->term, lt.last_term,  lt.commited_ops, lt.first_index, lt.last_index);
+                        ENDIAN_CONVERSION_LT(lt, _gf_false); //htonl
+                        memcpy(iobuf->ptr, &lt, size);
+                        goto out;
+                }
+		default:
+		{
+                        recon_main_log (this->name, GF_LOG_ERROR,
+                                       "nsr_recon_readv called with wrong offset\n");
+			op_errno = -1;
+			break;
+		}
+        }
+
+out:
+        if (op_errno == 0) {
+                iov.iov_base = iobuf->ptr;
+                ret = iov.iov_len = size;
+        }
+
+        STACK_UNWIND_STRICT (readv, frame, ret, op_errno, &iov, 1, NULL, iobref , NULL);
+
+        if (iobref)
+                iobref_unref (iobref);
+        if (iobuf)
+                iobuf_unref (iobuf);
+        return 0;
+}
+
+int
+nsr_recon_lookup (call_frame_t *frame, xlator_t *this,
+                  loc_t *loc, dict_t *xdata)
+{
+        struct iatt buf = {0, };
+        // dirty hack to set root as regular but seems to work.
+        buf.ia_type = IA_IFREG;
+        recon_main_log (this->name, GF_LOG_INFO, "nsr_recon_lookup called \n");
+
+        STACK_UNWIND_STRICT (lookup, frame, 0, 0, this->itable->root, &buf, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+nsr_recon_flush (call_frame_t *frame, xlator_t *this,
+             fd_t *fd, dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (flush, frame, 0, 0, NULL);
+        return 0;
+}
+
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("recon", this, out);
+
+        ret = xlator_mem_acct_init (this, gf_mt_recon_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Memory accounting init" "failed");
+                return ret;
+        }
+out:
+        return ret;
+}
+
+
+int32_t
+init (xlator_t *this)
+{
+        nsr_recon_private_t *priv        = NULL;
+        char  *local, *members;
+        unsigned int i=0;
+
+        priv = GF_CALLOC (1, sizeof (*priv), gf_mt_recon_private_t);
+        if (!priv) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "priv allocation error\n");
+                return -1;
+        }
+        GF_OPTION_INIT ("replica-group-size", priv->replica_group_size, uint32, err);
+        GF_OPTION_INIT ("vol-name", priv->volname, str, err);
+        if (!priv->volname) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "missing volname option (required)");
+                return -1;
+        }
+        GF_OPTION_INIT ("changelog-dir", priv->changelog_base_path, str, err);
+        if (!priv->changelog_base_path) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "missing changelog directory option (required)");
+                return -1;
+        }
+        GF_OPTION_INIT ("base-dir", priv->base_dir, str, err);
+        if (!priv->base_dir) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "missing brick base directory option (required)");
+                return -1;
+        }
+        GF_OPTION_INIT ("replica-group-members", members, str, err);
+        if (!members) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "missing membership option (required)");
+                return -1;
+        }
+        GF_OPTION_INIT ("local-member", local, str, err);
+        if (!local) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "missing local member option (required)");
+                return -1;
+        }
+
+        priv->replica_group_members = GF_CALLOC (priv->replica_group_size,
+                                                 sizeof(char *),
+                                                 gf_mt_recon_members_list_t);
+        priv->replica_group_members[0] = GF_CALLOC (1,
+                                                    strlen(local),
+                                                    gf_mt_recon_member_name_t);
+        if (!priv->replica_group_members || !(priv->replica_group_members[0])) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "str allocation error\n");
+                return -1;
+        }
+        strcpy(priv->replica_group_members[0], local);
+        for (i=1; i < priv->replica_group_size; i++) {
+                char *member;
+                if (i == 1)
+                        member = strtok(members, ",");
+                else
+                        member = strtok(NULL, ",");
+                priv->replica_group_members[i] = GF_CALLOC (1,
+                        strlen(member) + 1, gf_mt_recon_member_name_t);
+                if (!priv->replica_group_members[i]) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "str allocation error\n");
+                        return -1;
+                }
+                strcpy(priv->replica_group_members[i], member);
+        }
+
+
+        priv->this = this;
+        this->private = (void *)priv;
+
+        priv->fp = recon_create_log (priv->replica_group_members[0], "recon-main-log");
+        if (!priv->fp)
+                return -1;
+
+        recon_main_log (this->name, GF_LOG_INFO, "creating reconciliation driver \n");
+
+        if (pthread_create(&priv->thread_id, NULL, nsr_reconciliation_driver, priv)) {
+                recon_main_log (this->name, GF_LOG_ERROR,
+                        "pthread creation error \n");
+                return -1;
+        }
+
+        INIT_LIST_HEAD(&(priv->list));
+
+
+	return 0;
+
+err:
+        return -1;
+}
+
+
+void
+fini (xlator_t *this)
+{
+        nsr_recon_private_t *priv        = NULL;
+        void *ret = NULL;
+
+        priv = (nsr_recon_private_t *)this->private;
+
+        pthread_cancel(priv->thread_id);
+        pthread_join(priv->thread_id, &ret);
+}
+
+
+struct xlator_fops fops = {
+        .open = nsr_recon_open,
+        .readv = nsr_recon_readv,
+        .writev = nsr_recon_writev,
+        .lookup = nsr_recon_lookup,
+        .flush = nsr_recon_flush
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+        { .key = {"replica-group-size"},
+          .type = GF_OPTION_TYPE_INT,
+          .min = 2,
+          .max = INT_MAX,
+          .default_value = "2",
+          .description = "Number of bricks in replica group. can be derived but putting it here for testing."
+        },
+        {
+          .key = {"vol-name"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "volume name"
+        },
+        {
+          .key = {"local-member"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "member(brick) for which this translator is responsible."
+        },
+        {
+          .key = {"replica-group-members"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "Comma seperated member names other than local."
+        },
+        {
+          .key = {"changelog-dir"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "Base directory where per term changelogs are maintained."
+        },
+        {
+          .key = {"base-dir"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "Base directory for this brick. This should go away once we fix gfid based lookups"
+        },
+	{ .key = {NULL} },
+};
diff --git a/xlators/cluster/nsr-recon/src/recon_xlator.h b/xlators/cluster/nsr-recon/src/recon_xlator.h
new file mode 100644
index 000000000..d9692a632
--- /dev/null
+++ b/xlators/cluster/nsr-recon/src/recon_xlator.h
@@ -0,0 +1,92 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __RECON_XLATOR_H__
+#define __RECON_XLATOR_H__
+
+#include <semaphore.h>
+#include <pthread.h>
+
+enum gf_dht_mem_types_ {
+        gf_mt_recon_changelog_buf_t = gf_common_mt_end + 1,
+        gf_mt_recon_driver_ctx_t,
+        gf_mt_recon_fd_t,
+        gf_mt_recon_id_t,
+        gf_mt_recon_member_name_t,
+        gf_mt_recon_members_list_t,
+        gf_mt_recon_per_node_worker_t,
+        gf_mt_recon_private_t,
+        gf_mt_recon_reconciliator_info_t,
+        gf_mt_recon_record_t,
+        gf_mt_recon_record_details_t,
+        gf_mt_recon_role_work_t,
+        gf_mt_recon_work_t,
+        gf_mt_recon_work_data_t,
+        gf_mt_recon_worker_t,
+        gf_mt_recon_end,
+};
+
+enum nsr_recon_xlator_sector_t {
+        nsr_recon_xlator_sector_0 = 0, // to report back the status of given transaction ids
+        nsr_recon_xlator_sector_1 = 512, //  to write here information about leadership changes from the brick
+        nsr_recon_xlator_sector_2 = (512 * 2), // to write here individual roles and wait for that role to be done
+        nsr_recon_xlator_sector_3 = (512 *3), // read from here to get term info for given term
+        nsr_recon_xlator_sector_4 = (512 * 4), // read from here to get last term info
+};
+
+
+typedef struct _nsr_recon_private_s {
+        xlator_t *this; //back pointer
+        unsigned int replica_group_size; // number of static members of replica group
+        char **replica_group_members; // replica group members (including itself in first slot)
+        pthread_t thread_id; // driver thread id
+        nsr_recon_driver_ctx_t *driver_thread_context; //driver thread context
+        unsigned int outstanding; // for communicating with driver thread
+        call_frame_t *frame; // old frame that is pending (just one as of now)
+        struct list_head list;
+        char *volname;
+        uint32_t txn_id;
+        char *changelog_base_path;
+        char *base_dir;
+#if defined(NSR_DEBUG)
+        FILE *fp;
+#endif
+} nsr_recon_private_t;
+
+#define atomic_cmpxchg         __sync_val_compare_and_swap
+
+#if defined(NSR_DEBUG)
+
+extern void
+_recon_main_log (const char *func, int line, char *member, FILE *fp,
+                 char *fmt, ...);
+
+#define recon_main_log(dom, levl, fmt...) do {                          \
+        FMT_WARN (fmt);                                                 \
+        if (levl <= nsr_debug_level) {                                  \
+                nsr_recon_private_t     *priv   = this->private;        \
+                _recon_main_log (__FUNCTION__, __LINE__,                \
+                                 priv->replica_group_members[0],        \
+                                 priv->fp,                              \
+                                 ##fmt);                                \
+        }                                                               \
+} while (0)
+
+#else
+#define recon_main_log(dom, levl, fmt...) gf_log(dom, levl, fmt)
+#endif
+
+void nsr_recon_libchangelog_get_this_term_info(xlator_t *this, char *bp, int32_t term, nsr_recon_last_term_info_t *lt);
+void nsr_recon_libchangelog_get_last_term_info(xlator_t *this, char *bp, int32_t term, nsr_recon_last_term_info_t *lt);
+void nsr_recon_return_back(nsr_recon_private_t *priv, uint32_t term, int32_t status, int32_t op_errno);
+gf_boolean_t nsr_recon_libchangelog_get_records(xlator_t *this, char *bp, int32_t term, uint32_t first, uint32_t last, void *buf);
+
+
+#endif /* #ifndef __RECON_XLATOR_H__ */
diff --git a/xlators/protocol/legacy/lib/Makefile.am b/xlators/cluster/nsr-server/Makefile.am
index d471a3f92..d471a3f92 100644
--- a/xlators/protocol/legacy/lib/Makefile.am
+++ b/xlators/cluster/nsr-server/Makefile.am
diff --git a/xlators/cluster/nsr-server/src/Makefile.am b/xlators/cluster/nsr-server/src/Makefile.am
new file mode 100644
index 000000000..0092aad4f
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/Makefile.am
@@ -0,0 +1,43 @@
+noinst_PYTHON = codegen.py gen-fops.py
+
+xlator_LTLIBRARIES = nsr.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
+
+nsr_la_LDFLAGS = -module -avoid-version -lcurl
+
+if ENABLE_ETCD_SIM
+nsr_la_SOURCES = nsr.c leader.c recon_notify.c etcd-sim.c
+else
+nsr_la_SOURCES = nsr.c leader.c recon_notify.c etcd-api.c \
+		 yajl.c yajl_alloc.c yajl_buf.c yajl_encode.c yajl_gen.c \
+		 yajl_lex.c yajl_parser.c yajl_tree.c yajl_version.c
+endif
+
+
+nsr_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+				$(top_builddir)/api/src/libgfapi.la
+
+noinst_HEADERS = nsr-internal.h etcd-api.h all-templates.c \
+	         yajl_alloc.h yajl_buf.h yajl_bytestack.h yajl_encode.h \
+		 yajl_lex.h yajl_parser.h yajl/yajl_common.h yajl/yajl_gen.h \
+		 yajl/yajl_parse.h yajl/yajl_tree.h yajl/yajl_version.h \
+		 $(top_srcdir)/xlators/lib/src/libxlator.h \
+		 $(top_srcdir)/glusterfsd/src/glusterfsd.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/xlators/lib/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src -DSBIN_DIR=\"$(sbindir)\"
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+XLATOR_HEADER	= $(top_srcdir)/libglusterfs/src/xlator.h
+
+CLEANFILES = nsr-cg.c
+
+nsr-cg.c: gen-fops.py codegen.py $(XLATOR_HEADER) all-templates.c
+	$(PYTHON) ./gen-fops.py $(XLATOR_HEADER) all-templates.c > $@
+
+nsr.lo: nsr-cg.c
+
+uninstall-local:
+	rm -f $(DESTDIR)$(xlatordir)/nsr.so
diff --git a/xlators/cluster/nsr-server/src/all-templates.c b/xlators/cluster/nsr-server/src/all-templates.c
new file mode 100644
index 000000000..fa29de7b2
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/all-templates.c
@@ -0,0 +1,345 @@
+/*
+ * You can put anything here - it doesn't even have to be a comment - and it
+ * will be ignored until we reach the first template-name comment.
+ */
+
+
+// template-name read-fop
+$TYPE$
+nsr_$NAME$ (call_frame_t *frame, xlator_t *this,
+            $ARGS_LONG$)
+{
+        nsr_private_t   *priv   = this->private;
+        gf_boolean_t in_recon = _gf_false;
+        int32_t recon_term, recon_index;
+
+        // allow reads during reconciliation
+        // TBD: allow "dirty" reads on non-leaders
+        if (xdata &&
+            (dict_get_int32(xdata, RECON_TERM_XATTR, &recon_term) == 0) &&
+            (dict_get_int32(xdata, RECON_INDEX_XATTR, &recon_index) == 0)) {
+            in_recon = _gf_true;
+        }
+
+        if ((!priv->leader) && (in_recon == _gf_false)) {
+                goto err;
+        }
+
+        STACK_WIND (frame, default_$NAME$_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->$NAME$,
+                    $ARGS_SHORT$);
+        return 0;
+
+err:
+        STACK_UNWIND_STRICT ($NAME$, frame, -1, EREMOTE,
+                             $DEFAULTS$);
+        return 0;
+}
+
+// template-name read-dispatch
+/* No "dispatch" function needed for $NAME$ */
+
+// template-name read-fan-in
+/* No "fan-in" function needed for $NAME$ */
+
+// template-name read-continue
+/* No "continue" function needed for $NAME$ */
+
+// template-name read-complete
+/* No "complete" function needed for $NAME$ */
+
+// template-name write-fop
+$TYPE$
+nsr_$NAME$ (call_frame_t *frame, xlator_t *this,
+            $ARGS_LONG$)
+{
+        nsr_local_t     *local          = NULL;
+        nsr_private_t   *priv           = this->private;
+        int             op_errno        = ENOMEM;
+        int             from_leader;
+        int             from_recon;
+	uint32_t 	ti = 0;
+        double          must_be_up;
+        double          are_up;
+
+        /*
+         * Our first goal here is to avoid "split brain surprise" for users who
+         * specify exactly 50% with two- or three-way replication.  That means
+         * either a more-than check against half the total replicas or an
+         * at-least check against half of our peers (one less).  Of the two,
+         * only an at-least check supports the intuitive use of 100% to mean
+         * all replicas must be present, because "more than 100%" will never
+         * succeed regardless of which count we use.  This leaves us with a
+         * slightly non-traditional definition of quorum ("at least X% of peers
+         * not including ourselves") but one that's useful enough to be worth
+         * it.
+         *
+         * Note that n_children and up_children *do* include the local
+         * subvolume, so we need to subtract one in each case.
+         */
+        must_be_up = ((double)(priv->n_children - 1)) * priv->quorum_pct;
+        are_up = ((double)(priv->up_children - 1)) * 100.0;
+        if (are_up < must_be_up) {
+                /* Emulate the AFR client-side-quorum behavior. */
+                op_errno = EROFS;
+                goto err;
+        }
+
+        local = mem_get0(this->local_pool);
+        if (!local) {
+                goto err;
+        }
+#if defined(NSR_CG_NEED_FD)
+        local->fd = fd_ref(fd);
+#else
+        local->fd = NULL;
+#endif
+        INIT_LIST_HEAD(&local->qlinks);
+        frame->local = local;
+
+        if (xdata) {
+                from_leader = !!dict_get(xdata,NSR_TERM_XATTR);
+                from_recon = !!dict_get(xdata,RECON_TERM_XATTR)
+                          && !!dict_get(xdata,RECON_INDEX_XATTR);
+        }
+        else {
+                from_leader = from_recon = _gf_false;
+        }
+
+        // follower/recon path
+        // just send it to local node
+        if (from_leader || from_recon) {
+                atomic_inc(&priv->ops_in_flight);
+                STACK_WIND (frame, nsr_$NAME$_complete,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->$NAME$,
+                            $ARGS_SHORT$);
+                return 0;
+        }
+
+
+        if (!priv->leader/* || priv->fence_io*/) {
+                op_errno = EREMOTE;
+                goto err;
+        }
+
+
+        if (!xdata) {
+                xdata = dict_new();
+                if (!xdata) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to allocate xdata");
+                        goto err;
+                }
+        }
+
+        if (dict_set_int32(xdata,NSR_TERM_XATTR,priv->current_term) != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to set nsr-term");
+                goto err;
+        }
+
+        LOCK(&priv->index_lock);
+        ti = ++(priv->index);
+        UNLOCK(&priv->index_lock);
+        if (dict_set_int32(xdata,NSR_INDEX_XATTR,ti) != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to set index");
+                goto err;
+        }
+
+        local->stub = fop_$NAME$_stub (frame,nsr_$NAME$_continue,
+                                       $ARGS_SHORT$);
+        if (!local->stub) {
+                goto err;
+        }
+
+
+#if defined(NSR_CG_QUEUE)
+        nsr_inode_ctx_t         *ictx   = nsr_get_inode_ctx(this,fd->inode);
+        if (!ictx) {
+                op_errno = EIO;
+                goto err;
+        }
+        LOCK(&ictx->lock);
+                if (ictx->active) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "queuing request due to conflict");
+                        /*
+                         * TBD: enqueue only for real conflict
+                         *
+                         * Currently we just act like all writes are in
+                         * conflict with one another.  What we should really do
+                         * is check the active/pending queues and defer only if
+                         * there's a conflict there.
+                         *
+                         * It's important to check the pending queue because we
+                         * might have an active request X which conflicts with
+                         * a pending request Y, and this request Z might
+                         * conflict with Y but not X.  If we checked only the
+                         * active queue then Z could jump ahead of Y, which
+                         * would be incorrect.
+                         */
+                        local->qstub = fop_$NAME$_stub (frame,
+                                                        nsr_$NAME$_dispatch,
+                                                        $ARGS_SHORT$);
+                        if (!local->qstub) {
+                                UNLOCK(&ictx->lock);
+                                goto err;
+                        }
+                        list_add_tail(&local->qlinks,&ictx->pqueue);
+                        ++(ictx->pending);
+                        UNLOCK(&ictx->lock);
+                        return 0;
+                }
+                else {
+                        list_add_tail(&local->qlinks,&ictx->aqueue);
+                        ++(ictx->active);
+                }
+        UNLOCK(&ictx->lock);
+#endif
+
+        return nsr_$NAME$_dispatch (frame, this, $ARGS_SHORT$);
+
+err:
+        if (local) {
+                if (local->stub) {
+                        call_stub_destroy(local->stub);
+                }
+                if (local->qstub) {
+                        call_stub_destroy(local->qstub);
+                }
+                if (local->fd) {
+                        fd_unref(local->fd);
+                }
+                mem_put(local);
+        }
+        STACK_UNWIND_STRICT ($NAME$, frame, -1, op_errno,
+                             $DEFAULTS$);
+        return 0;
+}
+
+// template-name write-dispatch
+$TYPE$
+nsr_$NAME$_dispatch (call_frame_t *frame, xlator_t *this,
+                     $ARGS_LONG$)
+{
+        nsr_local_t     *local  = frame->local;
+        nsr_private_t   *priv   = this->private;
+        xlator_list_t   *trav;
+
+        atomic_inc(&priv->ops_in_flight);
+
+        /*
+         * TBD: unblock pending request(s) if we fail after this point but
+         * before we get to nsr_$NAME$_complete (where that code currently
+         * resides).
+         */
+
+        local->call_count = priv->n_children - 1;
+        for (trav = this->children->next; trav; trav = trav->next) {
+                STACK_WIND (frame, nsr_$NAME$_fan_in,
+                            trav->xlator, trav->xlator->fops->$NAME$,
+                            $ARGS_SHORT$);
+        }
+
+        // TBD: variable Issue count
+        return 0;
+}
+
+// template-name write-fan-in
+$TYPE$
+nsr_$NAME$_fan_in (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno,
+                   $ARGS_LONG$)
+{
+        nsr_local_t     *local  = frame->local;
+        uint8_t         call_count;
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "op_ret = %d, op_errno = %d\n", op_ret, op_errno);
+
+        LOCK(&frame->lock);
+        call_count = --(local->call_count);
+        UNLOCK(&frame->lock);
+
+        // TBD: variable Completion count
+        if (call_count == 0) {
+                call_resume(local->stub);
+        }
+
+        return 0;
+}
+
+// template-name write-continue
+$TYPE$
+nsr_$NAME$_continue (call_frame_t *frame, xlator_t *this,
+                     $ARGS_LONG$)
+{
+        STACK_WIND (frame, nsr_$NAME$_complete,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->$NAME$,
+                    $ARGS_SHORT$);
+        return 0;
+}
+
+// template-name write-complete
+$TYPE$
+nsr_$NAME$_complete (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno,
+                     $ARGS_LONG$)
+{
+        nsr_private_t   *priv           = this->private;
+#if defined(NSR_CG_NEED_FD)
+        nsr_local_t *local = frame->local;
+#endif
+
+#if defined(NSR_CG_QUEUE)
+        nsr_inode_ctx_t *ictx;
+        nsr_local_t     *next;
+        if (local->qlinks.next != &local->qlinks) {
+                list_del(&local->qlinks);
+                ictx = nsr_get_inode_ctx(this,local->fd->inode);
+                if (ictx) {
+                    LOCK(&ictx->lock);
+                        if (ictx->pending) {
+                                /*
+                                 * TBD: dequeue *all* non-conflicting reqs
+                                 *
+                                 * With the stub implementation there can only
+                                 * be one request active at a time (zero here)
+                                 * so it's not an issue.  In a real
+                                 * implementation there might still be other
+                                 * active requests to check against, and
+                                 * multiple pending requests that could
+                                 * continue.
+                                 */
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "unblocking next request");
+                                --(ictx->pending);
+                                next = list_entry (ictx->pqueue.next,
+                                                   nsr_local_t, qlinks);
+                                list_del(&next->qlinks);
+                                list_add_tail(&next->qlinks,&ictx->aqueue);
+                                call_resume(next->qstub);
+                        }
+                        else {
+                                --(ictx->active);
+                        }
+                    UNLOCK(&ictx->lock);
+                }
+        }
+#endif
+
+#if defined(NSR_CG_FSYNC)
+        nsr_mark_fd_dirty(this,local);
+#endif
+
+#if defined(NSR_CG_NEED_FD)
+        fd_unref(local->fd);
+#endif
+
+        STACK_UNWIND_STRICT ($NAME$, frame, op_ret, op_errno,
+                                     $ARGS_SHORT$);
+        atomic_dec(&priv->ops_in_flight);
+        return 0;
+        
+}
diff --git a/xlators/cluster/nsr-server/src/codegen.py b/xlators/cluster/nsr-server/src/codegen.py
new file mode 100755
index 000000000..709f5662f
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/codegen.py
@@ -0,0 +1,174 @@
+#!/usr/bin/python
+
+# This module lets us auto-generate boilerplate versions of fops and cbks,
+# both for the client side and (eventually) on the server side as well.  This
+# allows us to implement common logic (e.g. leader fan-out and sequencing)
+# once, without all the problems that come with copying and pasting the same
+# code into dozens of functions (or failing to).
+#
+# I've tried to make this code pretty generic, since it's already likely to
+# be used multiple ways within NSR.  Really, we should use something like this
+# to generate defaults.[ch] as well, to avoid the same sorts of mismatches
+# that we've already seen and to which this approach makes NSR immune.  That
+# would require using something other than defaults.h as the input, but that
+# format could be even simpler so that's a good thing too.
+
+
+import re
+import sys
+
+decl_re = re.compile("([a-z0-9_]+)$")
+tmpl_re = re.compile("// template-name (.*)")
+
+class CodeGenerator:
+
+	def __init__ (self):
+		self.decls = {}
+		self.skip = 0
+		self.templates = {}
+		self.make_defaults = self._make_defaults
+
+	# Redefine this to preprocess the name in a declaration, e.g.
+	#	fop_lookup_t => nsrc_lookup
+	def munge_name (self, orig):
+		return orig
+
+	# By default, this will convert the argument string into a sequence of
+	# (type, name) tuples minus the first self.skip (default zero) arguments.
+	# You can redefine it to skip the conversion, do a different conversion,
+	# or rearrange the arguments however you like.
+	def munge_args (self, orig):
+		args = []
+		for decl in orig.strip("(); ").split(","):
+			m = decl_re.search(decl)
+			if m:
+				args.append((m.group(1),decl[:m.start(1)].strip()))
+			else:
+				raise RuntimeError("can't split %s into type+name"%decl)
+		return args[self.skip:]
+
+	def add_decl (self, fname, ftype, fargs):
+		self.decls[self.munge_name(fname)] = (ftype, self.munge_args(fargs))
+
+	def parse_decls (self, path, pattern):
+		regex = re.compile(pattern)
+		f = open(path,"r")
+		have_decl = False
+		while True:
+			line = f.readline()
+			if not line:
+				break
+			m = regex.search(line)
+			if m:
+				if have_decl:
+					self.add_decl(f_name,f_type,f_args)
+				f_name = m.group(2)
+				f_type = m.group(1)
+				f_args = line[m.end(0):-1].strip()
+				if f_args.rfind(")") >= 0:
+					self.add_decl(f_name,f_type,f_args)
+				else:
+					have_decl = True
+			elif have_decl:
+				if line.strip() == "":
+					self.add_decl(f_name,f_type,f_args)
+					have_decl = False
+				else:
+					f_args += " "
+					f_args += line[:-1].strip()
+		if have_decl:
+			self.add_decl(f_name,f_type,f_args)
+
+	# Legacy function (yeah, already) to load a single template.  If you're
+	# using multiple templates, you're better off loading them all from one
+	# file using load_templates (note plural) instead.
+	def load_template (self, name, path):
+		self.templates[name] = open(path,"r").readlines()
+
+	# Load multiple templates.  Each is introduced by a special comment of
+	# the form
+	#
+	#	// template-name xyz
+	#
+	# One side effect is that the block before the first such comment will be
+	# ignored.  This seems like it might be useful some day so I'll leave it
+	# in, but if people trip over it maybe it will change.
+	#
+	# It is recommended to define templates in expected execution order, to
+	# make the result more readable than the inverted order (e.g. callback
+	# then fop) common in the rest of our code.
+	def load_templates (self, path):
+		t_name = None
+		for line in open(path,"r").readlines():
+			if not line:
+				break
+			m = tmpl_re.match(line)
+			if m:
+				if t_name:
+					self.templates[t_name] = t_contents
+				t_name = m.group(1).strip()
+				t_contents = []
+			elif t_name:
+				t_contents.append(line)
+		if t_name:
+			self.templates[t_name] = t_contents
+
+	# Emit the template, with the following expansions:
+	#
+	#	$NAME$			=> function name (as passed in)
+	#	$TYPE$			=> function return value
+	#	$ARGS_SHORT$	=> argument list, including types
+	#	$ARGS_LONG$		=> argument list, *not* including types
+	#	$DEFAULTS$		=> default callback args (see below)
+	#
+	# The $DEFAULTS$ substitution is for the case where a fop (which has one
+	# set of arguments) needs to signal an error via STACK_UNWIND (which
+	# requires a different set of arguments).  In this case we look up the
+	# argument list for the opposite direction, using self.make_defaults which
+	# the user must explicitly set to the method for the opposite direction.
+	# If an argument is a pointer, we replace it with NULL; otherwise we
+	# replace it with zero.  It's a hack, but it's the only thing we do that
+	# doesn't require specific knowledge of our environment and the specific
+	# call we're handling.  If this doesn't suffice, we'll have to add
+	# something like $ARG0$ which can be passed in for specific cases.
+	def emit (self, f_name, tmpl):
+		args = self.decls[f_name][1]
+		zipper = lambda x: x[0]
+		a_short = ", ".join(map(zipper,args))
+		zipper = lambda x: x[1] + " " + x[0]
+		a_long = ", ".join(map(zipper,args))
+		for line in self.templates[tmpl]:
+			line = line.replace("$NAME$",f_name)
+			line = line.replace("$TYPE$",self.decls[f_name][0])
+			line = line.replace("$ARGS_SHORT$",a_short)
+			line = line.replace("$ARGS_LONG$",a_long)
+			line = line.replace("$DEFAULTS$",self.make_defaults(f_name))
+			print(line.rstrip())
+
+	def _make_defaults (self, f_name):
+		result = []
+		for arg in self.decls[f_name][1]:
+			if arg[1][-1] == "*":
+				result.append("NULL")
+			else:
+				result.append("0")
+		return ", ".join(result)
+
+if __name__ == "__main__":
+	type_re = "([a-z_0-9]+)"
+	name_re = "\(\*fop_([a-z0-9]+)_t\)"
+	full_re = type_re + " *" + name_re
+	cg = CodeGenerator()
+	cg.skip = 2
+	cg.parse_decls(sys.argv[1],full_re)
+	"""
+	for k, v in cg.decls.iteritems():
+		print("=== %s" % k)
+		print("  return type %s" % v[0])
+		for arg in v[1]:
+			print("  arg %s (type %s)" % arg)
+	"""
+	cg.load_template("fop",sys.argv[2])
+	cg.emit("lookup","fop")
+	cg.emit("rename","fop")
+	cg.emit("setxattr","fop")
diff --git a/xlators/cluster/nsr-server/src/etcd-api.c b/xlators/cluster/nsr-server/src/etcd-api.c
new file mode 100644
index 000000000..a07019244
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/etcd-api.c
@@ -0,0 +1,831 @@
+/*
+ * Copyright (c) 2013, Red Hat
+ * All rights reserved.
+
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.  Redistributions in binary
+ * form must reproduce the above copyright notice, this list of conditions and
+ * the following disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* For asprintf */
+#if !defined(_GNU_SOURCE)
+#define _GNU_SOURCE
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <curl/curl.h>
+#include <yajl/yajl_tree.h>
+#include "etcd-api.h"
+
+
+#define DEFAULT_ETCD_PORT       4001
+#define SL_DELIM                "\n\r\t ,;"
+
+typedef struct {
+        etcd_server     *servers;
+} _etcd_session;
+
+typedef struct {
+        char            *key;
+        char            *value;
+        int             *index_in;      /* pointer so NULL can be special */
+        int             index_out;      /* NULL would be meaningless */
+} etcd_watch_t;
+
+typedef size_t curl_callback_t (void *, size_t, size_t, void *);
+
+int             g_inited        = 0;
+const char      *value_path[]   = { "node", "value", NULL };
+const char      *nodes_path[]   = { "node", "nodes", NULL };
+const char      *entry_path[]   = { "key", NULL };
+
+/*
+ * We only call this in case where it should be safe, but gcc doesn't know
+ * that so we use this to shut it up.
+ */
+char *
+MY_YAJL_GET_STRING (yajl_val x)
+{
+        char *y = YAJL_GET_STRING(x);
+
+        return y ? y : "bogus";
+}
+
+#if defined(DEBUG)
+void
+print_curl_error (char *intro, CURLcode res)
+{
+        printf("%s: %s\n",intro,curl_easy_strerror(res));
+}
+#else
+#define print_curl_error(intro,res)
+#endif
+
+ 
+etcd_session
+etcd_open (etcd_server *server_list)
+{
+        _etcd_session   *session;
+
+        if (!g_inited) {
+                curl_global_init(CURL_GLOBAL_ALL);
+                g_inited = 1;
+        }
+
+        session = malloc(sizeof(*session));
+        if (!session) {
+                return NULL;
+        }
+
+        /*
+         * Some day we'll set up more persistent connections, and keep track
+         * (via redirects) of which server is leader so that we can always
+         * try it first.  For now we just push that to the individual request
+         * functions, which do the most brain-dead thing that can work.
+         */
+
+        session->servers = server_list;
+        return session;
+}
+
+
+void
+etcd_close (etcd_session session)
+{
+        free(session);
+}
+
+/*
+ * Normal yajl_tree_get is returning NULL for these paths even when I can
+ * verify (in gdb) that they exist.  I suppose I could debug this for them, but
+ * this is way easier.
+ *
+ * TBD: see if common distros are packaging a JSON library that isn't total
+ * crap.
+ */
+yajl_val
+my_yajl_tree_get (yajl_val root, char const **path, yajl_type type)
+{
+        yajl_val        obj    = root;
+        int             i;
+
+        for (;;) {
+                if (!*path) {
+                        if (obj && (obj->type != type)) {
+                                return NULL;
+                        }
+                        return obj;
+                }
+                if (obj->type != yajl_t_object) {
+                        return NULL;
+                }
+                for (i = 0; /* nothing */; ++i) {
+                        if (i >= obj->u.object.len) {
+                                return NULL;
+                        }
+                        if (!strcmp(obj->u.object.keys[i],*path)) {
+                                obj = obj->u.object.values[i];
+                                ++path;
+                                break;
+                        }
+                }
+        }
+}
+
+
+/*
+ * Looking directly at node->u.array seems terribly un-modular, but the YAJL
+ * tree interface doesn't seem to have any exposed API for iterating over the
+ * elements of an array.  I tried using yajl_tree_get with an index in the
+ * path, either as a type-casted integer or as a string, but that didn't work.
+ */
+char *
+parse_array_response (yajl_val parent)
+{
+        size_t          i;
+        yajl_val        item;
+        yajl_val        value;
+        char            *retval = NULL;
+        char            *saved;
+        yajl_val        node;
+
+        node = my_yajl_tree_get(parent,nodes_path,yajl_t_array);
+        if (!node) {
+                return NULL;
+        }
+
+        for (i = 0; i < node->u.array.len; ++i) {
+                item = node->u.array.values[i];
+                if (!item) {
+                        break;
+                }
+                value = my_yajl_tree_get(item,entry_path,yajl_t_string);
+                if (!value) {
+                        break;
+                }
+                if (retval) {
+                        saved = retval;
+                        retval = NULL;
+                        (void)asprintf (&retval, "%s\n%s",
+                                        saved, MY_YAJL_GET_STRING(value));
+                        free(saved);
+                }
+                else {
+                        retval = strdup(MY_YAJL_GET_STRING(value));
+                }
+                if (!retval) {
+                        break;
+                }
+        }
+
+        return retval;
+}
+
+size_t
+parse_get_response (void *ptr, size_t size, size_t nmemb, void *stream)
+{
+        yajl_val        node;
+        yajl_val        value;
+
+        node = yajl_tree_parse(ptr,NULL,0);
+        if (node) {
+                value = my_yajl_tree_get(node,value_path,yajl_t_string);
+                if (value) {
+                        /* 
+                         * YAJL probably copied it once, now we're going to
+                         * copy it again.  If anybody really cares for such
+                         * small and infrequently used values, we'd have to do
+                         * do something much more complicated (like using the
+                         * stream interface) to avoid the copy.  Right now it's
+                         * just not worth it.
+                         */
+                        *((char **)stream) = strdup(MY_YAJL_GET_STRING(value));
+                }
+                else {
+                        /* Might as well try this. */
+                        *((char **)stream) = parse_array_response(node);
+                }
+                yajl_tree_free(node);
+        }
+
+        return size*nmemb;
+}
+
+
+etcd_result
+etcd_get_one (_etcd_session *session, char *key, etcd_server *srv, char *prefix,
+              char *post, curl_callback_t cb, char **stream)
+{
+        char            *url;
+        CURL            *curl;
+        CURLcode        curl_res;
+        etcd_result     res             = ETCD_WTF;
+        void            *err_label      = &&done;
+
+        if (asprintf(&url,"http://%s:%u/v2/%s%s",
+                     srv->host,srv->port,prefix,key) < 0) {
+                goto *err_label;
+        }
+        printf("url = %s\n",url);
+        err_label = &&free_url;
+
+        curl = curl_easy_init();
+        if (!curl) {
+                goto *err_label;
+        }
+        err_label = &&cleanup_curl;
+
+        /* TBD: add error checking for these */
+        curl_easy_setopt(curl,CURLOPT_URL,url);
+        curl_easy_setopt(curl,CURLOPT_FOLLOWLOCATION,1L);
+        curl_easy_setopt(curl,CURLOPT_WRITEFUNCTION,cb);
+        curl_easy_setopt(curl,CURLOPT_WRITEDATA,stream);
+        if (post) {
+                curl_easy_setopt(curl,CURLOPT_POST,1L);
+                curl_easy_setopt(curl,CURLOPT_POSTFIELDS,post);
+        }
+#if defined(DEBUG)
+        curl_easy_setopt(curl,CURLOPT_VERBOSE,1L);
+#endif
+
+        curl_res = curl_easy_perform(curl);
+        if (curl_res != CURLE_OK) {
+                print_curl_error("perform",curl_res);
+                goto *err_label;
+        }
+
+        res = ETCD_OK;
+
+cleanup_curl:
+        curl_easy_cleanup(curl);
+free_url:
+        free(url);
+done:
+        return res;
+}
+
+
+char *
+etcd_get (etcd_session session_as_void, char *key)
+{
+        _etcd_session   *session   = session_as_void;
+        etcd_server     *srv;
+        etcd_result     res;
+        char            *value  = NULL;
+
+        for (srv = session->servers; srv->host; ++srv) {
+                res = etcd_get_one(session,key,srv,"keys/",NULL,
+                                   parse_get_response,&value);
+                if ((res == ETCD_OK) && value) {
+                        return value;
+                }
+        }
+
+        return NULL;
+}
+
+
+size_t
+parse_watch_response (void *ptr, size_t size, size_t nmemb, void *stream)
+{
+        yajl_val                node;
+        yajl_val                value;
+        etcd_watch_t            *watch  = stream;
+        static const char       *i_path[] = { "node", "modifiedIndex", NULL };
+        static const char       *k_path[] = { "node", "key", NULL };
+        static const char       *v_path[] = { "node", "value", NULL };
+
+        node = yajl_tree_parse(ptr,NULL,0);
+        if (node) {
+                value = my_yajl_tree_get(node,i_path,yajl_t_number);
+                if (value) {
+                        watch->index_out = strtoul(YAJL_GET_NUMBER(value),
+                                                   NULL,10);
+                }
+                value = my_yajl_tree_get(node,k_path,yajl_t_string);
+                if (value) {
+                        watch->key = strdup(MY_YAJL_GET_STRING(value));
+                }
+                value = my_yajl_tree_get(node,v_path,yajl_t_string);
+                if (value) {
+                        watch->value = strdup(MY_YAJL_GET_STRING(value));
+                }
+        }
+
+        return size*nmemb;
+}
+
+
+etcd_result
+etcd_watch (etcd_session session_as_void, char *pfx,
+            char **keyp, char **valuep, int *index_in, int *index_out)
+{
+        _etcd_session   *session   = session_as_void;
+        etcd_server     *srv;
+        etcd_result     res;
+        etcd_watch_t    watch;
+        char            *path;
+
+        if (index_in) {
+                if (asprintf(&path,"%s?wait=true&recursive=true&waitIndex=%d",
+                             pfx,*index_in) < 0) {
+                        return ETCD_WTF;
+                }
+        }
+        else {
+                if (asprintf(&path,"%s?wait=true&recursive=true",pfx) < 0) {
+                        return ETCD_WTF;
+                }
+        }
+
+        memset(&watch,0,sizeof(watch));
+        watch.index_in = index_in;
+
+        for (srv = session->servers; srv->host; ++srv) {
+                res = etcd_get_one(session,path,srv,"keys/",NULL,
+                                   parse_watch_response,(char **)&watch);
+                if (res == ETCD_OK) {
+                        if (keyp) {
+                                *keyp = watch.key;
+                        }
+                        if (valuep) {
+                                *valuep = watch.value;
+                        }
+                        if (index_out) {
+                                *index_out = watch.index_out;
+                        }
+                        break;
+                }
+        }
+
+        free(path);
+        return res;
+}
+
+
+size_t
+parse_set_response (void *ptr, size_t size, size_t nmemb, void *stream)
+{
+        yajl_val        node;
+        yajl_val        value;
+        etcd_result     res     = ETCD_PROTOCOL_ERROR;
+        /*
+         * Success responses contain prevValue and index.  Failure responses
+         * contain errorCode and cause.  Among all these, index seems to be the
+         * one we're most likely to need later, so look for that.
+         */
+        static const char       *path[] = { "node", "modifiedIndex", NULL };
+
+        node = yajl_tree_parse(ptr,NULL,0);
+        if (node) {
+                value = my_yajl_tree_get(node,path,yajl_t_number);
+                if (value) {
+                        res = ETCD_OK;
+                }
+        }
+
+        *((etcd_result *)stream) = res;
+        return size*nmemb;
+}
+
+
+size_t
+parse_lock_response (void *ptr, size_t size, size_t nmemb, void *stream)
+{
+        *((char **)stream) = strdup(ptr);
+        return size*nmemb;
+}
+
+
+/* 
+ * There are two use cases, based on is_lock.
+ *
+ * If is_lock is null, we use the "keys" namespace.  A null value means an
+ * HTTP DELETE; precond and ttl are both ignored.  Otherwise we're setting a
+ * value, with *optional* precond and ttl.
+ *
+ * If is_lock is set, we use the "locks" namespace.  A null value means an
+ * HTTP DELETE as before, and we still ignore ttl as before, but now precond
+ * must be set to represent the lock index.  Otherwise ttl must be present,
+ * and we decide what to do based on precond.  If it's null, this is an
+ * initial lock so we use an HTTP POST.  Otherwise it's a renewal so we use
+ * an HTTP PUT instead.
+ */
+etcd_result
+etcd_set_one (_etcd_session *session, char *key, char *value,
+              char *precond, unsigned int ttl, etcd_server *srv,
+              char **is_lock)
+{
+        char                    *url;
+        char                    *contents       = NULL;
+        CURL                    *curl;
+        etcd_result             res             = ETCD_WTF;
+        CURLcode                curl_res;
+        void                    *err_label      = &&done;
+        char                    *namespace;
+        char                    *http_cmd;
+        char                    *orig_index;
+
+        if (is_lock) {
+                namespace = "mod/v2/lock";
+                if (value) {
+                        if (!ttl) {
+                                /* Lock/renew must specify ttl. */
+                                return ETCD_WTF;
+                        }
+                        http_cmd = precond ? "PUT" : "POST";
+                }
+                else {
+                        if (!precond) {
+                                /* Unlock must specify index. */
+                                return ETCD_WTF;
+                        }
+                        http_cmd = "DELETE";
+                }
+                orig_index = *is_lock;
+        }
+        else {
+                namespace = "v2/keys";
+                http_cmd = value ? "PUT" : "DELETE";
+        }
+
+        if (asprintf(&url,"http://%s:%u/%s/%s",
+                     srv->host,srv->port,namespace,key) < 0) {
+                goto *err_label;
+        }
+        err_label = &&free_url;
+
+        if (is_lock) {
+                if (precond) {
+                        if (asprintf(&contents,"index=%s",precond) < 0) {
+                                goto *err_label;
+                        }
+                        err_label = &&free_contents;
+                }
+                if (ttl) {
+                        if (contents) {
+                                char *c2;
+                                if (asprintf(&c2,"ttl=%u;%s",ttl,contents) < 0) {
+                                        goto *err_label;
+                                }
+                                free(contents);
+                                contents = c2;
+                        }
+                        else {
+                                if (asprintf(&contents,"ttl=%u",ttl) < 0) {
+                                        goto *err_label;
+                                }
+                        }
+                        err_label = &&free_contents;
+                }
+        }
+        else {
+                if (value) {
+                        if (asprintf(&contents,"value=%s",value) < 0) {
+                                goto *err_label;
+                        }
+                        err_label = &&free_contents;
+                }
+                if (precond) {
+                        char *c2;
+                        if (asprintf(&c2,"%s;prevValue=%s",contents,
+                                     precond) < 0) {
+                                goto *err_label;
+                        }
+                        free(contents);
+                        contents = c2;
+                        err_label = &&free_contents;
+                }
+                if (ttl) {
+                        char *c2;
+                        if (asprintf(&c2,"%s;ttl=%u",contents,ttl) < 0) {
+                                goto *err_label;
+                        }
+                        free(contents);
+                        contents = c2;
+                        err_label = &&free_contents;
+                }
+        }
+
+        curl = curl_easy_init();
+        if (!curl) {
+                goto *err_label;
+        }
+        err_label = &&cleanup_curl;
+
+        /* TBD: add error checking for these */
+        curl_easy_setopt(curl,CURLOPT_CUSTOMREQUEST,http_cmd);
+        curl_easy_setopt(curl,CURLOPT_URL,url);
+        curl_easy_setopt(curl,CURLOPT_FOLLOWLOCATION,1L);
+        curl_easy_setopt(curl,CURLOPT_POSTREDIR,CURL_REDIR_POST_ALL);
+
+        if (is_lock && value && !precond) {
+                /* Only do this for an initial lock, not a renewal. */
+                curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION,
+                                  parse_lock_response);
+                curl_easy_setopt(curl,CURLOPT_WRITEDATA,is_lock);
+        }
+        else {
+                curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION,
+                                  parse_set_response);
+                curl_easy_setopt(curl,CURLOPT_WRITEDATA,&res);
+        }
+
+        /*
+         * CURLOPT_HTTPPOST would be easier, but it looks like etcd will barf on
+         * that.  Sigh.
+         */
+        if (contents) {
+                curl_easy_setopt(curl,CURLOPT_POST,1L);
+                curl_easy_setopt(curl,CURLOPT_POSTFIELDS,contents);
+        }
+#if defined(DEBUG)
+        curl_easy_setopt(curl,CURLOPT_VERBOSE,1L);
+#endif
+
+        curl_res = curl_easy_perform(curl);
+        if (curl_res != CURLE_OK) {
+                print_curl_error("perform",curl_res);
+                goto *err_label;
+        }
+
+        if (is_lock && value) {
+                if (!precond) {
+                        /*
+                         * If this is an initial lock, parse_lock_response would
+                         * have been unable to set "res" for us.  Instead, we
+                         * set it here if the index string got updated.
+                         */
+                        if (*is_lock != orig_index) {
+                                res = ETCD_OK;
+                        }
+                }
+                else {
+                        /*
+                         * If this is a lock renewal, then a successful call
+                         * will pass through neither parse_lock_response nor
+                         * parse_get_response.  The curl response code alone
+                         * is sufficient.
+                         */
+                        res = ETCD_OK;
+                }
+        }
+
+        /*
+         * If the request succeeded, or at least got to the server and failed
+         * there, parse_set_response should have set res appropriately.
+         */
+
+cleanup_curl:
+        curl_easy_cleanup(curl);
+free_contents:
+        free(contents); /* might already be NULL for delete, but that's OK */
+free_url:
+        free(url);
+done:
+        return res;
+}
+
+
+etcd_result
+etcd_set (etcd_session session_as_void, char *key, char *value,
+          char *precond, unsigned int ttl)
+{
+        _etcd_session   *session   = session_as_void;
+        etcd_server     *srv;
+        etcd_result     res;
+
+        for (srv = session->servers; srv->host; ++srv) {
+                res = etcd_set_one(session,key,value,precond,ttl,srv,NULL);
+                /*
+                 * Protocol errors are likely to be things like precondition
+                 * failures, which won't be helped by retrying on another
+                 * server.
+                 */
+                if ((res == ETCD_OK) || (res == ETCD_PROTOCOL_ERROR)) {
+                        return res;
+                }
+        }
+
+        return ETCD_WTF;
+}
+
+
+/*
+ * This uses the same path and status checks as SET, but with a different HTTP
+ * command instead of data.  Precondition and TTL are obviously not used in
+ * this case, though a conditional delete would be a cool feature for etcd.  I
+ * think you can get a timed delete by doing a conditional set to the current
+ * value with a TTL, but I haven't actually tried it.
+ */
+etcd_result
+etcd_delete (etcd_session session_as_void, char *key)
+{
+        _etcd_session   *session   = session_as_void;
+        etcd_server     *srv;
+        etcd_result     res;
+
+        for (srv = session->servers; srv->host; ++srv) {
+                res = etcd_set_one(session,key,NULL,NULL,0,srv,NULL);
+                if (res == ETCD_OK) {
+                        break;
+                }
+        }
+
+        return res;
+}
+
+
+etcd_result
+etcd_lock (etcd_session session_as_void, char *key, unsigned int ttl,
+           char *index_in, char **index_out)
+{
+        _etcd_session   *session        = session_as_void;
+        etcd_server     *srv;
+        etcd_result     res;
+        char            *tmp            = NULL;
+
+        for (srv = session->servers; srv->host; ++srv) {
+                res = etcd_set_one(session,key,"hack",index_in,ttl,srv,&tmp);
+                if (res == ETCD_OK) {
+                        if (index_out) {
+                                *index_out = tmp;
+                        }
+                        break;
+                }
+        }
+
+        return res;
+}
+
+
+etcd_result
+etcd_unlock (etcd_session session_as_void, char *key, char *index)
+{
+        _etcd_session   *session   = session_as_void;
+        etcd_server     *srv;
+        etcd_result     res;
+        char            *tmp            = NULL;
+
+        for (srv = session->servers; srv->host; ++srv) {
+                res = etcd_set_one(session,key,NULL,index,0,srv,&tmp);
+                if (res == ETCD_OK) {
+                        break;
+                }
+        }
+
+        return res;
+}
+size_t
+store_leader (void *ptr, size_t size, size_t nmemb, void *stream)
+{
+        *((char **)stream) = strdup(ptr);
+        return size * nmemb;
+}
+
+
+char *
+etcd_leader (etcd_session session_as_void)
+{
+        _etcd_session   *session   = session_as_void;
+        etcd_server     *srv;
+        etcd_result     res;
+        char            *value  = NULL;
+
+        for (srv = session->servers; srv->host; ++srv) {
+                res = etcd_get_one(session,"leader",srv,"",NULL,
+                                   store_leader,&value);
+                if ((res == ETCD_OK) && value) {
+                        return value;
+                }
+        }
+
+        return NULL;
+}
+
+
+void
+free_sl (etcd_server *server_list)
+{
+        size_t          num_servers;
+
+        for (num_servers = 0; server_list[num_servers].host; ++num_servers) {
+                free(server_list[num_servers].host);
+        }
+        free(server_list);
+}
+
+
+int
+_count_matching (char *text, char *cset, int result)
+{
+        char    *t;
+        int     res     = 0;
+
+        for (t = text; *t; ++t) {
+                if ((strchr(cset,*t) != NULL) != result) {
+                        break;
+                }
+                ++res;
+        }
+
+        return res;
+}
+
+#define count_matching(t,cs)    _count_matching(t,cs,1)
+#define count_nonmatching(t,cs) _count_matching(t,cs,0)
+
+
+etcd_session
+etcd_open_str (char *server_names)
+{
+        char            *snp;
+        int             run_len;
+        int             host_len;
+        size_t           num_servers;
+        etcd_server     *server_list;
+        etcd_session    *session;
+
+        /*
+         * Yeah, we iterate over the string twice so we can allocate an
+         * appropriately sized array instead of turning it into a linked list.
+         * Unfortunately this means we can't use strtok* which is destructive
+         * with no platform-independent way to reverse the destructive effects.
+         */
+
+        num_servers = 0;
+        snp = server_names;
+        while (*snp) {
+                run_len = count_nonmatching(snp,SL_DELIM);
+                if (!run_len) {
+                        snp += count_matching(snp,SL_DELIM);
+                        continue;
+                }
+                ++num_servers;
+                snp += run_len;
+        }
+
+        if (!num_servers) {
+                return NULL;
+        }
+
+        server_list = calloc(num_servers+1,sizeof(*server_list));
+        if (!server_list) {
+                return NULL;
+        }
+        num_servers = 0;
+
+        snp = server_names;
+        while (*snp) {
+                run_len = count_nonmatching(snp,SL_DELIM);
+                if (!run_len) {
+                        snp += count_matching(snp,SL_DELIM);
+                        continue;
+                }
+                host_len = count_nonmatching(snp,":");
+                if ((run_len - host_len) > 1) {
+                        server_list[num_servers].host = strndup(snp,host_len);
+                        server_list[num_servers].port = (unsigned short)
+                                strtoul(snp+host_len+1,NULL,10);
+                }
+                else {
+                        server_list[num_servers].host = strndup(snp,run_len);
+                        server_list[num_servers].port = DEFAULT_ETCD_PORT;
+                }
+                ++num_servers;
+                snp += run_len;
+        }
+
+        session = etcd_open(server_list);
+        if (!session) {
+                free_sl(server_list);
+        }
+        return session;
+}
+
+
+void
+etcd_close_str (etcd_session session)
+{
+        free_sl(((_etcd_session *)session)->servers);
+        etcd_close(session);
+}
diff --git a/xlators/cluster/nsr-server/src/etcd-api.h b/xlators/cluster/nsr-server/src/etcd-api.h
new file mode 100644
index 000000000..66275d40d
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/etcd-api.h
@@ -0,0 +1,214 @@
+/*
+ * Copyright (c) 2013, Red Hat
+ * All rights reserved.
+
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.  Redistributions in binary
+ * form must reproduce the above copyright notice, this list of conditions and
+ * the following disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Description of an etcd server.  For now it just includes the name and
+ * port, but some day it might include other stuff like SSL certificate
+ * information.
+ */
+
+typedef enum {
+        ETCD_OK = 0,
+        ETCD_PROTOCOL_ERROR,
+                                /* TBD: add other error categories here */
+        ETCD_WTF                /* anything we can't easily categorize */
+} etcd_result;
+
+typedef struct {
+        char            *host;
+        unsigned short  port;
+} etcd_server;
+
+typedef void *etcd_session;
+
+/*
+ * etcd_open
+ *
+ * Establish a session to an etcd cluster, with automatic reconnection and
+ * so on.
+ *
+ *      server_list
+ *      Array of etcd_server structures, with the last having host=NULL.  The
+ *      caller is responsible for ensuring that this remains valid as long as
+ *      the session exists.
+ */
+etcd_session    etcd_open       (etcd_server *server_list);
+
+
+/*
+ * etcd_open_str
+ *
+ * Same as etcd_open, except that the servers are specified as a list of
+ * host:port strings, separated by comma/semicolon or whitespace.
+ */
+etcd_session    etcd_open_str   (char *server_names);
+
+
+/*
+ * etcd_close
+ *
+ * Terminate a session, closing connections and freeing memory (or any other
+ * resources) associated with it.
+ */
+void            etcd_close      (etcd_session session);
+
+
+/*
+ * etcd_close
+ *
+ * Same as etcd_close, but also free the server list as etcd_open_str would
+ * have allocated it.
+ */
+void            etcd_close_str  (etcd_session session);
+
+
+/*
+ * etcd_get
+ *
+ * Fetch a key from one of the servers in a session.  The return value is a
+ * newly allocated string, which must be freed by the caller.
+ *
+ *      key
+ *      The etcd key (path) to fetch.
+ */
+char *          etcd_get (etcd_session session, char *key);
+
+
+/*
+ * etcd_watch
+ * Watch the set of keys matching a prefix.
+ *
+ *      pfx
+ *      The etcd key prefix (like a path) to watch.
+ *
+ *      keyp
+ *      Space for a pointer to the key that was added/modified/deleted.
+ *
+ *      valuep
+ *      Space for a pointer to the value if a key was added/modified.  A delete
+ *      is signified by this being set to NULL.
+ *
+ *      index_in
+ *      Pointer to an index to be used for *issuing* the watch request, or
+ *      NULL for a watch without an index.
+ *
+ *      index_out
+ *      Pointer to space for an index *returned* by etcd, or NULL to mean don't
+ *      bother.
+ *
+ * In normal usage, index_in will be NULL and index_out will be set to receive
+ * the index for the first watch.  Subsequently, index_in will be set to
+ * provide the previous index (plus one) and index_out will be set to receive
+ * the next.  It's entirely legitimate to point both at the same variable.
+ */
+
+etcd_result     etcd_watch (etcd_session session, char *pfx,
+                            char **keyp, char **valuep,
+                            int *index_in, int *index_out);
+
+
+/*
+ * etcd_set
+ *
+ * Write a key, with optional TTL and/or previous value (as a precondition).
+ *
+ *      key
+ *      The etcd key (path) to set.
+ *
+ *      value
+ *      New value as a null-terminated string.  Unlike etcd_get, we can derive
+ *      the length ourselves instead of needing it to be passed in separately.
+ *
+ *      precond
+ *      Required previous value as a null-terminated string, or NULL to mean
+ *      an unconditional set.
+ *
+ *      ttl
+ *      Time in seconds after which the value will automatically expire and be
+ *      deleted, or zero to mean no auto-expiration.
+ */
+
+etcd_result     etcd_set        (etcd_session session, char *key, char *value,
+                                 char *precond, unsigned int ttl);
+
+
+/*
+ * etcd_delete
+ *
+ * Delete a key from one of the servers in a session.
+ *
+ *      key
+ *      The etcd key (path) to delete.
+ */
+
+etcd_result     etcd_delete     (etcd_session session, char *key);
+
+
+/*
+ * etcd_leader
+ *
+ * Get the identify of the current leader.
+ */
+
+char *          etcd_leader     (etcd_session session);
+
+/*
+ * etcd_lock
+ *
+ * Take or renew a lock - really a lease but the etcd folks call it a lock so
+ * we'll follow suit.
+ *
+ *      key
+ *      The path (in the "locks" namespace) for the lock.
+ *
+ *      ttl
+ *      Time in seconds for the lock.
+ *
+ *      index_in (optional, indicates renewal)
+ *      Lock index from previous lock call.
+ *
+ *      index_out (only used for initial lock)
+ *      Place for the new lock index.  You must free this.
+ */
+
+etcd_result     etcd_lock (etcd_session session_as_void, char *key,
+                           unsigned int ttl, char *index_in, char **index_out);
+
+/*
+ * etcd_unlock
+ *
+ * Release a lock (see etcd_lock regarding terminology).
+ *
+ *      key
+ *      The path (in the "locks" namespace) for the lock.
+ *
+ *      index
+ *      Lock index from previous lock call.
+ */
+
+etcd_result     etcd_unlock (etcd_session session_as_void, char *key,
+                             char *index);
+
diff --git a/xlators/cluster/nsr-server/src/etcd-sim.c b/xlators/cluster/nsr-server/src/etcd-sim.c
new file mode 100644
index 000000000..d0bea12c7
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/etcd-sim.c
@@ -0,0 +1,280 @@
+/*
+ * Copyright (c) 2014, Red Hat
+ * All rights reserved.
+
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.  Redistributions in binary
+ * form must reproduce the above copyright notice, this list of conditions and
+ * the following disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <sys/file.h>
+
+#include "mem-pool.h"
+
+/*
+ * Mock implementation of etcd
+ * The etcd file is simulated in /tmp/<server-names>
+ * Writes from Multiple writers are protected using file lock.
+*/
+
+#include "etcd-api.h"
+#define MAX_KEY_LEN 64
+#define MAX_VALUE_LEN 64
+#define MAX_EXPIRE_LEN 16
+
+etcd_session
+etcd_open (etcd_server *server_list)
+{
+        return NULL;
+}
+
+typedef struct _etcd_sim_s {
+        char *path;
+} etcd_sim_t;
+
+void
+etcd_close (etcd_session this)
+{
+        etcd_sim_t *sim = (etcd_sim_t *)this;
+        free(sim->path);
+        free(this);
+}
+
+
+char *
+etcd_get_1 (FILE *stream, char *key)
+{
+        char *str = NULL;
+        size_t len;
+        unsigned long expires;
+	char *ret;
+
+        // Read the file
+        while(1) {
+                if(str) {
+                        free(str);
+                        str = NULL;
+                }
+                if (getline((char **)&str, &len,stream) == -1) {
+                        break;
+                }
+                if (!strncmp(str, key, strlen(key))) {
+			char k[256], s[256];
+			sscanf(str,"%s %s %lu",k, s, &expires);
+			// check if key is expired.
+			if (time(NULL) > expires) {
+				/* Keep looking for an unexpired entry. */
+				continue;
+			}
+		        ret = calloc(1, strlen(s) + 1);
+		        strcpy(ret,s);
+		        free(str);
+		        return(ret);
+                }
+        }
+        return NULL;
+}
+
+
+char *
+etcd_get (etcd_session this, char *key)
+{
+        etcd_sim_t      *sim            = (etcd_sim_t *)this;
+        int             fd;
+        FILE            *stream;
+        char            *retval;
+
+        fd = open(sim->path,O_RDONLY);
+        if (!fd) {
+                return NULL;
+        }
+
+        stream = fdopen(fd,"r");
+        (void)flock(fd,LOCK_SH);
+        retval = etcd_get_1(stream,key);
+        (void)flock(fd,LOCK_UN);
+        fclose(stream); /* closes fd as well */
+
+        return retval;
+}
+
+
+etcd_result
+etcd_set_1 (FILE *stream, char *key, char *value,
+            char *precond, unsigned int ttl)
+{
+        char *str = NULL;
+        char tp[255];
+        size_t len;
+        unsigned long expires;
+
+        while(1) {
+                if(str) {
+                        free(str);
+                        str = NULL;
+                }
+                if (getline((char **)&str, &len,stream) == -1) {
+                        break;
+                }
+                if (!strncmp(str, key, strlen(key))) {
+                        char k[256], s[256];
+                        sscanf(str,"%s %s %lu",k, s, &expires);
+                        // check if the present key is expired
+                        if (time(NULL) > expires) {
+				/* Keep looking for an unexpired entry. */
+				continue;
+			}
+			/*
+			 * The only case in which we should fail here is if a
+			 * precondition was specified and does not match the
+                         * current (non-expired) value.
+			 */
+			if (precond && strcmp(precond, s)) {
+				free(str);
+				return ETCD_WTF;
+			}
+                        fseek(stream, -strlen(str), SEEK_CUR);               
+                        free(str);
+                        goto here;
+                }
+        }
+here:
+        memset(tp, 0, 255);
+        sprintf(tp,"%*s %*s %*lu\n",
+                -MAX_KEY_LEN, key, -MAX_VALUE_LEN, value,
+                -MAX_EXPIRE_LEN, ttl ? time(NULL) + ttl : ~0);
+        if (fwrite(tp, 1,strlen(tp), stream) != strlen(tp)) {
+                return ETCD_WTF;
+        }
+        fflush(stream);
+        fsync(fileno(stream));
+        return ETCD_OK;
+}
+
+
+etcd_result
+etcd_set (etcd_session this, char *key, char *value,
+          char *precond, unsigned int ttl)
+{
+        etcd_sim_t      *sim = (etcd_sim_t *)this;
+        int             fd;
+        FILE            *stream;
+        etcd_result     retval;
+
+        fd = open(sim->path,O_RDWR);
+        if (fd < 0) {
+                return ETCD_WTF;
+        }
+
+        stream = fdopen(fd,"r+");
+        (void)flock(fd,LOCK_EX);
+        retval = etcd_set_1(stream,key,value,precond,ttl);
+        (void)flock(fd,LOCK_UN);
+        fclose(stream); /* closes fd as well */
+
+        return retval;
+}
+
+
+etcd_session
+etcd_open_str (char *server_names)
+{
+        etcd_sim_t      *sim;
+        int             fd;
+
+        sim = calloc(1, sizeof(etcd_sim_t));
+        (void)asprintf(&sim->path,"/tmp/%s",server_names);
+
+        fd = open(sim->path, O_RDWR | O_CREAT, 0777);
+        if (fd == -1) {
+                free(sim->path);
+                free(sim);
+                return NULL;
+        }
+
+        close(fd);
+        return ((void *)sim);
+}
+
+
+void
+etcd_close_str (etcd_session this)
+{
+        etcd_close(this);
+}
+
+etcd_result
+etcd_delete (etcd_session this, char *key)
+{
+	return ETCD_WTF;
+}
+
+char *
+etcd_leader (etcd_session this_as_void)
+{
+	return NULL;
+}
+
+etcd_result
+etcd_watch (etcd_session this, char *pfx, char **keyp, char **valuep,
+	    int *index_in, int *index_out)
+{
+	return ETCD_WTF;
+}
+
+etcd_result
+etcd_lock (etcd_session session_as_void, char *key, unsigned int ttl,
+           char *index_in, char **index_out)
+{
+        char    *path;
+        int     fd;
+
+        if (!index_in) {
+                if (gf_asprintf(&path,"/var/tmp/%s",key) < 0) {
+                        return ETCD_WTF;
+                }
+                fd = open(path,O_RDWR|O_CREAT,0666);
+                GF_FREE(path);
+                if (fd < 0) {
+                        return ETCD_WTF;
+                }
+                if (flock(fd,LOCK_EX) < 0) {
+                        close(fd);
+                        return ETCD_WTF;
+                }
+                *index_out = strdup("42");
+        }
+
+        /*
+         * Yes, we leak an fd by not closing it here (and nobody else even
+         * knows about it).  That would be awful in any other context, but
+         * for test scripts it won't matter.
+         */
+        return ETCD_OK;
+}
+
diff --git a/xlators/cluster/nsr-server/src/gen-fops.py b/xlators/cluster/nsr-server/src/gen-fops.py
new file mode 100755
index 000000000..1639f489c
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/gen-fops.py
@@ -0,0 +1,120 @@
+#!/usr/bin/python
+
+# This script generates the boilerplate versions of most fops and cbks in the
+# server.  This allows the details of leadership-status checking, sequencing
+# between leader and followers (including fan-out), and basic error checking
+# to be centralized one place, with per-operation code kept to a minimum.
+
+import sys
+import codegen
+
+type_re = "([a-z_0-9]+)"
+name_re = "\(\*fop_([a-z0-9]+)_t\)"
+full_re = type_re + " *" + name_re
+fop_cg = codegen.CodeGenerator()
+fop_cg.skip = 2
+fop_cg.parse_decls(sys.argv[1],full_re)
+fop_cg.load_templates(sys.argv[2])
+
+# Use the multi-template feature to generate multiple callbacks from the same
+# parsed declarations.
+type_re = "([a-z_0-9]+)"
+name_re = "\(\*fop_([a-z0-9]+)_cbk_t\)"
+full_re = type_re + " *" + name_re
+cbk_cg = codegen.CodeGenerator()
+cbk_cg.skip = 5
+cbk_cg.parse_decls(sys.argv[1],full_re)
+cbk_cg.load_templates(sys.argv[2])
+
+# This is a nasty little trick to handle the case where a generated fop needs
+# a set of default arguments for the corresponding callback.
+fop_cg.make_defaults = cbk_cg.make_defaults
+
+# We need two types of templates.  The first, for pure read operations, just
+# needs to do a simple am-i-leader check (augmented to allow dirty reads).
+# The second, for pure writes, needs to do fan-out to followers between those
+# initial checks and local execution.  There are other operations that don't
+# fit neatly into either category - e.g. lock ops or fsync - so we'll just have
+# to handle those manually.  The table thus includes entries only for those we
+# can categorize.  The special cases, plus any new operations we've never even
+# heard of, aren't in there.
+#
+# Various keywords can be used to define/undefine preprocessor symbols used
+# in the templates, on a per-function basis.  For example, if the keyword here
+# is "fsync" (lowercase word or abbreviation) that will cause NSR_CG_FSYNC
+# (prefix plus uppercase version) to be defined above all of the generated code
+# for that fop.
+
+fop_table = {
+	"access":		"read",
+	"create":		"write",
+	"discard":		"write",
+#	"entrylk":		"read",
+	"fallocate":	"write",
+#	"fentrylk":		"read",
+	"fgetxattr":	"read",
+#	"finodelk":		"read",
+#	"flush":		"read",
+	"fremovexattr":	"write",
+	"fsetattr":		"write",
+	"fsetxattr":	"write",
+	"fstat":		"read",
+#	"fsync":		"read",
+#	"fsyncdir":		"read",
+	"ftruncate":	"write",
+	"fxattrop":		"write",
+	"getxattr":		"read",
+#	"inodelk":		"read",
+	"link":			"write",
+#	"lk":			"read",
+#	"lookup":		"read",
+	"mkdir":		"write",
+	"mknod":		"write",
+	"open":			"write",
+	"opendir":		"read",
+	"rchecksum":	"read",
+	"readdir":		"read",
+	"readdirp":		"read",
+	"readlink":		"read",
+	"readv":		"read",
+	"removexattr":	"write",
+	"rename":		"write",
+	"rmdir":		"write",
+	"setattr":		"write",
+	"setxattr":		"write",
+	"stat":			"read",
+	"statfs":		"read",
+	"symlink":		"write",
+	"truncate":		"write",
+	"unlink":		"write",
+	"writev":		"write,fsync,queue",
+	"xattrop":		"write",
+}
+
+fops_done = []
+for x in sorted(fop_cg.decls.keys()):
+	if x in fop_table.keys():
+		info = fop_table[x].split(",")
+		kind = info[0]
+		flags = info[1:]
+		if ("fsync" in flags) or ("queue" in flags):
+			flags.append("need_fd")
+		for fname in flags:
+			print "#define NSR_CG_%s" % fname.upper()
+		cbk_cg.emit(x,kind+"-complete")
+		fop_cg.emit(x,kind+"-continue")
+		cbk_cg.emit(x,kind+"-fan-in")
+		fop_cg.emit(x,kind+"-dispatch")
+		fop_cg.emit(x,kind+"-fop")
+		for fname in flags:
+			print "#undef NSR_CG_%s" % fname.upper()
+		fops_done.append(x)
+	else:
+		print("/* No code emitted for %s */"%x)
+		print("")
+
+# Just for fun, emit the fops table too.
+print("struct xlator_fops fops = {")
+for x in fops_done:
+	print("	.%s = nsr_%s,"%(x,x))
+print("};")
diff --git a/xlators/cluster/nsr-server/src/leader.c b/xlators/cluster/nsr-server/src/leader.c
new file mode 100644
index 000000000..02a2609c8
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/leader.c
@@ -0,0 +1,138 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <regex.h>
+//#include <stdlib.h>
+#include <string.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "call-stub.h"
+#include "defaults.h"
+#include "xlator.h"
+#include "api/src/glfs.h"
+#include "api/src/glfs-internal.h"
+
+#ifndef NSR_SIM_ETCD
+#include "etcd-api.h"
+#endif
+#include "nsr-internal.h"
+#include "../../nsr-recon/src/recon_driver.h"
+#include "../../nsr-recon/src/recon_xlator.h"
+
+#define NSR_TTL         5
+
+static void
+nsr_set_leader (xlator_t *this, etcd_session etcd)
+{
+        long            term = 0;
+        etcd_result     res;
+        nsr_private_t   *priv   = this->private;
+        char             n_t[sizeof(long)+1];
+        char            *text   = NULL;
+
+        gf_log (this->name, GF_LOG_INFO, "Just became leader");
+
+        text = etcd_get(etcd, priv->term_key);
+        if(text == NULL) {
+                term = 0;
+        } else {
+                term = strtol(text, NULL, 10);
+        }
+        sprintf(n_t,"%ld",term+1);
+        res = etcd_set(etcd, priv->term_key,n_t,text,0);
+        if(res != ETCD_OK) {
+		gf_log (this->name, GF_LOG_ERROR, "failed to set term");
+		return;
+	}
+        priv->leader = _gf_true;
+
+        priv->current_term = term + 1;
+
+        if (priv->nsr_recon_start == _gf_false) {
+                atomic_fetch_and(&(priv->fence_io), 0);
+                return;
+        }
+
+        // Move this inside recon notify???
+        atomic_fetch_or(&(priv->fence_io), 1);
+
+        nsr_recon_notify_event_set_leader(priv);
+
+        return;
+}
+
+void *
+nsr_leader_thread (void *arg)
+{
+        xlator_t        *this           = (xlator_t *) arg;
+        nsr_private_t   *priv           = this->private;
+        etcd_result     res;
+        char            *index_in       = NULL;
+        char            *index_out      = NULL;
+
+        gf_log (this->name, GF_LOG_INFO,
+                "calling etcd_open_str on servers %s", priv->etcd_servers);
+
+        priv->etcd = etcd_open_str(priv->etcd_servers);
+        if (!(priv->etcd)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to open etcd session\n");
+                return NULL;
+        }
+
+        priv->leader_inited = 1;
+
+        for (;;) {
+                /* Not leader yet.  Try to become leader. */
+                for (;;) {
+                        res = etcd_lock (priv->etcd, priv->leader_key, NSR_TTL,
+                                         index_in, &index_out);
+                        if (res == ETCD_OK) {
+                                break;
+                        }
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "etcd_lock failed (%d)", res);
+                        sleep(1);
+                }
+                /* We're there.  Notify other parts of the code. */
+                nsr_set_leader(this,priv->etcd);
+                /* Try to retain leadership. */
+                index_in = index_out;
+                index_out = NULL;
+                for (;;) {
+                        res = etcd_lock (priv->etcd, priv->leader_key, NSR_TTL,
+                                         index_in, &index_out);
+                        if (index_out && (index_in != index_out)) {
+                                if (index_in) {
+                                        free(index_in);
+                                }
+                                index_in = index_out;
+                                index_out = NULL;
+                        }
+                        if (res != ETCD_OK) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "lost leadership (%d)", res);
+                                if (index_out) {
+                                        free(index_out);
+                                }
+                                break;
+                        }
+                        sleep(1);
+                }
+        }
+
+        etcd_close_str(priv->etcd);
+        return NULL;
+}
+
diff --git a/xlators/cluster/nsr-server/src/nsr-internal.h b/xlators/cluster/nsr-server/src/nsr-internal.h
new file mode 100644
index 000000000..72b61bfa5
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/nsr-internal.h
@@ -0,0 +1,101 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#define LEADER_XATTR            "user.nsr.leader"
+#define SECOND_CHILD(xl)        (xl->children->next->xlator)
+
+enum {
+        gf_mt_nsr_private_t = gf_common_mt_end + 1,
+        gf_mt_nsr_fd_ctx_t,
+        gf_mt_nsr_inode_ctx_t,
+        gf_mt_nsr_dirty_t,
+        gf_mt_nsr_end
+};
+
+typedef enum nsr_recon_notify_ev_id_t {
+        NSR_RECON_SET_LEADER = 1,
+        NSR_RECON_ADD_CHILD = 2
+} nsr_recon_notify_ev_id_t;
+
+typedef struct _nsr_recon_notify_ev_s {
+        nsr_recon_notify_ev_id_t id;
+        uint32_t index; // in case of add
+        struct list_head list;
+} nsr_recon_notify_ev_t;
+
+typedef struct {
+        char                    *etcd_servers;
+        char                    *subvol_uuid;
+        char                    *leader_key;
+        char                    *term_key;
+        char                    *brick_uuid;
+        gf_boolean_t            leader;
+        uint8_t                 up_children;
+        uint8_t                 n_children;
+        char                    *vol_file;
+        etcd_session            etcd;
+        volatile unsigned int   fence_io;
+        uint32_t                current_term;
+#ifdef  NSR_DEBUG
+        uint32_t                leader_log_fd;
+#endif
+        volatile int            recon_notify_inited;
+        volatile int            leader_inited;
+        uint32_t                kid_state;
+        gf_lock_t               dirty_lock;
+        struct list_head        dirty_fds;
+        gf_boolean_t            nsr_recon_start;
+        void *                  recon_ctx;
+        volatile uint32_t       ops_in_flight;
+	uint32_t                index;
+	gf_lock_t               index_lock;
+        double                  quorum_pct;
+} nsr_private_t;
+
+typedef struct {
+        call_stub_t             *stub;
+        call_stub_t             *qstub;
+        uint8_t                 call_count;
+        fd_t                    *fd;
+        struct list_head        qlinks;
+} nsr_local_t;
+
+/*
+ * This should match whatever changelog returns on the pre-op for us to pass
+ * when we're ready for our post-op.
+ */
+typedef uint32_t log_id_t;
+
+typedef struct {
+        struct list_head        links;
+        log_id_t                id;
+} nsr_dirty_list_t;
+
+typedef struct {
+        fd_t                    *fd;
+        struct list_head        dirty_list;
+        struct list_head        fd_list;
+} nsr_fd_ctx_t;
+
+typedef struct {
+        gf_lock_t               lock;
+        uint32_t                active;
+        struct list_head        aqueue;
+        uint32_t                pending;
+        struct list_head        pqueue;
+} nsr_inode_ctx_t;
+
+void nsr_recon_notify_event_set_leader(nsr_private_t *priv);
+void nsr_recon_notify_event_add_child(nsr_private_t *priv, uint32_t index);
+void* nsr_recon_notify_thread (void *this);
+
diff --git a/xlators/cluster/nsr-server/src/nsr.c b/xlators/cluster/nsr-server/src/nsr.c
new file mode 100644
index 000000000..85eba09b5
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/nsr.c
@@ -0,0 +1,812 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "call-stub.h"
+#include "defaults.h"
+#include "xlator.h"
+#include "api/src/glfs.h"
+#include "api/src/glfs-internal.h"
+#include "run.h"
+#include "common-utils.h"
+#include "syncop.h"
+
+#include "etcd-api.h"
+#include "nsr-internal.h"
+#include "../../nsr-recon/src/recon_driver.h"
+#include "../../nsr-recon/src/recon_xlator.h"
+
+
+#define GLUSTERD_DEFAULT_WORKDIR "/var/lib/glusterd"
+#define GLUSTERD_VOLUME_DIR_PREFIX "vols"
+#define GLUSTERD_BRICK_INFO_DIR "bricks"
+
+#define NSR_FLUSH_INTERVAL      5
+
+nsr_inode_ctx_t *
+nsr_get_inode_ctx (xlator_t *this, inode_t *inode)
+{
+        uint64_t                ctx_int         = 0LL;
+        nsr_inode_ctx_t         *ctx_ptr;
+
+        if (__inode_ctx_get(inode,this,&ctx_int) == 0) {
+                ctx_ptr = (nsr_inode_ctx_t *)(long)ctx_int;
+        }
+        else {
+                ctx_ptr = GF_CALLOC (1, sizeof(*ctx_ptr),
+                                     gf_mt_nsr_inode_ctx_t);
+                if (ctx_ptr) {
+                        ctx_int = (uint64_t)(long)ctx_ptr;
+                        if (__inode_ctx_set(inode,this,&ctx_int) == 0) {
+                                LOCK_INIT(&ctx_ptr->lock);
+                                INIT_LIST_HEAD(&ctx_ptr->aqueue);
+                                INIT_LIST_HEAD(&ctx_ptr->pqueue);
+                        }
+                        else {
+                                GF_FREE(ctx_ptr);
+                                ctx_ptr = NULL;
+                        }
+                }
+                                     
+        }
+
+        return ctx_ptr;
+}
+
+nsr_fd_ctx_t *
+nsr_get_fd_ctx (xlator_t *this, fd_t *fd)
+{
+        uint64_t                ctx_int         = 0LL;
+        nsr_fd_ctx_t            *ctx_ptr;
+
+        if (__fd_ctx_get(fd,this,&ctx_int) == 0) {
+                ctx_ptr = (nsr_fd_ctx_t *)(long)ctx_int;
+        }
+        else {
+                ctx_ptr = GF_CALLOC (1, sizeof(*ctx_ptr), gf_mt_nsr_fd_ctx_t);
+                if (ctx_ptr) {
+                        if (__fd_ctx_set(fd,this,(uint64_t)ctx_ptr) == 0) {
+                                INIT_LIST_HEAD(&ctx_ptr->dirty_list);
+                                INIT_LIST_HEAD(&ctx_ptr->fd_list);
+                        }
+                        else {
+                                GF_FREE(ctx_ptr);
+                                ctx_ptr = NULL;
+                        }
+                }
+                                     
+        }
+
+        return ctx_ptr;
+}
+
+void
+nsr_mark_fd_dirty (xlator_t *this, nsr_local_t *local)
+{
+        fd_t                    *fd             = local->fd;
+        nsr_fd_ctx_t            *ctx_ptr;
+        nsr_dirty_list_t        *dirty;
+        nsr_private_t           *priv           = this->private;
+
+        /*
+         * TBD: don't do any of this for O_SYNC/O_DIRECT writes.
+         * Unfortunately, that optimization requires that we distinguish
+         * between writev and other "write" calls, saving the original flags
+         * and checking them in the callback.  Too much work for too little
+         * gain right now.
+         */
+
+        LOCK(&fd->lock);
+                ctx_ptr = nsr_get_fd_ctx(this,fd);
+                dirty = GF_CALLOC(1,sizeof(*dirty),gf_mt_nsr_dirty_t);
+                if (ctx_ptr && dirty) {
+                        gf_log (this->name, GF_LOG_TRACE,
+                                "marking fd %p as dirty (%p)", fd, dirty);
+                        /* TBD: fill dirty->id from what changelog gave us */
+                        list_add_tail(&dirty->links,&ctx_ptr->dirty_list);
+                        if (list_empty(&ctx_ptr->fd_list)) {
+                                /* Add a ref so _release doesn't get called. */
+                                ctx_ptr->fd = fd_ref(fd);
+                                LOCK(&priv->dirty_lock);
+                                        list_add_tail (&ctx_ptr->fd_list,
+                                                       &priv->dirty_fds);
+                                UNLOCK(&priv->dirty_lock);
+                        }
+                }
+                else {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not mark %p dirty", fd);
+                        if (ctx_ptr) {
+                                GF_FREE(ctx_ptr);
+                        }
+                        if (dirty) {
+                                GF_FREE(dirty);
+                        }
+                }
+        UNLOCK(&fd->lock);
+}
+
+#define NSR_TERM_XATTR          "trusted.nsr.term"
+#define NSR_INDEX_XATTR         "trusted.nsr.index"
+#define RECON_TERM_XATTR        "trusted.nsr.recon-term"
+#define RECON_INDEX_XATTR       "trusted.nsr.recon-index"
+#define NSR_REP_COUNT_XATTR     "trusted.nsr.rep-count"
+#include "nsr-cg.c"
+
+uint8_t
+nsr_count_up_kids (nsr_private_t *priv)
+{
+        uint8_t         retval  = 0;
+        uint8_t         i;
+
+        for (i = 0; i < priv->n_children; ++i) {
+                if (priv->kid_state & (1 << i)) {
+                        ++retval;
+                }
+        }
+
+        return retval;
+}
+
+/*
+ * The fsync machinery looks a lot like that for any write call, but there are
+ * some important differences that are easy to miss.  First, we don't care
+ * about the xdata that shows whether the call came from a leader or
+ * reconciliation process.  If we're the leader we fan out; if we're not we
+ * don't.  Second, we don't wait for followers before we issue the local call.
+ * The code generation system could be updated to handle this, and still might
+ * if we need to implement other "almost identical" paths (e.g. for open), but
+ * a copy is more readable as long as it's just one.
+ */
+
+int32_t
+nsr_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
+{
+        nsr_local_t     *local  = frame->local;
+        gf_boolean_t    unwind;
+
+        LOCK(&frame->lock);
+                unwind = !--(local->call_count);
+        UNLOCK(&frame->lock);
+
+        if (unwind) {
+                STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf,
+                                     postbuf, xdata);
+        }
+        return 0;
+}
+
+int32_t
+nsr_fsync_local_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        nsr_dirty_list_t        *dirty;
+        nsr_dirty_list_t        *dtmp;
+        nsr_local_t             *local  = frame->local;
+
+        list_for_each_entry_safe (dirty, dtmp, &local->qlinks, links) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "sending post-op on %p (%p)", local->fd, dirty);
+                GF_FREE(dirty);
+        }
+
+        return nsr_fsync_cbk (frame, cookie, this, op_ret, op_errno,
+                              prebuf, postbuf, xdata);
+}
+
+int32_t
+nsr_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+           dict_t *xdata)
+{
+        nsr_private_t   *priv   = this->private;
+        nsr_local_t     *local;
+        uint64_t        ctx_int         = 0LL;
+        nsr_fd_ctx_t    *ctx_ptr;
+        xlator_list_t   *trav;
+
+        local = mem_get0(this->local_pool);
+        if (!local) {
+                STACK_UNWIND_STRICT(fsync,frame,-1,ENOMEM,NULL,NULL,xdata);
+                return 0;
+        }
+        INIT_LIST_HEAD(&local->qlinks);
+        frame->local = local;
+
+        /* Move the dirty list from the fd to the fsync request. */
+        LOCK(&fd->lock);
+                if (__fd_ctx_get(fd,this,&ctx_int) == 0) {
+                        ctx_ptr = (nsr_fd_ctx_t *)(long)ctx_int;
+                        list_splice_init (&ctx_ptr->dirty_list,
+                                          &local->qlinks);
+                }
+        UNLOCK(&fd->lock);
+
+        /* Issue the local call. */
+        local->call_count = priv->leader ? priv->n_children : 1;
+        STACK_WIND (frame, nsr_fsync_local_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->fsync,
+                    fd, flags, xdata);
+
+        /* Issue remote calls if we're the leader. */
+        if (priv->leader) {
+                for (trav = this->children->next; trav; trav = trav->next) {
+                        STACK_WIND (frame, nsr_fsync_cbk,
+                                    FIRST_CHILD(this),
+                                    FIRST_CHILD(this)->fops->fsync,
+                                    fd, flags, xdata);
+                }
+        }
+
+        return 0;
+}
+
+int32_t
+nsr_getxattr_special (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                      const char *name, dict_t *xdata)
+{
+        dict_t          *result;
+        nsr_private_t   *priv   = this->private;
+
+        if (!priv->leader) {
+                STACK_UNWIND_STRICT (getxattr, frame, -1, EREMOTE, NULL, NULL);
+                return 0;
+        }
+
+        if (!name || (strcmp(name,NSR_REP_COUNT_XATTR) != 0)) {
+                STACK_WIND_TAIL (frame,
+                                 FIRST_CHILD(this),
+                                 FIRST_CHILD(this)->fops->getxattr,
+                                 loc, name, xdata);
+                return 0;
+        }
+
+        result = dict_new();
+        if (!result) {
+                goto dn_failed;
+        }
+
+        priv->up_children = nsr_count_up_kids(this->private);
+        if (dict_set_uint32(result,NSR_REP_COUNT_XATTR,priv->up_children) != 0) {
+                goto dsu_failed;
+        }
+
+        STACK_UNWIND_STRICT (getxattr, frame, 0, 0, result, NULL);
+        dict_destroy(result);
+        return 0;
+
+dsu_failed:
+        dict_destroy(result);
+dn_failed:
+        STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
+        return 0;
+}
+
+void
+nsr_flush_fd (xlator_t *this, nsr_fd_ctx_t *fd_ctx)
+{
+        nsr_dirty_list_t        *dirty;
+        nsr_dirty_list_t        *dtmp;
+
+        list_for_each_entry_safe (dirty, dtmp, &fd_ctx->dirty_list, links) {
+                gf_log (this->name, GF_LOG_TRACE,
+                        "sending post-op on %p (%p)", fd_ctx->fd, dirty);
+                GF_FREE(dirty);
+        }
+
+        INIT_LIST_HEAD(&fd_ctx->dirty_list);
+}
+
+void *
+nsr_flush_thread (void *ctx)
+{
+        xlator_t                *this   = ctx;
+        nsr_private_t           *priv   = this->private;
+        struct list_head        dirty_fds;
+        nsr_fd_ctx_t            *fd_ctx;
+        nsr_fd_ctx_t            *fd_tmp;
+        int                     ret;
+
+        for (;;) {
+                /*
+                 * We have to be very careful to avoid lock inversions here, so
+                 * we can't just hold priv->dirty_lock while we take and
+                 * release locks for each fd.  Instead, we only hold dirty_lock
+                 * at the beginning of each iteration, as we (effectively) make
+                 * a copy of the current list head and then clear the original.
+                 * This leads to four scenarios for adding the first entry to
+                 * an fd and potentially putting it on the global list.
+                 *
+                 * (1) While we're asleep.  No lock contention, it just gets
+                 *     added and will be processed on the next iteration.
+                 *
+                 * (2) After we've made a local copy, but before we've started
+                 *     processing that fd.  The new entry will be added to the
+                 *     fd (under its lock), and we'll process it on the current
+                 *     iteration.
+                 *
+                 * (3) While we're processing the fd.  They'll block on the fd
+                 *     lock, then see that the list is empty and put it on the
+                 *     global list.  We'll process it here on the next
+                 *     iteration.
+                 *
+                 * (4) While we're working, but after we've processed that fd.
+                 *     Same as (1) as far as that fd is concerned.
+                 */
+                INIT_LIST_HEAD(&dirty_fds);
+                LOCK(&priv->dirty_lock);
+                list_splice_init(&priv->dirty_fds,&dirty_fds);
+                UNLOCK(&priv->dirty_lock);
+
+                list_for_each_entry_safe (fd_ctx, fd_tmp, &dirty_fds, fd_list) {
+                        ret = syncop_fsync(FIRST_CHILD(this),fd_ctx->fd,0);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "failed to fsync %p (%d)",
+                                        fd_ctx->fd, -ret);
+                        }
+
+                        LOCK(&fd_ctx->fd->lock);
+                                nsr_flush_fd(this,fd_ctx);
+                                list_del_init(&fd_ctx->fd_list);
+                        UNLOCK(&fd_ctx->fd->lock);
+                        fd_unref(fd_ctx->fd);
+                }
+                                          
+                sleep(NSR_FLUSH_INTERVAL);
+        }
+
+        return NULL;
+}
+
+int32_t
+nsr_forget (xlator_t *this, inode_t *inode)
+{
+        uint64_t        ctx     = 0LL;
+
+        if ((inode_ctx_del(inode,this,&ctx) == 0) && ctx) {
+                GF_FREE((void *)(long)ctx);
+        }
+
+        return 0;
+}
+
+int32_t
+nsr_release (xlator_t *this, fd_t *fd)
+{
+        uint64_t        ctx     = 0LL;
+
+        if ((fd_ctx_del(fd,this,&ctx) == 0) && ctx) {
+                GF_FREE((void *)(long)ctx);
+        }
+
+        return 0;
+}
+
+struct xlator_cbks cbks = {
+        .forget  = nsr_forget,
+        .release = nsr_release,
+};
+
+int
+nsr_reconfigure (xlator_t *this, dict_t *options)
+{
+        nsr_private_t   *priv   = this->private;
+
+	GF_OPTION_RECONF ("leader", priv->leader, options, bool, err);
+        gf_log (this->name, GF_LOG_INFO,
+                "reconfigure called. setting priv->leader to %d\n", priv->leader);
+        return 0;
+
+err:
+        return -1;
+}
+
+int
+nsr_get_child_index (xlator_t *this, xlator_t *kid)
+{
+        xlator_list_t   *trav;
+        int             retval = -1;
+
+        for (trav = this->children; trav; trav = trav->next) {
+                ++retval;
+                if (trav->xlator == kid) {
+                        return retval;
+                }
+        }
+
+        return -1;
+}
+
+/*
+ * Child notify handling is unreasonably FUBAR.  Sometimes we'll get a
+ * CHILD_DOWN for a protocol/client child before we ever got a CHILD_UP for it.
+ * Other times we won't.  Because it's effectively random (probably racy), we
+ * can't just maintain a count.  We actually have to keep track of the state
+ * for each child separately, to filter out the bogus CHILD_DOWN events, and
+ * then generate counts on demand.
+ */
+int
+nsr_notify (xlator_t *this, int event, void *data, ...)
+{
+        nsr_private_t   *priv   = this->private;
+        int             index;
+
+        switch (event) {
+        case GF_EVENT_CHILD_UP:
+                index = nsr_get_child_index(this,data);
+                if (index >= 0) {
+                        priv->kid_state |= (1 << index);
+                        priv->up_children = nsr_count_up_kids(priv);
+                        gf_log (this->name, GF_LOG_INFO,
+                                "got CHILD_UP for %s, now %u kids",
+                                ((xlator_t *)data)->name,
+                                priv->up_children);
+                        if (priv->nsr_recon_start == _gf_true) {
+                                nsr_recon_notify_event_add_child(priv, index);
+                        }
+                }
+                break;
+        case GF_EVENT_CHILD_DOWN:
+                index = nsr_get_child_index(this,data);
+                if (index >= 0) {
+                        priv->kid_state &= ~(1 << index);
+                        priv->up_children = nsr_count_up_kids(priv);
+                        gf_log (this->name, GF_LOG_INFO,
+                                "got CHILD_DOWN for %s, now %u kids",
+                                ((xlator_t *)data)->name,
+                                priv->up_children);
+                }
+                break;
+        default:
+                ;
+        }
+
+        return default_notify(this,event,data);
+}
+
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("nsr", this, out);
+
+        ret = xlator_mem_acct_init (this, gf_mt_nsr_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Memory accounting init" "failed");
+                return ret;
+        }
+out:
+        return ret;
+}
+
+
+extern void *nsr_leader_thread (void *);
+
+void
+nsr_deallocate_priv (nsr_private_t *priv)
+{
+        if (!priv) {
+                return;
+        }
+
+        if (priv->leader_key) {
+                GF_FREE(priv->leader_key);
+        }
+
+        if (priv->term_key) {
+                GF_FREE(priv->term_key);
+        }
+
+        GF_FREE(priv);
+}
+
+
+int32_t
+nsr_init (xlator_t *this)
+{
+        xlator_list_t   *remote;
+        xlator_list_t   *local;
+        nsr_private_t   *priv   = NULL;
+        xlator_list_t   *trav;
+        pthread_t       kid;
+        uuid_t          tmp_uuid;
+        char            *my_name = NULL, *morph_name = NULL, *recon_file = NULL, *recon_pid_file = NULL, *ptr = NULL;
+        char *volname;
+        extern xlator_t global_xlator;
+        glusterfs_ctx_t *oldctx = global_xlator.ctx;
+        runner_t runner  = {0,};
+        int32_t ret  = -1;
+        struct stat buf;
+        char *recon_log = NULL, *recon_log_dir = NULL;
+
+        /*
+         * Any fop that gets special treatment has to be patched in here,
+         * because the compiled-in table is produced by the code generator and
+         * only contains generated functions.  Note that we have to go through
+         * this->fops because of some dynamic-linking strangeness; modifying
+         * the static table doesn't work.
+         */
+        this->fops->getxattr = nsr_getxattr_special;
+        this->fops->fsync = nsr_fsync;
+
+        local = this->children;
+        if (!local) {
+                gf_log (this->name, GF_LOG_ERROR, "no local subvolume");
+                goto err;
+        }
+
+        remote = local->next;
+        if (!remote) {
+                gf_log (this->name, GF_LOG_ERROR, "no remote subvolumes");
+                goto err;
+        }
+
+        this->local_pool = mem_pool_new (nsr_local_t, 128);
+        if (!this->local_pool) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create nsr_local_t pool");
+                goto err;
+        }
+
+        priv = GF_CALLOC (1, sizeof(*priv), gf_mt_nsr_private_t);
+        if (!priv) {
+                gf_log (this->name, GF_LOG_ERROR, "could not allocate priv");
+                goto err;
+        }
+
+        // set this so that unless leader election is done, IO is fenced
+        priv->fence_io = 1;
+
+        for (trav = this->children; trav; trav = trav->next) {
+                ++(priv->n_children);
+        }
+
+        LOCK_INIT(&priv->dirty_lock);
+	LOCK_INIT(&priv->index_lock);
+        INIT_LIST_HEAD(&priv->dirty_fds);
+
+        this->private = priv;
+
+        GF_OPTION_INIT ("etcd-servers", priv->etcd_servers, str, err);
+        if (!priv->etcd_servers) {
+                gf_log (this->name, GF_LOG_ERROR, "etcd servers not generated. ???");
+                goto err;
+        }
+
+
+        GF_OPTION_INIT ("quorum-percent", priv->quorum_pct, percent, err);
+
+        GF_OPTION_INIT ("subvol-uuid", priv->subvol_uuid, str, err);
+        gf_log (this->name, GF_LOG_INFO, "subvol_uuid = %s", priv->subvol_uuid);
+        if (gf_asprintf(&priv->leader_key,"%s:leader",priv->subvol_uuid) <= 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not generate leader key");
+                goto err;
+        }
+        if (gf_asprintf(&priv->term_key,"%s:term",priv->subvol_uuid) <= 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not generate term key");
+                goto err;
+        }
+        uuid_generate(tmp_uuid);
+        priv->brick_uuid = strdup(uuid_utoa(tmp_uuid));
+        gf_log (this->name, GF_LOG_INFO, "brick_uuid = %s\n", priv->brick_uuid);
+
+        GF_OPTION_INIT ("my-name", my_name, str, err);
+        if (!my_name) {
+                gf_log (this->name, GF_LOG_ERROR, "brick name not generated. ???");
+                goto err;
+        }
+        GF_OPTION_INIT ("vol-name", volname, str, err);
+        if (!volname) {
+                gf_log (this->name, GF_LOG_ERROR, "vol name not generated. ???");
+                goto err;
+        }
+        
+        morph_name = GF_CALLOC (1, strlen(my_name) + 1, gf_mt_nsr_private_t);
+        strcpy(morph_name, my_name);
+        recon_file = GF_CALLOC (1,PATH_MAX + strlen(morph_name) + strlen("con") +1, gf_mt_nsr_private_t);
+        recon_pid_file = GF_CALLOC (1,PATH_MAX + strlen(morph_name) + strlen("recon") +1, gf_mt_nsr_private_t);
+        if ((!recon_file) || (!recon_pid_file)) {
+                gf_log (this->name, GF_LOG_ERROR, "could not allocate reconciliation file name");
+                goto err;
+        }
+        ptr = strchr (morph_name, '/');
+        while (ptr) {
+                *ptr = '-';
+                ptr = strchr (morph_name, '/');
+        }
+
+        sprintf(recon_file,"/%s/%s/%s/%s/",GLUSTERD_DEFAULT_WORKDIR,
+                                     GLUSTERD_VOLUME_DIR_PREFIX,
+                                     volname,
+                                     GLUSTERD_BRICK_INFO_DIR);
+        strcat(recon_file, morph_name);
+        strcat(recon_file, "-nsr-recon.vol");
+
+        sprintf(recon_pid_file,"/%s/%s/%s/%s/",GLUSTERD_DEFAULT_WORKDIR,
+                                     GLUSTERD_VOLUME_DIR_PREFIX,
+                                     volname,
+                                     "run");
+        strcat(recon_pid_file, morph_name);
+        strcat(recon_pid_file, "-recon.pid");
+
+        priv->vol_file = GF_CALLOC (1,PATH_MAX + strlen(morph_name) + strlen("con") +1, gf_mt_nsr_private_t);
+        if (!priv->vol_file) {
+                gf_log (this->name, GF_LOG_ERROR, "could not allocate reconciliation file name");
+                goto err;
+        }
+        sprintf(priv->vol_file,"%s/%s/%s/%s/",
+                GLUSTERD_DEFAULT_WORKDIR,
+                GLUSTERD_VOLUME_DIR_PREFIX,
+                volname,
+                GLUSTERD_BRICK_INFO_DIR);
+        strcat(priv->vol_file, "con:");
+        strcat(priv->vol_file, morph_name);
+
+        if (pthread_create(&kid,NULL,nsr_flush_thread,this) != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not start flush thread");
+                /* TBD: treat this as a fatal error? */
+        }
+
+        // Start the recon process. Then start the leader thread.
+        /*
+         * REVIEW
+         * Logs belong in /var/log not /tmp.
+         */
+
+        ret = mkdir (NSR_LOG_DIR, 0777);
+        if (ret != 0) {
+                if (errno != EEXIST) {
+                        gf_log (this->name, GF_LOG_ERROR, "Couldn't create"
+                                " nsr log directory (%s)", strerror (errno));
+                        goto err;
+                }
+        }
+
+        recon_log_dir = GF_CALLOC (1, strlen (NSR_LOG_DIR) + strlen(morph_name)
+                                   + 2, gf_mt_nsr_private_t);
+        if (!recon_log_dir) {
+                gf_log (this->name, GF_LOG_ERROR, "Couldn't allocate recon log "
+                        "dir name");
+                goto err;
+        }
+        sprintf (recon_log_dir, "%s/%s", NSR_LOG_DIR, morph_name);
+        ret = mkdir (recon_log_dir, 0777);
+
+        if (ret != 0){
+                if (errno != EEXIST) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Couldn't create brick log dir (%s)",
+                                strerror (errno));
+                        goto err;
+                }
+        }
+
+        recon_log = GF_CALLOC (1, strlen (recon_log_dir)+
+                               strlen ("reconciliation.log") + 2,
+                               gf_mt_nsr_private_t);
+        if (!recon_log) {
+                gf_log (this->name, GF_LOG_ERROR, "Couldn't allocate recon log"
+                        " file name");
+                goto err;
+        }
+        sprintf (recon_log, "%s/reconciliation.log", recon_log_dir);
+
+        if (!stat(priv->vol_file, &buf)) {
+
+                runinit (&runner);
+                runner_add_args(&runner, SBIN_DIR "/glusterfs",
+                        "-f", recon_file,
+                        "-p", recon_pid_file,
+                        "-l", recon_log,
+                        NULL);
+                ret = runner_run (&runner);
+                if (ret != 0)  {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not exec reconciliation process %s",
+                                SBIN_DIR "/glusterfs");
+                        goto err;
+                }
+
+                // TBD - convert this to make sure recon process runs
+                sleep(2);
+                priv->nsr_recon_start = _gf_true;
+        }
+
+
+        (void)pthread_create(&kid,NULL,nsr_recon_notify_thread,this);
+        while (priv->recon_notify_inited == 0) {
+                  sleep(1);
+        }
+
+        if (pthread_create(&kid,NULL,nsr_leader_thread,this) != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to start leader thread");
+        }
+        while (priv->leader_inited == 0) {
+                  sleep(1);
+        }
+
+
+        /*
+         * Calling glfs_new changes old->ctx, even if THIS still points
+         * to global_xlator.  That causes problems later in the main
+         * thread, when gf_log_dump_graph tries to use the FILE after
+         * we've mucked with it and gets a segfault in __fprintf_chk.
+         * We can avoid all that by undoing the damage before we
+         * continue.
+         */
+        global_xlator.ctx = oldctx;
+
+	return 0;
+
+err:
+        nsr_deallocate_priv(priv);
+        return -1;
+}
+
+
+void
+nsr_fini (xlator_t *this)
+{
+        nsr_deallocate_priv(this->private);
+}
+
+class_methods_t class_methods = {
+        .init           = nsr_init,
+        .fini           = nsr_fini,
+        .reconfigure    = nsr_reconfigure,
+        .notify         = nsr_notify,
+};
+
+struct volume_options options[] = {
+        { .key = {"leader"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "false",
+          .description = "Start in the leader role.  This is only for "
+                         "bootstrapping the code, and should go away when we "
+                         "have real leader election."
+        },
+        { .key ={"vol-name"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "volume name"
+        },
+        { .key = {"my-name"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "brick name in form of host:/path"
+        },
+        { .key = {"etcd-servers"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "list of comma seperated etc servers"
+        },
+        { .key = {"subvol-uuid"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "UUID for this NSR (sub)volume"
+        },
+        { .key = {"quorum-percent"},
+          .type = GF_OPTION_TYPE_PERCENT,
+          .default_value = "50.0",
+          .description = "percentage of rep_count-1 that must be up"
+        },
+	{ .key = {NULL} },
+};
diff --git a/xlators/cluster/nsr-server/src/recon_notify.c b/xlators/cluster/nsr-server/src/recon_notify.c
new file mode 100644
index 000000000..1c50de234
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/recon_notify.c
@@ -0,0 +1,389 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <string.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "call-stub.h"
+#include "defaults.h"
+#include "xlator.h"
+#include "api/src/glfs.h"
+#include "api/src/glfs-internal.h"
+#include "etcd-api.h"
+#include "nsr-internal.h"
+#include "../../nsr-recon/src/recon_driver.h"
+#include "../../nsr-recon/src/recon_xlator.h"
+
+
+
+typedef struct _nsr_recon_notify_ctx_t {
+        nsr_recon_notify_ev_t   recon_head;
+        pthread_mutex_t         recon_mutex;
+        pthread_cond_t          recon_cv;
+        char                    **hosts; // list of hosts ordered depending on child indices
+        uint32_t                current_term;
+        uint32_t                last_reconciled_term;
+        glfs_t                  *fs;
+        glfs_fd_t               *fd;
+} nsr_recon_notify_ctx_t;
+
+static int
+xlator_get_option (xlator_t *xl, char *key, char **value)
+{
+        GF_ASSERT (xl);
+        return dict_get_str (xl->options, key, value);
+}
+
+void nsr_recon_notify_event_set_leader(nsr_private_t *priv)
+{
+        nsr_recon_notify_ev_t *ev;
+        nsr_recon_notify_ctx_t *ctx = (nsr_recon_notify_ctx_t *)priv->recon_ctx;
+
+        ev = GF_CALLOC (1, sizeof (nsr_recon_notify_ev_t), 0);
+        ev->id = NSR_RECON_SET_LEADER;
+        INIT_LIST_HEAD(&(ev->list));
+        pthread_mutex_lock(&ctx->recon_mutex);
+        list_add_tail(&ev->list, &ctx->recon_head.list);
+        pthread_cond_signal(&ctx->recon_cv);
+        pthread_mutex_unlock(&ctx->recon_mutex);
+}
+
+void nsr_recon_notify_event_add_child(nsr_private_t *priv, uint32_t index) 
+{
+        nsr_recon_notify_ev_t *ev;
+        nsr_recon_notify_ctx_t *ctx = (nsr_recon_notify_ctx_t *)priv->recon_ctx;
+
+        ev = GF_CALLOC (1, sizeof (nsr_recon_notify_ev_t), 0);
+        ev->id = NSR_RECON_ADD_CHILD;
+        ev->index = index;
+        INIT_LIST_HEAD(&(ev->list));
+        pthread_mutex_lock(&ctx->recon_mutex);
+        list_add_tail(&ev->list, &ctx->recon_head.list);
+        pthread_cond_signal(&ctx->recon_cv);
+        pthread_mutex_unlock(&ctx->recon_mutex);
+}
+
+
+static void
+nsr_recon_set_leader (xlator_t *this)
+{
+
+        nsr_private_t   *priv   = this->private;
+        nsr_recon_notify_ctx_t *ctx = (nsr_recon_notify_ctx_t *)priv->recon_ctx;
+        nsr_recon_role_t role;
+        xlator_t        *old            = this;
+        uint32_t i=0;
+
+        if (priv->leader != _gf_true)
+                return;
+
+        if (ctx->last_reconciled_term == priv->current_term)
+                return;
+
+        /*
+         * Quorum for reconciliation is not the same as quorum for I/O.  Here,
+         * we require a true majority.  The +1 is because we don't count
+         * ourselves as part of n_children or up_children.
+         *
+         * TBD: re-evaluate when to reconcile (including partial)
+         */
+        if (priv->up_children <= (priv->n_children / 2))
+                return;
+
+	gf_log (this->name, GF_LOG_INFO,
+                "Sending message to do recon with %d nodes\n",
+                priv->up_children);
+
+	role.num = 0;
+        role.role = leader;
+        for (i = 0; i < priv->n_children; ++i) {
+                if (priv->kid_state & (1 << i)) {
+		        gf_log (this->name, GF_LOG_INFO,
+                                "Recon using host %s",
+                                 ctx->hosts[i]);
+                        strcpy(role.info[role.num].name, ctx->hosts[i]);
+                        (role.num)++;
+                }
+        }
+
+	gf_log (this->name, GF_LOG_INFO,
+                "setting current term as %d", priv->current_term);
+        role.current_term = priv->current_term;
+        ENDIAN_CONVERSION_RR(role, _gf_false);
+
+        // inform the reconciliator that this is leader
+        // in the callback (once reconciliation is done),
+        // we will unfence the IOs.
+        // TBD - error handling later.
+        if (glfs_lseek(ctx->fd, nsr_recon_xlator_sector_1, SEEK_SET) == -1) {
+		gf_log (this->name, GF_LOG_ERROR,
+                        "doing lseek failed\n");
+		return;
+	}
+
+        glusterfs_this_set(old);
+        gf_log (this->name, GF_LOG_INFO,
+                "Writing to local node to set leader");
+	do {
+		 if (priv->leader != _gf_true) {
+			glusterfs_this_set(old);
+			gf_log (this->name, GF_LOG_ERROR, "no longer leader\n");
+			return;
+		 }
+		 if (glfs_write(ctx->fd, &role, sizeof(role), 0) == -1) {
+			if (errno == EAGAIN) {
+				// Wait for old reconciliation to bail out.
+				glusterfs_this_set(old);
+				gf_log (this->name, GF_LOG_ERROR,
+                        		"write failed with retry. retrying after some time\n");
+				sleep(5);
+				continue;
+			}
+			else{
+				glusterfs_this_set(old);
+				gf_log (this->name, GF_LOG_ERROR,
+                        		"doing write failed\n");
+				// This is because reconciliation has returned with error
+				// because some node has died in between.
+				// What should be done? Either we retry being leader
+				// or hook to CHILD_DOWN notification.
+				// Put that logic later. As of now we will just retry.
+				// This is easier.
+				sleep(5);
+				continue;
+			}
+		} else {
+			glusterfs_this_set(old);
+			gf_log (this->name, GF_LOG_INFO, "doing write with success\n");
+			break;
+		}
+	} while(1);
+        glusterfs_this_set(old);
+	gf_log (this->name, GF_LOG_INFO,
+                "glfs_write returned. unfencing IO\n");
+
+        // TBD - error handling
+
+        ctx->last_reconciled_term = priv->current_term;
+	priv->index = 0; // reset changelog index
+        atomic_fetch_and(&(priv->fence_io), 0);
+
+        return;
+}
+
+static void
+nsr_recon_add_child (xlator_t *this, uint32_t index)
+{
+        nsr_private_t   *priv   = this->private;
+        nsr_recon_notify_ctx_t *ctx = (nsr_recon_notify_ctx_t *)priv->recon_ctx;
+        nsr_recon_role_t role;
+        xlator_t        *old            = this;
+
+        if (priv->leader != _gf_true)
+                return;
+
+        // reconciliation still pending.
+        // Check if we have majority
+        if (ctx->last_reconciled_term != priv->current_term) {
+                nsr_recon_set_leader(this);
+        } else {
+                // Reconciliation done.
+                // new child joining the majority/
+                // Do reconciliation only fot this child but after fencing new IO and draining old IO
+                role.num = 1;
+                role.role = joiner;
+
+                atomic_fetch_or(&(priv->fence_io), 1);
+                while(priv->ops_in_flight) {
+                        sleep(1);
+                }
+                        
+                strcpy(role.info[0].name, ctx->hosts[index]);
+                role.current_term = priv->current_term;
+                ENDIAN_CONVERSION_RR(role, _gf_false);
+                glfs_lseek(ctx->fd, nsr_recon_xlator_sector_1, SEEK_SET);
+                glusterfs_this_set(old);
+                gf_log (this->name, GF_LOG_INFO,
+                        "Writing to local node to join %s\n", role.info[0].name);
+                glfs_write(ctx->fd, &role,
+                           sizeof(role), 0);
+                glusterfs_this_set(old);
+                gf_log (this->name, GF_LOG_INFO,
+                        "Write to local node to set joiner returned\n");
+
+                // TBD - error handling
+                atomic_fetch_and(&(priv->fence_io), 0);
+        }
+
+        return;
+}
+
+static uint32_t
+nsr_setup_recon (xlator_t *this)
+{
+        nsr_private_t   *priv           = this->private;
+        xlator_t        *old            = this;
+        uint32_t         ret             = 0;
+        nsr_recon_notify_ctx_t *ctx = (nsr_recon_notify_ctx_t *)priv->recon_ctx;
+
+        if (priv->nsr_recon_start == _gf_false)
+                return 0;
+
+        ctx->fs = glfs_new(priv->subvol_uuid);
+        if (!ctx->fs) {
+                ret = 1;
+                gf_log (this->name, GF_LOG_ERROR, "failed to initialise glfs \n");
+                goto done;
+        }
+
+        glusterfs_this_set(old);
+        ret = glfs_set_volfile(ctx->fs, priv->vol_file);
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to set volfile \n");
+                goto done;
+        }
+
+        glusterfs_this_set(old);
+        /*
+         * REVIEW
+         * Logs belong in /var/log not /tmp.
+         */
+        glfs_set_logging (ctx->fs,"/tmp/glfs-log", 7);
+        if (glfs_init(ctx->fs) < 0) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to init volfile \n");
+                ret = 1;
+                goto done;
+        }
+
+        glusterfs_this_set(old);
+        ctx->fd = glfs_open (ctx->fs, "/", O_RDWR);
+        if (ctx->fd == NULL) {
+                ret = 1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to open fd to communicate with recon process \n");
+                goto done;
+        }
+
+
+done:
+        glusterfs_this_set(old);
+        return ret;
+}
+
+
+static void
+nsr_setup_hosts(xlator_t *this)
+{
+        xlator_list_t   *trav;
+        nsr_private_t   *priv   = this->private;
+        uint32_t        i       = 0;
+        nsr_recon_notify_ctx_t *ctx = (nsr_recon_notify_ctx_t *)priv->recon_ctx;
+
+        ctx->hosts = GF_CALLOC(sizeof(char *), priv->n_children, gf_mt_nsr_private_t);
+        // Iterate thru all the children
+        for (trav = this->children; trav; trav = trav->next) {
+                char            *hostname = NULL, *vol = NULL;
+                int             ret1 = 0, ret2 = 0, ret = 0;
+                xlator_t *xl = trav->xlator;
+                // If the child type is that of protocol/client
+                if (!strcmp(trav->xlator->type, "protocol/client")) {
+                        ret1 = xlator_get_option (xl, "remote-host", &hostname);
+                        ret2 = xlator_get_option (xl, "remote-subvolume", &vol);
+                        if (!ret1 && !ret2) {
+                                // add the name of that host to the hosts
+                                ctx->hosts[i] = GF_CALLOC(sizeof(char), strlen(hostname) + strlen(vol) + 2, 0);
+                                strcpy(ctx->hosts[i], hostname);
+                                strcat(ctx->hosts[i], ":");
+                                strcat(ctx->hosts[i], vol);
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "adding hosts %s to recon notfiy list", ctx->hosts[i]);
+                        } else {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "CANNOT FIND HOSTNAME FOR A CHILD");
+                                GF_ASSERT(0);
+                        }
+                //  local brick
+                } else {
+                        ret = xlator_get_option (this, "my-name", &hostname);
+                        if (!ret) {
+                                uint32_t len = strlen(hostname);
+                                ctx->hosts[i] = GF_CALLOC(sizeof(char),
+                                                           len+1,
+                                                           gf_mt_nsr_private_t);
+                                strcpy(ctx->hosts[i], hostname);
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "adding  my host %s to recon notfiy list", ctx->hosts[i]);
+                        } else {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "CANNOT FIND MY HOSTNAME");
+                                GF_ASSERT(0);
+                        }
+                }
+                i++;
+        }
+}
+
+void *
+nsr_recon_notify_thread (void *arg)
+{
+        xlator_t *this = (xlator_t *)arg;
+        nsr_private_t   *priv   = this->private;
+        nsr_recon_notify_ev_t *ev;
+        nsr_recon_notify_ctx_t *ctx;
+
+        priv->recon_ctx = GF_CALLOC(1, sizeof(nsr_recon_notify_ctx_t), gf_mt_nsr_private_t);
+        if (!priv->recon_ctx) {
+                gf_log (this->name, GF_LOG_ERROR, "calloc error");
+                return NULL;
+        }
+        ctx = priv->recon_ctx;
+
+        pthread_mutex_init(&(ctx->recon_mutex), NULL);
+        pthread_cond_init(&(ctx->recon_cv), NULL);
+        INIT_LIST_HEAD(&(ctx->recon_head.list));
+
+        nsr_setup_hosts(this);
+
+        if (nsr_setup_recon(this)) {
+                gf_log (this->name, GF_LOG_ERROR, "recon notify thread : initing glfs error");
+                return NULL;
+        }
+
+        priv->recon_notify_inited = 1;
+
+        while(1) {
+                pthread_mutex_lock(&ctx->recon_mutex);
+                while (list_empty(&(ctx->recon_head.list))) {
+                        pthread_cond_wait(&ctx->recon_cv, &ctx->recon_mutex);
+                }
+                pthread_mutex_unlock(&ctx->recon_mutex);
+
+                list_for_each_entry(ev, &(ctx->recon_head.list), list) {
+
+                        if (ev->id == NSR_RECON_SET_LEADER) {
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "got add leader notfiy event");
+                                nsr_recon_set_leader(this);
+                        } else if (ev->id == NSR_RECON_ADD_CHILD) {
+                                gf_log (this->name, GF_LOG_INFO,
+                                        "got add child notify event");
+                                nsr_recon_add_child(this, ev->index);
+                        }
+                }
+                list_del_init (&ev->list);
+        }
+
+        return NULL;
+}
+
diff --git a/xlators/cluster/nsr-server/src/yajl.c b/xlators/cluster/nsr-server/src/yajl.c
new file mode 100644
index 000000000..54e6474fc
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/yajl.c
@@ -0,0 +1,175 @@
+/*
+ * Copyright (c) 2007-2011, Lloyd Hilaiel <lloyd@hilaiel.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "yajl/yajl_parse.h"
+#include "yajl_lex.h"
+#include "yajl_parser.h"
+#include "yajl_alloc.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdarg.h>
+#include <assert.h>
+
+const char *
+yajl_status_to_string(yajl_status stat)
+{
+    const char * statStr = "unknown";
+    switch (stat) {
+        case yajl_status_ok:
+            statStr = "ok, no error";
+            break;
+        case yajl_status_client_canceled:
+            statStr = "client canceled parse";
+            break;
+        case yajl_status_error:
+            statStr = "parse error";
+            break;
+    }
+    return statStr;
+}
+
+yajl_handle
+yajl_alloc(const yajl_callbacks * callbacks,
+           yajl_alloc_funcs * afs,
+           void * ctx)
+{
+    yajl_handle hand = NULL;
+    yajl_alloc_funcs afsBuffer;
+
+    /* first order of business is to set up memory allocation routines */
+    if (afs != NULL) {
+        if (afs->malloc == NULL || afs->realloc == NULL || afs->free == NULL)
+        {
+            return NULL;
+        }
+    } else {
+        yajl_set_default_alloc_funcs(&afsBuffer);
+        afs = &afsBuffer;
+    }
+
+    hand = (yajl_handle) YA_MALLOC(afs, sizeof(struct yajl_handle_t));
+
+    /* copy in pointers to allocation routines */
+    memcpy((void *) &(hand->alloc), (void *) afs, sizeof(yajl_alloc_funcs));
+
+    hand->callbacks = callbacks;
+    hand->ctx = ctx;
+    hand->lexer = NULL; 
+    hand->bytesConsumed = 0;
+    hand->decodeBuf = yajl_buf_alloc(&(hand->alloc));
+    hand->flags	    = 0;
+    yajl_bs_init(hand->stateStack, &(hand->alloc));
+    yajl_bs_push(hand->stateStack, yajl_state_start);
+
+    return hand;
+}
+
+int
+yajl_config(yajl_handle h, yajl_option opt, ...)
+{
+    int rv = 1;
+    va_list ap;
+    va_start(ap, opt);
+
+    switch(opt) {
+        case yajl_allow_comments:
+        case yajl_dont_validate_strings:
+        case yajl_allow_trailing_garbage:
+        case yajl_allow_multiple_values:
+        case yajl_allow_partial_values:
+            if (va_arg(ap, int)) h->flags |= opt;
+            else h->flags &= ~opt;
+            break;
+        default:
+            rv = 0;
+    }
+    va_end(ap);
+
+    return rv;
+}
+
+void
+yajl_free(yajl_handle handle)
+{
+    yajl_bs_free(handle->stateStack);
+    yajl_buf_free(handle->decodeBuf);
+    if (handle->lexer) {
+        yajl_lex_free(handle->lexer);
+        handle->lexer = NULL;
+    }
+    YA_FREE(&(handle->alloc), handle);
+}
+
+yajl_status
+yajl_parse(yajl_handle hand, const unsigned char * jsonText,
+           size_t jsonTextLen)
+{
+    yajl_status status;
+
+    /* lazy allocation of the lexer */
+    if (hand->lexer == NULL) {
+        hand->lexer = yajl_lex_alloc(&(hand->alloc),
+                                     hand->flags & yajl_allow_comments,
+                                     !(hand->flags & yajl_dont_validate_strings));
+    }
+
+    status = yajl_do_parse(hand, jsonText, jsonTextLen);
+    return status;
+}
+
+
+yajl_status
+yajl_complete_parse(yajl_handle hand)
+{
+    /* The lexer is lazy allocated in the first call to parse.  if parse is
+     * never called, then no data was provided to parse at all.  This is a
+     * "premature EOF" error unless yajl_allow_partial_values is specified.
+     * allocating the lexer now is the simplest possible way to handle this
+     * case while preserving all the other semantics of the parser
+     * (multiple values, partial values, etc). */
+    if (hand->lexer == NULL) {
+        hand->lexer = yajl_lex_alloc(&(hand->alloc),
+                                     hand->flags & yajl_allow_comments,
+                                     !(hand->flags & yajl_dont_validate_strings));
+    }
+
+    return yajl_do_finish(hand);
+}
+
+unsigned char *
+yajl_get_error(yajl_handle hand, int verbose,
+               const unsigned char * jsonText, size_t jsonTextLen)
+{
+    return yajl_render_error_string(hand, jsonText, jsonTextLen, verbose);
+}
+
+size_t
+yajl_get_bytes_consumed(yajl_handle hand)
+{
+    if (!hand) return 0;
+    else return hand->bytesConsumed;
+}
+
+
+void
+yajl_free_error(yajl_handle hand, unsigned char * str)
+{
+    /* use memory allocation functions if set */
+    YA_FREE(&(hand->alloc), str);
+}
+
+/* XXX: add utility routines to parse from file */
diff --git a/xlators/cluster/nsr-server/src/yajl/yajl_common.h b/xlators/cluster/nsr-server/src/yajl/yajl_common.h
new file mode 100644
index 000000000..49ca3a5cb
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/yajl/yajl_common.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2007-2011, Lloyd Hilaiel <lloyd@hilaiel.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef __YAJL_COMMON_H__
+#define __YAJL_COMMON_H__
+
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif    
+
+#define YAJL_MAX_DEPTH 128
+
+/* msft dll export gunk.  To build a DLL on windows, you
+ * must define WIN32, YAJL_SHARED, and YAJL_BUILD.  To use a shared
+ * DLL, you must define YAJL_SHARED and WIN32 */
+#if defined(WIN32) && defined(YAJL_SHARED)
+#  ifdef YAJL_BUILD
+#    define YAJL_API __declspec(dllexport)
+#  else
+#    define YAJL_API __declspec(dllimport)
+#  endif
+#else
+#  if defined(__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__) >= 303
+#    define YAJL_API __attribute__ ((visibility("default")))
+#  else
+#    define YAJL_API
+#  endif
+#endif 
+
+/** pointer to a malloc function, supporting client overriding memory
+ *  allocation routines */
+typedef void * (*yajl_malloc_func)(void *ctx, size_t sz);
+
+/** pointer to a free function, supporting client overriding memory
+ *  allocation routines */
+typedef void (*yajl_free_func)(void *ctx, void * ptr);
+
+/** pointer to a realloc function which can resize an allocation. */
+typedef void * (*yajl_realloc_func)(void *ctx, void * ptr, size_t sz);
+
+/** A structure which can be passed to yajl_*_alloc routines to allow the
+ *  client to specify memory allocation functions to be used. */
+typedef struct
+{
+    /** pointer to a function that can allocate uninitialized memory */
+    yajl_malloc_func malloc;
+    /** pointer to a function that can resize memory allocations */
+    yajl_realloc_func realloc;
+    /** pointer to a function that can free memory allocated using
+     *  reallocFunction or mallocFunction */
+    yajl_free_func free;
+    /** a context pointer that will be passed to above allocation routines */
+    void * ctx;
+} yajl_alloc_funcs;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/xlators/cluster/nsr-server/src/yajl/yajl_gen.h b/xlators/cluster/nsr-server/src/yajl/yajl_gen.h
new file mode 100644
index 000000000..52fa99fc2
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/yajl/yajl_gen.h
@@ -0,0 +1,157 @@
+/*
+ * Copyright (c) 2007-2011, Lloyd Hilaiel <lloyd@hilaiel.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+/**
+ * \file yajl_gen.h
+ * Interface to YAJL's JSON generation facilities.
+ */
+
+#include <yajl/yajl_common.h>
+
+#ifndef __YAJL_GEN_H__
+#define __YAJL_GEN_H__
+
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+    /** generator status codes */
+    typedef enum {
+        /** no error */
+        yajl_gen_status_ok = 0,
+        /** at a point where a map key is generated, a function other than
+         *  yajl_gen_string was called */
+        yajl_gen_keys_must_be_strings,
+        /** YAJL's maximum generation depth was exceeded.  see
+         *  YAJL_MAX_DEPTH */
+        yajl_max_depth_exceeded,
+        /** A generator function (yajl_gen_XXX) was called while in an error
+         *  state */
+        yajl_gen_in_error_state,
+        /** A complete JSON document has been generated */
+        yajl_gen_generation_complete,                
+        /** yajl_gen_double was passed an invalid floating point value
+         *  (infinity or NaN). */
+        yajl_gen_invalid_number,
+        /** A print callback was passed in, so there is no internal
+         * buffer to get from */
+        yajl_gen_no_buf,
+        /** returned from yajl_gen_string() when the yajl_gen_validate_utf8
+         *  option is enabled and an invalid was passed by client code.
+         */
+        yajl_gen_invalid_string
+    } yajl_gen_status;
+
+    /** an opaque handle to a generator */
+    typedef struct yajl_gen_t * yajl_gen;
+
+    /** a callback used for "printing" the results. */
+    typedef void (*yajl_print_t)(void * ctx,
+                                 const char * str,
+                                 size_t len);
+
+    /** configuration parameters for the parser, these may be passed to
+     *  yajl_gen_config() along with option specific argument(s).  In general,
+     *  all configuration parameters default to *off*. */
+    typedef enum {
+        /** generate indented (beautiful) output */
+        yajl_gen_beautify = 0x01,
+        /**
+         * Set an indent string which is used when yajl_gen_beautify
+         * is enabled.  Maybe something like \\t or some number of
+         * spaces.  The default is four spaces ' '.
+         */
+        yajl_gen_indent_string = 0x02,
+        /**
+         * Set a function and context argument that should be used to
+         * output generated json.  the function should conform to the
+         * yajl_print_t prototype while the context argument is a
+         * void * of your choosing.
+         *
+         * example:
+         *   yajl_gen_config(g, yajl_gen_print_callback, myFunc, myVoidPtr);
+         */
+        yajl_gen_print_callback = 0x04,
+        /**
+         * Normally the generator does not validate that strings you
+         * pass to it via yajl_gen_string() are valid UTF8.  Enabling
+         * this option will cause it to do so.
+         */
+        yajl_gen_validate_utf8 = 0x08,
+        /**
+         * the forward solidus (slash or '/' in human) is not required to be
+         * escaped in json text.  By default, YAJL will not escape it in the
+         * iterest of saving bytes.  Setting this flag will cause YAJL to
+         * always escape '/' in generated JSON strings.
+         */
+        yajl_gen_escape_solidus = 0x10
+    } yajl_gen_option;
+
+    /** allow the modification of generator options subsequent to handle
+     *  allocation (via yajl_alloc)
+     *  \returns zero in case of errors, non-zero otherwise
+     */
+    YAJL_API int yajl_gen_config(yajl_gen g, yajl_gen_option opt, ...);
+
+    /** allocate a generator handle
+     *  \param allocFuncs an optional pointer to a structure which allows
+     *                    the client to overide the memory allocation
+     *                    used by yajl.  May be NULL, in which case
+     *                    malloc/free/realloc will be used.
+     *
+     *  \returns an allocated handle on success, NULL on failure (bad params)
+     */
+    YAJL_API yajl_gen yajl_gen_alloc(const yajl_alloc_funcs * allocFuncs);
+
+    /** free a generator handle */
+    YAJL_API void yajl_gen_free(yajl_gen handle);
+
+    YAJL_API yajl_gen_status yajl_gen_integer(yajl_gen hand, long long int number);
+    /** generate a floating point number.  number may not be infinity or
+     *  NaN, as these have no representation in JSON.  In these cases the
+     *  generator will return 'yajl_gen_invalid_number' */
+    YAJL_API yajl_gen_status yajl_gen_double(yajl_gen hand, double number);
+    YAJL_API yajl_gen_status yajl_gen_number(yajl_gen hand,
+                                             const char * num,
+                                             size_t len);
+    YAJL_API yajl_gen_status yajl_gen_string(yajl_gen hand,
+                                             const unsigned char * str,
+                                             size_t len);
+    YAJL_API yajl_gen_status yajl_gen_null(yajl_gen hand);
+    YAJL_API yajl_gen_status yajl_gen_bool(yajl_gen hand, int boolean);
+    YAJL_API yajl_gen_status yajl_gen_map_open(yajl_gen hand);
+    YAJL_API yajl_gen_status yajl_gen_map_close(yajl_gen hand);
+    YAJL_API yajl_gen_status yajl_gen_array_open(yajl_gen hand);
+    YAJL_API yajl_gen_status yajl_gen_array_close(yajl_gen hand);
+
+    /** access the null terminated generator buffer.  If incrementally
+     *  outputing JSON, one should call yajl_gen_clear to clear the
+     *  buffer.  This allows stream generation. */
+    YAJL_API yajl_gen_status yajl_gen_get_buf(yajl_gen hand,
+                                              const unsigned char ** buf,
+                                              size_t * len);
+
+    /** clear yajl's output buffer, but maintain all internal generation
+     *  state.  This function will not "reset" the generator state, and is
+     *  intended to enable incremental JSON outputing. */
+    YAJL_API void yajl_gen_clear(yajl_gen hand);
+
+#ifdef __cplusplus
+}
+#endif    
+
+#endif
diff --git a/xlators/cluster/nsr-server/src/yajl/yajl_parse.h b/xlators/cluster/nsr-server/src/yajl/yajl_parse.h
new file mode 100644
index 000000000..55c831101
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/yajl/yajl_parse.h
@@ -0,0 +1,226 @@
+/*
+ * Copyright (c) 2007-2011, Lloyd Hilaiel <lloyd@hilaiel.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+/**
+ * \file yajl_parse.h
+ * Interface to YAJL's JSON stream parsing facilities.
+ */
+
+#include <yajl/yajl_common.h>
+
+#ifndef __YAJL_PARSE_H__
+#define __YAJL_PARSE_H__
+
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+    /** error codes returned from this interface */
+    typedef enum {
+        /** no error was encountered */
+        yajl_status_ok,
+        /** a client callback returned zero, stopping the parse */
+        yajl_status_client_canceled,
+        /** An error occured during the parse.  Call yajl_get_error for
+         *  more information about the encountered error */
+        yajl_status_error
+    } yajl_status;
+
+    /** attain a human readable, english, string for an error */
+    YAJL_API const char * yajl_status_to_string(yajl_status code);
+
+    /** an opaque handle to a parser */
+    typedef struct yajl_handle_t * yajl_handle;
+
+    /** yajl is an event driven parser.  this means as json elements are
+     *  parsed, you are called back to do something with the data.  The
+     *  functions in this table indicate the various events for which
+     *  you will be called back.  Each callback accepts a "context"
+     *  pointer, this is a void * that is passed into the yajl_parse
+     *  function which the client code may use to pass around context.
+     *
+     *  All callbacks return an integer.  If non-zero, the parse will
+     *  continue.  If zero, the parse will be canceled and
+     *  yajl_status_client_canceled will be returned from the parse.
+     *
+     *  \attention {
+     *    A note about the handling of numbers:
+     *
+     *    yajl will only convert numbers that can be represented in a
+     *    double or a 64 bit (long long) int.  All other numbers will
+     *    be passed to the client in string form using the yajl_number
+     *    callback.  Furthermore, if yajl_number is not NULL, it will
+     *    always be used to return numbers, that is yajl_integer and
+     *    yajl_double will be ignored.  If yajl_number is NULL but one
+     *    of yajl_integer or yajl_double are defined, parsing of a
+     *    number larger than is representable in a double or 64 bit
+     *    integer will result in a parse error.
+     *  }
+     */
+    typedef struct {
+        int (* yajl_null)(void * ctx);
+        int (* yajl_boolean)(void * ctx, int boolVal);
+        int (* yajl_integer)(void * ctx, long long integerVal);
+        int (* yajl_double)(void * ctx, double doubleVal);
+        /** A callback which passes the string representation of the number
+         *  back to the client.  Will be used for all numbers when present */
+        int (* yajl_number)(void * ctx, const char * numberVal,
+                            size_t numberLen);
+
+        /** strings are returned as pointers into the JSON text when,
+         * possible, as a result, they are _not_ null padded */
+        int (* yajl_string)(void * ctx, const unsigned char * stringVal,
+                            size_t stringLen);
+
+        int (* yajl_start_map)(void * ctx);
+        int (* yajl_map_key)(void * ctx, const unsigned char * key,
+                             size_t stringLen);
+        int (* yajl_end_map)(void * ctx);
+
+        int (* yajl_start_array)(void * ctx);
+        int (* yajl_end_array)(void * ctx);
+    } yajl_callbacks;
+
+    /** allocate a parser handle
+     *  \param callbacks  a yajl callbacks structure specifying the
+     *                    functions to call when different JSON entities
+     *                    are encountered in the input text.  May be NULL,
+     *                    which is only useful for validation.
+     *  \param afs        memory allocation functions, may be NULL for to use
+     *                    C runtime library routines (malloc and friends) 
+     *  \param ctx        a context pointer that will be passed to callbacks.
+     */
+    YAJL_API yajl_handle yajl_alloc(const yajl_callbacks * callbacks,
+                                    yajl_alloc_funcs * afs,
+                                    void * ctx);
+
+
+    /** configuration parameters for the parser, these may be passed to
+     *  yajl_config() along with option specific argument(s).  In general,
+     *  all configuration parameters default to *off*. */
+    typedef enum {
+        /** Ignore javascript style comments present in
+         *  JSON input.  Non-standard, but rather fun
+         *  arguments: toggled off with integer zero, on otherwise.
+         *
+         *  example:
+         *    yajl_config(h, yajl_allow_comments, 1); // turn comment support on
+         */
+        yajl_allow_comments = 0x01,
+        /**
+         * When set the parser will verify that all strings in JSON input are
+         * valid UTF8 and will emit a parse error if this is not so.  When set,
+         * this option makes parsing slightly more expensive (~7% depending
+         * on processor and compiler in use)
+         *
+         * example:
+         *   yajl_config(h, yajl_dont_validate_strings, 1); // disable utf8 checking
+         */
+        yajl_dont_validate_strings     = 0x02,
+        /**
+         * By default, upon calls to yajl_complete_parse(), yajl will
+         * ensure the entire input text was consumed and will raise an error
+         * otherwise.  Enabling this flag will cause yajl to disable this
+         * check.  This can be useful when parsing json out of a that contains more
+         * than a single JSON document.
+         */
+        yajl_allow_trailing_garbage = 0x04,
+        /**
+         * Allow multiple values to be parsed by a single handle.  The
+         * entire text must be valid JSON, and values can be seperated
+         * by any kind of whitespace.  This flag will change the
+         * behavior of the parser, and cause it continue parsing after
+         * a value is parsed, rather than transitioning into a
+         * complete state.  This option can be useful when parsing multiple
+         * values from an input stream.
+         */
+        yajl_allow_multiple_values = 0x08,
+        /**
+         * When yajl_complete_parse() is called the parser will
+         * check that the top level value was completely consumed.  I.E.,
+         * if called whilst in the middle of parsing a value
+         * yajl will enter an error state (premature EOF).  Setting this
+         * flag suppresses that check and the corresponding error.
+         */
+        yajl_allow_partial_values = 0x10
+    } yajl_option;
+
+    /** allow the modification of parser options subsequent to handle
+     *  allocation (via yajl_alloc)
+     *  \returns zero in case of errors, non-zero otherwise
+     */
+    YAJL_API int yajl_config(yajl_handle h, yajl_option opt, ...);
+
+    /** free a parser handle */
+    YAJL_API void yajl_free(yajl_handle handle);
+
+    /** Parse some json!
+     *  \param hand - a handle to the json parser allocated with yajl_alloc
+     *  \param jsonText - a pointer to the UTF8 json text to be parsed
+     *  \param jsonTextLength - the length, in bytes, of input text
+     */
+    YAJL_API yajl_status yajl_parse(yajl_handle hand,
+                                    const unsigned char * jsonText,
+                                    size_t jsonTextLength);
+
+    /** Parse any remaining buffered json.
+     *  Since yajl is a stream-based parser, without an explicit end of
+     *  input, yajl sometimes can't decide if content at the end of the
+     *  stream is valid or not.  For example, if "1" has been fed in,
+     *  yajl can't know whether another digit is next or some character
+     *  that would terminate the integer token.
+     *
+     *  \param hand - a handle to the json parser allocated with yajl_alloc
+     */
+    YAJL_API yajl_status yajl_complete_parse(yajl_handle hand);
+
+    /** get an error string describing the state of the
+     *  parse.
+     *
+     *  If verbose is non-zero, the message will include the JSON
+     *  text where the error occured, along with an arrow pointing to
+     *  the specific char.
+     *
+     *  \returns A dynamically allocated string will be returned which should
+     *  be freed with yajl_free_error
+     */
+    YAJL_API unsigned char * yajl_get_error(yajl_handle hand, int verbose,
+                                            const unsigned char * jsonText,
+                                            size_t jsonTextLength);
+
+    /**
+     * get the amount of data consumed from the last chunk passed to YAJL.
+     *
+     * In the case of a successful parse this can help you understand if
+     * the entire buffer was consumed (which will allow you to handle
+     * "junk at end of input").
+     *
+     * In the event an error is encountered during parsing, this function
+     * affords the client a way to get the offset into the most recent
+     * chunk where the error occured.  0 will be returned if no error
+     * was encountered.
+     */
+    YAJL_API size_t yajl_get_bytes_consumed(yajl_handle hand);
+
+    /** free an error returned from yajl_get_error */
+    YAJL_API void yajl_free_error(yajl_handle hand, unsigned char * str);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/xlators/cluster/nsr-server/src/yajl/yajl_tree.h b/xlators/cluster/nsr-server/src/yajl/yajl_tree.h
new file mode 100644
index 000000000..8b377f636
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/yajl/yajl_tree.h
@@ -0,0 +1,177 @@
+/*
+ * Copyright (c) 2010-2011  Florian Forster  <ff at octo.it>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+/**
+ * \file yajl_tree.h
+ *
+ * Parses JSON data and returns the data in tree form.
+ *
+ * \author Florian Forster
+ * \date August 2010
+ *
+ * This interface makes quick parsing and extraction of
+ * smallish JSON docs trivial:
+ *
+ * \include example/parse_config.c
+ */
+
+#ifndef YAJL_TREE_H
+#define YAJL_TREE_H 1
+
+#include <yajl/yajl_common.h>
+
+/** possible data types that a yajl_val_s can hold */
+typedef enum {
+    yajl_t_string = 1,
+    yajl_t_number = 2,
+    yajl_t_object = 3,
+    yajl_t_array = 4,
+    yajl_t_true = 5,
+    yajl_t_false = 6,
+    yajl_t_null = 7,
+    /** The any type isn't valid for yajl_val_s.type, but can be
+     *  used as an argument to routines like yajl_tree_get().
+     */
+    yajl_t_any = 8
+} yajl_type;
+
+#define YAJL_NUMBER_INT_VALID    0x01
+#define YAJL_NUMBER_DOUBLE_VALID 0x02
+
+/** A pointer to a node in the parse tree */
+typedef struct yajl_val_s * yajl_val;
+
+/**
+ * A JSON value representation capable of holding one of the seven
+ * types above. For "string", "number", "object", and "array"
+ * additional data is available in the union.  The "YAJL_IS_*"
+ * and "YAJL_GET_*" macros below allow type checking and convenient
+ * value extraction.
+ */
+struct yajl_val_s
+{
+    /** Type of the value contained. Use the "YAJL_IS_*" macors to check for a
+     * specific type. */
+    yajl_type type;
+    /** Type-specific data. You may use the "YAJL_GET_*" macros to access these
+     * members. */
+    union
+    {
+        char * string;
+        struct {
+            long long i; /*< integer value, if representable. */
+            double  d;   /*< double value, if representable. */
+            /** Signals whether the \em i and \em d members are
+             * valid. See \c YAJL_NUMBER_INT_VALID and
+             * \c YAJL_NUMBER_DOUBLE_VALID. */
+            char   *r;   /*< unparsed number in string form. */
+            unsigned int flags;
+        } number;
+        struct {
+            const char **keys; /*< Array of keys */
+            yajl_val *values; /*< Array of values. */
+            size_t len; /*< Number of key-value-pairs. */
+        } object;
+        struct {
+            yajl_val *values; /*< Array of elements. */
+            size_t len; /*< Number of elements. */
+        } array;
+    } u;
+};
+
+/**
+ * Parse a string.
+ *
+ * Parses an null-terminated string containing JSON data and returns a pointer
+ * to the top-level value (root of the parse tree).
+ *
+ * \param input              Pointer to a null-terminated utf8 string containing
+ *                           JSON data.
+ * \param error_buffer       Pointer to a buffer in which an error message will
+ *                           be stored if \em yajl_tree_parse fails, or
+ *                           \c NULL. The buffer will be initialized before
+ *                           parsing, so its content will be destroyed even if
+ *                           \em yajl_tree_parse succeeds.
+ * \param error_buffer_size  Size of the memory area pointed to by
+ *                           \em error_buffer_size. If \em error_buffer_size is
+ *                           \c NULL, this argument is ignored.
+ *
+ * \returns Pointer to the top-level value or \c NULL on error. The memory
+ * pointed to must be freed using \em yajl_tree_free. In case of an error, a
+ * null terminated message describing the error in more detail is stored in
+ * \em error_buffer if it is not \c NULL.
+ */
+YAJL_API yajl_val yajl_tree_parse (const char *input,
+                                   char *error_buffer, size_t error_buffer_size);
+
+/**
+ * Free a parse tree returned by "yajl_tree_parse".
+ *
+ * \param v Pointer to a JSON value returned by "yajl_tree_parse". Passing NULL
+ * is valid and results in a no-op.
+ */
+YAJL_API void yajl_tree_free (yajl_val v);
+
+/**
+ * Access a nested value inside a tree.
+ *
+ * \param parent the node under which you'd like to extract values.
+ * \param path A null terminated array of strings, each the name of an object key
+ * \param type the yajl_type of the object you seek, or yajl_t_any if any will do.
+ *
+ * \returns a pointer to the found value, or NULL if we came up empty.
+ * 
+ * Future Ideas:  it'd be nice to move path to a string and implement support for
+ * a teeny tiny micro language here, so you can extract array elements, do things
+ * like .first and .last, even .length.  Inspiration from JSONPath and css selectors?
+ * No it wouldn't be fast, but that's not what this API is about.
+ */
+YAJL_API yajl_val yajl_tree_get(yajl_val parent, const char ** path, yajl_type type);
+
+/* Various convenience macros to check the type of a `yajl_val` */
+#define YAJL_IS_STRING(v) (((v) != NULL) && ((v)->type == yajl_t_string))
+#define YAJL_IS_NUMBER(v) (((v) != NULL) && ((v)->type == yajl_t_number))
+#define YAJL_IS_INTEGER(v) (YAJL_IS_NUMBER(v) && ((v)->u.flags & YAJL_NUMBER_INT_VALID))
+#define YAJL_IS_DOUBLE(v) (YAJL_IS_NUMBER(v) && ((v)->u.flags & YAJL_NUMBER_DOUBLE_VALID))
+#define YAJL_IS_OBJECT(v) (((v) != NULL) && ((v)->type == yajl_t_object))
+#define YAJL_IS_ARRAY(v)  (((v) != NULL) && ((v)->type == yajl_t_array ))
+#define YAJL_IS_TRUE(v)   (((v) != NULL) && ((v)->type == yajl_t_true  ))
+#define YAJL_IS_FALSE(v)  (((v) != NULL) && ((v)->type == yajl_t_false ))
+#define YAJL_IS_NULL(v)   (((v) != NULL) && ((v)->type == yajl_t_null  ))
+
+/** Given a yajl_val_string return a ptr to the bare string it contains,
+ *  or NULL if the value is not a string. */
+#define YAJL_GET_STRING(v) (YAJL_IS_STRING(v) ? (v)->u.string : NULL)
+
+/** Get the string representation of a number.  You should check type first,
+ *  perhaps using YAJL_IS_NUMBER */
+#define YAJL_GET_NUMBER(v) ((v)->u.number.r)
+
+/** Get the double representation of a number.  You should check type first,
+ *  perhaps using YAJL_IS_DOUBLE */
+#define YAJL_GET_DOUBLE(v) ((v)->u.number.d)
+
+/** Get the 64bit (long long) integer representation of a number.  You should
+ *  check type first, perhaps using YAJL_IS_INTEGER */
+#define YAJL_GET_INTEGER(v) ((v)->u.number.i)
+
+/** Get a pointer to a yajl_val_object or NULL if the value is not an object. */
+#define YAJL_GET_OBJECT(v) (YAJL_IS_OBJECT(v) ? &(v)->u.object : NULL)
+
+/** Get a pointer to a yajl_val_array or NULL if the value is not an object. */
+#define YAJL_GET_ARRAY(v)  (YAJL_IS_ARRAY(v)  ? &(v)->u.array  : NULL)
+
+#endif /* YAJL_TREE_H */
diff --git a/xlators/cluster/nsr-server/src/yajl/yajl_version.h b/xlators/cluster/nsr-server/src/yajl/yajl_version.h
new file mode 100644
index 000000000..0fba9b8fc
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/yajl/yajl_version.h
@@ -0,0 +1,23 @@
+#ifndef YAJL_VERSION_H_
+#define YAJL_VERSION_H_
+
+#include <yajl/yajl_common.h>
+
+#define YAJL_MAJOR 2
+#define YAJL_MINOR 0
+#define YAJL_MICRO 1
+
+#define YAJL_VERSION ((YAJL_MAJOR * 10000) + (YAJL_MINOR * 100) + YAJL_MICRO)
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern int YAJL_API yajl_version(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* YAJL_VERSION_H_ */
+
diff --git a/xlators/cluster/nsr-server/src/yajl_alloc.c b/xlators/cluster/nsr-server/src/yajl_alloc.c
new file mode 100644
index 000000000..276315af7
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/yajl_alloc.c
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2007-2011, Lloyd Hilaiel <lloyd@hilaiel.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+/**
+ * \file yajl_alloc.h
+ * default memory allocation routines for yajl which use malloc/realloc and
+ * free
+ */
+
+#include "yajl_alloc.h"
+#include <stdlib.h>
+
+static void * yajl_internal_malloc(void *ctx, size_t sz)
+{
+    return malloc(sz);
+}
+
+static void * yajl_internal_realloc(void *ctx, void * previous,
+                                    size_t sz)
+{
+    return realloc(previous, sz);
+}
+
+static void yajl_internal_free(void *ctx, void * ptr)
+{
+    free(ptr);
+}
+
+void yajl_set_default_alloc_funcs(yajl_alloc_funcs * yaf)
+{
+    yaf->malloc = yajl_internal_malloc;
+    yaf->free = yajl_internal_free;
+    yaf->realloc = yajl_internal_realloc;
+    yaf->ctx = NULL;
+}
+
diff --git a/xlators/cluster/nsr-server/src/yajl_alloc.h b/xlators/cluster/nsr-server/src/yajl_alloc.h
new file mode 100644
index 000000000..a8a9e45e6
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/yajl_alloc.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2007-2011, Lloyd Hilaiel <lloyd@hilaiel.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+/**
+ * \file yajl_alloc.h
+ * default memory allocation routines for yajl which use malloc/realloc and
+ * free
+ */
+
+#ifndef __YAJL_ALLOC_H__
+#define __YAJL_ALLOC_H__
+
+#include "yajl/yajl_common.h"
+
+#define YA_MALLOC(afs, sz) (afs)->malloc((afs)->ctx, (sz))
+#define YA_FREE(afs, ptr) (afs)->free((afs)->ctx, (ptr))
+#define YA_REALLOC(afs, ptr, sz) (afs)->realloc((afs)->ctx, (ptr), (sz))
+
+void yajl_set_default_alloc_funcs(yajl_alloc_funcs * yaf);
+
+#endif
diff --git a/xlators/cluster/nsr-server/src/yajl_buf.c b/xlators/cluster/nsr-server/src/yajl_buf.c
new file mode 100644
index 000000000..0d249d364
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/yajl_buf.c
@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2007-2011, Lloyd Hilaiel <lloyd@hilaiel.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "yajl_buf.h"
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define YAJL_BUF_INIT_SIZE 2048
+
+struct yajl_buf_t {
+    size_t len;
+    size_t used;
+    unsigned char * data;
+    yajl_alloc_funcs * alloc;
+};
+
+static
+void yajl_buf_ensure_available(yajl_buf buf, size_t want)
+{
+    size_t need;
+    
+    assert(buf != NULL);
+
+    /* first call */
+    if (buf->data == NULL) {
+        buf->len = YAJL_BUF_INIT_SIZE;
+        buf->data = (unsigned char *) YA_MALLOC(buf->alloc, buf->len);
+        buf->data[0] = 0;
+    }
+
+    need = buf->len;
+
+    while (want >= (need - buf->used)) need <<= 1;
+
+    if (need != buf->len) {
+        buf->data = (unsigned char *) YA_REALLOC(buf->alloc, buf->data, need);
+        buf->len = need;
+    }
+}
+
+yajl_buf yajl_buf_alloc(yajl_alloc_funcs * alloc)
+{
+    yajl_buf b = YA_MALLOC(alloc, sizeof(struct yajl_buf_t));
+    memset((void *) b, 0, sizeof(struct yajl_buf_t));
+    b->alloc = alloc;
+    return b;
+}
+
+void yajl_buf_free(yajl_buf buf)
+{
+    assert(buf != NULL);
+    if (buf->data) YA_FREE(buf->alloc, buf->data);
+    YA_FREE(buf->alloc, buf);
+}
+
+void yajl_buf_append(yajl_buf buf, const void * data, size_t len)
+{
+    yajl_buf_ensure_available(buf, len);
+    if (len > 0) {
+        assert(data != NULL);
+        memcpy(buf->data + buf->used, data, len);
+        buf->used += len;
+        buf->data[buf->used] = 0;
+    }
+}
+
+void yajl_buf_clear(yajl_buf buf)
+{
+    buf->used = 0;
+    if (buf->data) buf->data[buf->used] = 0;
+}
+
+const unsigned char * yajl_buf_data(yajl_buf buf)
+{
+    return buf->data;
+}
+
+size_t yajl_buf_len(yajl_buf buf)
+{
+    return buf->used;
+}
+
+void
+yajl_buf_truncate(yajl_buf buf, size_t len)
+{
+    assert(len <= buf->used);
+    buf->used = len;
+}
diff --git a/xlators/cluster/nsr-server/src/yajl_buf.h b/xlators/cluster/nsr-server/src/yajl_buf.h
new file mode 100644
index 000000000..94929a519
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/yajl_buf.h
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2007-2011, Lloyd Hilaiel <lloyd@hilaiel.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef __YAJL_BUF_H__
+#define __YAJL_BUF_H__
+
+#include "yajl/yajl_common.h"
+#include "yajl_alloc.h"
+
+/*
+ * Implementation/performance notes.  If this were moved to a header
+ * only implementation using #define's where possible we might be 
+ * able to sqeeze a little performance out of the guy by killing function
+ * call overhead.  YMMV.
+ */
+
+/**
+ * yajl_buf is a buffer with exponential growth.  the buffer ensures that
+ * you are always null padded.
+ */
+typedef struct yajl_buf_t * yajl_buf;
+
+/* allocate a new buffer */
+yajl_buf yajl_buf_alloc(yajl_alloc_funcs * alloc);
+
+/* free the buffer */
+void yajl_buf_free(yajl_buf buf);
+
+/* append a number of bytes to the buffer */
+void yajl_buf_append(yajl_buf buf, const void * data, size_t len);
+
+/* empty the buffer */
+void yajl_buf_clear(yajl_buf buf);
+
+/* get a pointer to the beginning of the buffer */
+const unsigned char * yajl_buf_data(yajl_buf buf);
+
+/* get the length of the buffer */
+size_t yajl_buf_len(yajl_buf buf);
+
+/* truncate the buffer */
+void yajl_buf_truncate(yajl_buf buf, size_t len);
+
+#endif
diff --git a/xlators/cluster/nsr-server/src/yajl_bytestack.h b/xlators/cluster/nsr-server/src/yajl_bytestack.h
new file mode 100644
index 000000000..1fc50c470
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/yajl_bytestack.h
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2007-2011, Lloyd Hilaiel <lloyd@hilaiel.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+/*
+ * A header only implementation of a simple stack of bytes, used in YAJL
+ * to maintain parse state.
+ */
+
+#ifndef __YAJL_BYTESTACK_H__
+#define __YAJL_BYTESTACK_H__
+
+#include "yajl/yajl_common.h"
+
+#define YAJL_BS_INC 128
+
+typedef struct yajl_bytestack_t
+{
+    unsigned char * stack;
+    size_t size;
+    size_t used;
+    yajl_alloc_funcs * yaf;
+} yajl_bytestack;
+
+/* initialize a bytestack */
+#define yajl_bs_init(obs, _yaf) {               \
+        (obs).stack = NULL;                     \
+        (obs).size = 0;                         \
+        (obs).used = 0;                         \
+        (obs).yaf = (_yaf);                     \
+    }                                           \
+
+
+/* initialize a bytestack */
+#define yajl_bs_free(obs)                 \
+    if ((obs).stack) (obs).yaf->free((obs).yaf->ctx, (obs).stack);
+
+#define yajl_bs_current(obs)               \
+    (assert((obs).used > 0), (obs).stack[(obs).used - 1])
+
+#define yajl_bs_push(obs, byte) {                       \
+    if (((obs).size - (obs).used) == 0) {               \
+        (obs).size += YAJL_BS_INC;                      \
+        (obs).stack = (obs).yaf->realloc((obs).yaf->ctx,\
+                                         (void *) (obs).stack, (obs).size);\
+    }                                                   \
+    (obs).stack[((obs).used)++] = (byte);               \
+}
+
+/* removes the top item of the stack, returns nothing */
+#define yajl_bs_pop(obs) { ((obs).used)--; }
+
+#define yajl_bs_set(obs, byte)                          \
+    (obs).stack[((obs).used) - 1] = (byte);
+
+
+#endif
diff --git a/xlators/cluster/nsr-server/src/yajl_encode.c b/xlators/cluster/nsr-server/src/yajl_encode.c
new file mode 100644
index 000000000..9dc9a3e81
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/yajl_encode.c
@@ -0,0 +1,220 @@
+/*
+ * Copyright (c) 2007-2011, Lloyd Hilaiel <lloyd@hilaiel.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "yajl_encode.h"
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+static void CharToHex(unsigned char c, char * hexBuf)
+{
+    const char * hexchar = "0123456789ABCDEF";
+    hexBuf[0] = hexchar[c >> 4];
+    hexBuf[1] = hexchar[c & 0x0F];
+}
+
+void
+yajl_string_encode(const yajl_print_t print,
+                   void * ctx,
+                   const unsigned char * str,
+                   size_t len,
+                   int escape_solidus)
+{
+    size_t beg = 0;
+    size_t end = 0;
+    char hexBuf[7];
+    hexBuf[0] = '\\'; hexBuf[1] = 'u'; hexBuf[2] = '0'; hexBuf[3] = '0';
+    hexBuf[6] = 0;
+
+    while (end < len) {
+        const char * escaped = NULL;
+        switch (str[end]) {
+            case '\r': escaped = "\\r"; break;
+            case '\n': escaped = "\\n"; break;
+            case '\\': escaped = "\\\\"; break;
+            /* it is not required to escape a solidus in JSON:
+             * read sec. 2.5: http://www.ietf.org/rfc/rfc4627.txt
+             * specifically, this production from the grammar:
+             *   unescaped = %x20-21 / %x23-5B / %x5D-10FFFF
+             */
+            case '/': if (escape_solidus) escaped = "\\/"; break;
+            case '"': escaped = "\\\""; break;
+            case '\f': escaped = "\\f"; break;
+            case '\b': escaped = "\\b"; break;
+            case '\t': escaped = "\\t"; break;
+            default:
+                if ((unsigned char) str[end] < 32) {
+                    CharToHex(str[end], hexBuf + 4);
+                    escaped = hexBuf;
+                }
+                break;
+        }
+        if (escaped != NULL) {
+            print(ctx, (const char *) (str + beg), end - beg);
+            print(ctx, escaped, (unsigned int)strlen(escaped));
+            beg = ++end;
+        } else {
+            ++end;
+        }
+    }
+    print(ctx, (const char *) (str + beg), end - beg);
+}
+
+static void hexToDigit(unsigned int * val, const unsigned char * hex)
+{
+    unsigned int i;
+    for (i=0;i<4;i++) {
+        unsigned char c = hex[i];
+        if (c >= 'A') c = (c & ~0x20) - 7;
+        c -= '0';
+        assert(!(c & 0xF0));
+        *val = (*val << 4) | c;
+    }
+}
+
+static void Utf32toUtf8(unsigned int codepoint, char * utf8Buf) 
+{
+    if (codepoint < 0x80) {
+        utf8Buf[0] = (char) codepoint;
+        utf8Buf[1] = 0;
+    } else if (codepoint < 0x0800) {
+        utf8Buf[0] = (char) ((codepoint >> 6) | 0xC0);
+        utf8Buf[1] = (char) ((codepoint & 0x3F) | 0x80);
+        utf8Buf[2] = 0;
+    } else if (codepoint < 0x10000) {
+        utf8Buf[0] = (char) ((codepoint >> 12) | 0xE0);
+        utf8Buf[1] = (char) (((codepoint >> 6) & 0x3F) | 0x80);
+        utf8Buf[2] = (char) ((codepoint & 0x3F) | 0x80);
+        utf8Buf[3] = 0;
+    } else if (codepoint < 0x200000) {
+        utf8Buf[0] =(char)((codepoint >> 18) | 0xF0);
+        utf8Buf[1] =(char)(((codepoint >> 12) & 0x3F) | 0x80);
+        utf8Buf[2] =(char)(((codepoint >> 6) & 0x3F) | 0x80);
+        utf8Buf[3] =(char)((codepoint & 0x3F) | 0x80);
+        utf8Buf[4] = 0;
+    } else {
+        utf8Buf[0] = '?';
+        utf8Buf[1] = 0;
+    }
+}
+
+void yajl_string_decode(yajl_buf buf, const unsigned char * str,
+                        size_t len)
+{
+    size_t beg = 0;
+    size_t end = 0;    
+
+    while (end < len) {
+        if (str[end] == '\\') {
+            char utf8Buf[5];
+            const char * unescaped = "?";
+            yajl_buf_append(buf, str + beg, end - beg);
+            switch (str[++end]) {
+                case 'r': unescaped = "\r"; break;
+                case 'n': unescaped = "\n"; break;
+                case '\\': unescaped = "\\"; break;
+                case '/': unescaped = "/"; break;
+                case '"': unescaped = "\""; break;
+                case 'f': unescaped = "\f"; break;
+                case 'b': unescaped = "\b"; break;
+                case 't': unescaped = "\t"; break;
+                case 'u': {
+                    unsigned int codepoint = 0;
+                    hexToDigit(&codepoint, str + ++end);
+                    end+=3;
+                    /* check if this is a surrogate */
+                    if ((codepoint & 0xFC00) == 0xD800) {
+                        end++;
+                        if (str[end] == '\\' && str[end + 1] == 'u') {
+                            unsigned int surrogate = 0;
+                            hexToDigit(&surrogate, str + end + 2);
+                            codepoint =
+                                (((codepoint & 0x3F) << 10) | 
+                                 ((((codepoint >> 6) & 0xF) + 1) << 16) | 
+                                 (surrogate & 0x3FF));
+                            end += 5;
+                        } else {
+                            unescaped = "?";
+                            break;
+                        }
+                    }
+                    
+                    Utf32toUtf8(codepoint, utf8Buf);
+                    unescaped = utf8Buf;
+
+                    if (codepoint == 0) {
+                        yajl_buf_append(buf, unescaped, 1);
+                        beg = ++end;
+                        continue;
+                    }
+
+                    break;
+                }
+                default:
+                    assert("this should never happen" == NULL);
+            }
+            yajl_buf_append(buf, unescaped, (unsigned int)strlen(unescaped));
+            beg = ++end;
+        } else {
+            end++;
+        }
+    }
+    yajl_buf_append(buf, str + beg, end - beg);
+}
+
+#define ADV_PTR s++; if (!(len--)) return 0;
+
+int yajl_string_validate_utf8(const unsigned char * s, size_t len)
+{
+    if (!len) return 1;
+    if (!s) return 0;
+    
+    while (len--) {
+        /* single byte */
+        if (*s <= 0x7f) {
+            /* noop */
+        }
+        /* two byte */ 
+        else if ((*s >> 5) == 0x6) {
+            ADV_PTR;
+            if (!((*s >> 6) == 0x2)) return 0;
+        }
+        /* three byte */
+        else if ((*s >> 4) == 0x0e) {
+            ADV_PTR;
+            if (!((*s >> 6) == 0x2)) return 0;
+            ADV_PTR;
+            if (!((*s >> 6) == 0x2)) return 0;
+        }
+        /* four byte */        
+        else if ((*s >> 3) == 0x1e) {
+            ADV_PTR;
+            if (!((*s >> 6) == 0x2)) return 0;
+            ADV_PTR;
+            if (!((*s >> 6) == 0x2)) return 0;
+            ADV_PTR;
+            if (!((*s >> 6) == 0x2)) return 0;
+        } else {
+            return 0;
+        }
+        
+        s++;
+    }
+    
+    return 1;
+}
diff --git a/xlators/cluster/nsr-server/src/yajl_encode.h b/xlators/cluster/nsr-server/src/yajl_encode.h
new file mode 100644
index 000000000..af1e8bbde
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/yajl_encode.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2007-2011, Lloyd Hilaiel <lloyd@hilaiel.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef __YAJL_ENCODE_H__
+#define __YAJL_ENCODE_H__
+
+#include "yajl_buf.h"
+#include "yajl/yajl_gen.h"
+
+void yajl_string_encode(const yajl_print_t printer,
+                        void * ctx,
+                        const unsigned char * str,
+                        size_t length,
+                        int escape_solidus);
+
+void yajl_string_decode(yajl_buf buf, const unsigned char * str,
+                        size_t length);
+
+int yajl_string_validate_utf8(const unsigned char * s, size_t len);
+
+#endif
diff --git a/xlators/cluster/nsr-server/src/yajl_gen.c b/xlators/cluster/nsr-server/src/yajl_gen.c
new file mode 100644
index 000000000..73763a9e0
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/yajl_gen.c
@@ -0,0 +1,350 @@
+/*
+ * Copyright (c) 2007-2011, Lloyd Hilaiel <lloyd@hilaiel.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "yajl/yajl_gen.h"
+#include "yajl_buf.h"
+#include "yajl_encode.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <math.h>
+#include <stdarg.h>
+
+typedef enum {
+    yajl_gen_start,
+    yajl_gen_map_start,
+    yajl_gen_map_key,
+    yajl_gen_map_val,
+    yajl_gen_array_start,
+    yajl_gen_in_array,
+    yajl_gen_complete,
+    yajl_gen_error
+} yajl_gen_state;
+
+struct yajl_gen_t
+{
+    unsigned int flags;
+    unsigned int depth;
+    const char * indentString;
+    yajl_gen_state state[YAJL_MAX_DEPTH];
+    yajl_print_t print;
+    void * ctx; /* yajl_buf */
+    /* memory allocation routines */
+    yajl_alloc_funcs alloc;
+};
+
+int
+yajl_gen_config(yajl_gen g, yajl_gen_option opt, ...)
+{
+    int rv = 1;
+    va_list ap;
+    va_start(ap, opt);
+
+    switch(opt) {
+        case yajl_gen_beautify:
+        case yajl_gen_validate_utf8:
+            if (va_arg(ap, int)) g->flags |= opt;
+            else g->flags &= ~opt;
+            break;
+        case yajl_gen_indent_string: {
+            const char *indent = va_arg(ap, const char *);
+            g->indentString = indent;
+            for (; *indent; indent++) {
+                if (*indent != '\n'
+                    && *indent != '\v'
+                    && *indent != '\f'
+                    && *indent != '\t'
+                    && *indent != '\r'
+                    && *indent != ' ')
+                {
+                    g->indentString = NULL;
+                    rv = 0;
+                }
+            }
+            break;
+        }
+        case yajl_gen_print_callback:
+            yajl_buf_free(g->ctx);
+            g->print = va_arg(ap, const yajl_print_t);
+            g->ctx = va_arg(ap, void *);
+            break;
+        default:
+            rv = 0;
+    }
+
+    va_end(ap);
+
+    return rv;
+}
+
+
+
+yajl_gen
+yajl_gen_alloc(const yajl_alloc_funcs * afs)
+{
+    yajl_gen g = NULL;
+    yajl_alloc_funcs afsBuffer;
+
+    /* first order of business is to set up memory allocation routines */
+    if (afs != NULL) {
+        if (afs->malloc == NULL || afs->realloc == NULL || afs->free == NULL)
+        {
+            return NULL;
+        }
+    } else {
+        yajl_set_default_alloc_funcs(&afsBuffer);
+        afs = &afsBuffer;
+    }
+
+    g = (yajl_gen) YA_MALLOC(afs, sizeof(struct yajl_gen_t));
+    if (!g) return NULL;
+
+    memset((void *) g, 0, sizeof(struct yajl_gen_t));
+    /* copy in pointers to allocation routines */
+    memcpy((void *) &(g->alloc), (void *) afs, sizeof(yajl_alloc_funcs));
+
+    g->print = (yajl_print_t)&yajl_buf_append;
+    g->ctx = yajl_buf_alloc(&(g->alloc));
+    g->indentString = "    ";
+
+    return g;
+}
+
+void
+yajl_gen_free(yajl_gen g)
+{
+    if (g->print == (yajl_print_t)&yajl_buf_append) yajl_buf_free((yajl_buf)g->ctx);
+    YA_FREE(&(g->alloc), g);
+}
+
+#define INSERT_SEP \
+    if (g->state[g->depth] == yajl_gen_map_key ||               \
+        g->state[g->depth] == yajl_gen_in_array) {              \
+        g->print(g->ctx, ",", 1);                               \
+        if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, "\n", 1);               \
+    } else if (g->state[g->depth] == yajl_gen_map_val) {        \
+        g->print(g->ctx, ":", 1);                               \
+        if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, " ", 1);                \
+   } 
+
+#define INSERT_WHITESPACE                                               \
+    if ((g->flags & yajl_gen_beautify)) {                                                    \
+        if (g->state[g->depth] != yajl_gen_map_val) {                   \
+            unsigned int _i;                                            \
+            for (_i=0;_i<g->depth;_i++)                                 \
+                g->print(g->ctx,                                        \
+                         g->indentString,                               \
+                         (unsigned int)strlen(g->indentString));        \
+        }                                                               \
+    }
+
+#define ENSURE_NOT_KEY \
+    if (g->state[g->depth] == yajl_gen_map_key ||       \
+        g->state[g->depth] == yajl_gen_map_start)  {    \
+        return yajl_gen_keys_must_be_strings;           \
+    }                                                   \
+
+/* check that we're not complete, or in error state.  in a valid state
+ * to be generating */
+#define ENSURE_VALID_STATE \
+    if (g->state[g->depth] == yajl_gen_error) {   \
+        return yajl_gen_in_error_state;\
+    } else if (g->state[g->depth] == yajl_gen_complete) {   \
+        return yajl_gen_generation_complete;                \
+    }
+
+#define INCREMENT_DEPTH \
+    if (++(g->depth) >= YAJL_MAX_DEPTH) return yajl_max_depth_exceeded;
+
+#define DECREMENT_DEPTH \
+  if (--(g->depth) >= YAJL_MAX_DEPTH) return yajl_gen_error;
+
+#define APPENDED_ATOM \
+    switch (g->state[g->depth]) {                   \
+        case yajl_gen_start:                        \
+            g->state[g->depth] = yajl_gen_complete; \
+            break;                                  \
+        case yajl_gen_map_start:                    \
+        case yajl_gen_map_key:                      \
+            g->state[g->depth] = yajl_gen_map_val;  \
+            break;                                  \
+        case yajl_gen_array_start:                  \
+            g->state[g->depth] = yajl_gen_in_array; \
+            break;                                  \
+        case yajl_gen_map_val:                      \
+            g->state[g->depth] = yajl_gen_map_key;  \
+            break;                                  \
+        default:                                    \
+            break;                                  \
+    }                                               \
+
+#define FINAL_NEWLINE                                        \
+    if ((g->flags & yajl_gen_beautify) && g->state[g->depth] == yajl_gen_complete) \
+        g->print(g->ctx, "\n", 1);
+
+yajl_gen_status
+yajl_gen_integer(yajl_gen g, long long int number)
+{
+    char i[32];
+    ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE;
+    sprintf(i, "%lld", number);
+    g->print(g->ctx, i, (unsigned int)strlen(i));
+    APPENDED_ATOM;
+    FINAL_NEWLINE;
+    return yajl_gen_status_ok;
+}
+
+#ifdef WIN32
+#include <float.h>
+#define isnan _isnan
+#define isinf !_finite
+#endif
+
+yajl_gen_status
+yajl_gen_double(yajl_gen g, double number)
+{
+    char i[32];
+    ENSURE_VALID_STATE; ENSURE_NOT_KEY; 
+    if (isnan(number) || isinf(number)) return yajl_gen_invalid_number;
+    INSERT_SEP; INSERT_WHITESPACE;
+    sprintf(i, "%.20g", number);
+    g->print(g->ctx, i, (unsigned int)strlen(i));
+    APPENDED_ATOM;
+    FINAL_NEWLINE;
+    return yajl_gen_status_ok;
+}
+
+yajl_gen_status
+yajl_gen_number(yajl_gen g, const char * s, size_t l)
+{
+    ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE;
+    g->print(g->ctx, s, l);
+    APPENDED_ATOM;
+    FINAL_NEWLINE;
+    return yajl_gen_status_ok;
+}
+
+yajl_gen_status
+yajl_gen_string(yajl_gen g, const unsigned char * str,
+                size_t len)
+{
+    // if validation is enabled, check that the string is valid utf8
+    // XXX: This checking could be done a little faster, in the same pass as
+    // the string encoding
+    if (g->flags & yajl_gen_validate_utf8) {
+        if (!yajl_string_validate_utf8(str, len)) {
+            return yajl_gen_invalid_string;
+        }
+    }
+    ENSURE_VALID_STATE; INSERT_SEP; INSERT_WHITESPACE;
+    g->print(g->ctx, "\"", 1);
+    yajl_string_encode(g->print, g->ctx, str, len, g->flags & yajl_gen_escape_solidus);
+    g->print(g->ctx, "\"", 1);
+    APPENDED_ATOM;
+    FINAL_NEWLINE;
+    return yajl_gen_status_ok;
+}
+
+yajl_gen_status
+yajl_gen_null(yajl_gen g)
+{
+    ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE;
+    g->print(g->ctx, "null", strlen("null"));
+    APPENDED_ATOM;
+    FINAL_NEWLINE;
+    return yajl_gen_status_ok;
+}
+
+yajl_gen_status
+yajl_gen_bool(yajl_gen g, int boolean)
+{
+    const char * val = boolean ? "true" : "false";
+
+	ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE;
+    g->print(g->ctx, val, (unsigned int)strlen(val));
+    APPENDED_ATOM;
+    FINAL_NEWLINE;
+    return yajl_gen_status_ok;
+}
+
+yajl_gen_status
+yajl_gen_map_open(yajl_gen g)
+{
+    ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE;
+    INCREMENT_DEPTH; 
+    
+    g->state[g->depth] = yajl_gen_map_start;
+    g->print(g->ctx, "{", 1);
+    if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, "\n", 1);
+    FINAL_NEWLINE;
+    return yajl_gen_status_ok;
+}
+
+yajl_gen_status
+yajl_gen_map_close(yajl_gen g)
+{
+    ENSURE_VALID_STATE; 
+    DECREMENT_DEPTH;
+    
+    if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, "\n", 1);
+    APPENDED_ATOM;
+    INSERT_WHITESPACE;
+    g->print(g->ctx, "}", 1);
+    FINAL_NEWLINE;
+    return yajl_gen_status_ok;
+}
+
+yajl_gen_status
+yajl_gen_array_open(yajl_gen g)
+{
+    ENSURE_VALID_STATE; ENSURE_NOT_KEY; INSERT_SEP; INSERT_WHITESPACE;
+    INCREMENT_DEPTH; 
+    g->state[g->depth] = yajl_gen_array_start;
+    g->print(g->ctx, "[", 1);
+    if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, "\n", 1);
+    FINAL_NEWLINE;
+    return yajl_gen_status_ok;
+}
+
+yajl_gen_status
+yajl_gen_array_close(yajl_gen g)
+{
+    ENSURE_VALID_STATE;
+    DECREMENT_DEPTH;
+    if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, "\n", 1);
+    APPENDED_ATOM;
+    INSERT_WHITESPACE;
+    g->print(g->ctx, "]", 1);
+    FINAL_NEWLINE;
+    return yajl_gen_status_ok;
+}
+
+yajl_gen_status
+yajl_gen_get_buf(yajl_gen g, const unsigned char ** buf,
+                 size_t * len)
+{
+    if (g->print != (yajl_print_t)&yajl_buf_append) return yajl_gen_no_buf;
+    *buf = yajl_buf_data((yajl_buf)g->ctx);
+    *len = yajl_buf_len((yajl_buf)g->ctx);
+    return yajl_gen_status_ok;
+}
+
+void
+yajl_gen_clear(yajl_gen g)
+{
+    if (g->print == (yajl_print_t)&yajl_buf_append) yajl_buf_clear((yajl_buf)g->ctx);
+}
diff --git a/xlators/cluster/nsr-server/src/yajl_lex.c b/xlators/cluster/nsr-server/src/yajl_lex.c
new file mode 100644
index 000000000..b098e6a99
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/yajl_lex.c
@@ -0,0 +1,763 @@
+/*
+ * Copyright (c) 2007-2011, Lloyd Hilaiel <lloyd@hilaiel.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "yajl_lex.h"
+#include "yajl_buf.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <string.h>
+
+#ifdef YAJL_LEXER_DEBUG
+static const char *
+tokToStr(yajl_tok tok) 
+{
+    switch (tok) {
+        case yajl_tok_bool: return "bool";
+        case yajl_tok_colon: return "colon";
+        case yajl_tok_comma: return "comma";
+        case yajl_tok_eof: return "eof";
+        case yajl_tok_error: return "error";
+        case yajl_tok_left_brace: return "brace";
+        case yajl_tok_left_bracket: return "bracket";
+        case yajl_tok_null: return "null";
+        case yajl_tok_integer: return "integer";
+        case yajl_tok_double: return "double";
+        case yajl_tok_right_brace: return "brace";
+        case yajl_tok_right_bracket: return "bracket";
+        case yajl_tok_string: return "string";
+        case yajl_tok_string_with_escapes: return "string_with_escapes";
+    }
+    return "unknown";
+}
+#endif
+
+/* Impact of the stream parsing feature on the lexer:
+ *
+ * YAJL support stream parsing.  That is, the ability to parse the first
+ * bits of a chunk of JSON before the last bits are available (still on
+ * the network or disk).  This makes the lexer more complex.  The
+ * responsibility of the lexer is to handle transparently the case where
+ * a chunk boundary falls in the middle of a token.  This is
+ * accomplished is via a buffer and a character reading abstraction. 
+ *
+ * Overview of implementation
+ *
+ * When we lex to end of input string before end of token is hit, we
+ * copy all of the input text composing the token into our lexBuf.
+ * 
+ * Every time we read a character, we do so through the readChar function.
+ * readChar's responsibility is to handle pulling all chars from the buffer
+ * before pulling chars from input text
+ */
+
+struct yajl_lexer_t {
+    /* the overal line and char offset into the data */
+    size_t lineOff;
+    size_t charOff;
+
+    /* error */
+    yajl_lex_error error;
+
+    /* a input buffer to handle the case where a token is spread over
+     * multiple chunks */ 
+    yajl_buf buf;
+
+    /* in the case where we have data in the lexBuf, bufOff holds
+     * the current offset into the lexBuf. */
+    size_t bufOff;
+
+    /* are we using the lex buf? */
+    unsigned int bufInUse;
+
+    /* shall we allow comments? */
+    unsigned int allowComments;
+
+    /* shall we validate utf8 inside strings? */
+    unsigned int validateUTF8;
+
+    yajl_alloc_funcs * alloc;
+};
+
+#define readChar(lxr, txt, off)                      \
+    (((lxr)->bufInUse && yajl_buf_len((lxr)->buf) && lxr->bufOff < yajl_buf_len((lxr)->buf)) ? \
+     (*((const unsigned char *) yajl_buf_data((lxr)->buf) + ((lxr)->bufOff)++)) : \
+     ((txt)[(*(off))++]))
+
+#define unreadChar(lxr, off) ((*(off) > 0) ? (*(off))-- : ((lxr)->bufOff--))
+
+yajl_lexer
+yajl_lex_alloc(yajl_alloc_funcs * alloc,
+               unsigned int allowComments, unsigned int validateUTF8)
+{
+    yajl_lexer lxr = (yajl_lexer) YA_MALLOC(alloc, sizeof(struct yajl_lexer_t));
+    memset((void *) lxr, 0, sizeof(struct yajl_lexer_t));
+    lxr->buf = yajl_buf_alloc(alloc);
+    lxr->allowComments = allowComments;
+    lxr->validateUTF8 = validateUTF8;
+    lxr->alloc = alloc;
+    return lxr;
+}
+
+void
+yajl_lex_free(yajl_lexer lxr)
+{
+    yajl_buf_free(lxr->buf);
+    YA_FREE(lxr->alloc, lxr);
+    return;
+}
+
+/* a lookup table which lets us quickly determine three things:
+ * VEC - valid escaped control char
+ * note.  the solidus '/' may be escaped or not.
+ * IJC - invalid json char
+ * VHC - valid hex char
+ * NFP - needs further processing (from a string scanning perspective)
+ * NUC - needs utf8 checking when enabled (from a string scanning perspective)
+ */
+#define VEC 0x01
+#define IJC 0x02
+#define VHC 0x04
+#define NFP 0x08
+#define NUC 0x10
+
+static const char charLookupTable[256] =
+{
+/*00*/ IJC    , IJC    , IJC    , IJC    , IJC    , IJC    , IJC    , IJC    ,
+/*08*/ IJC    , IJC    , IJC    , IJC    , IJC    , IJC    , IJC    , IJC    ,
+/*10*/ IJC    , IJC    , IJC    , IJC    , IJC    , IJC    , IJC    , IJC    ,
+/*18*/ IJC    , IJC    , IJC    , IJC    , IJC    , IJC    , IJC    , IJC    ,
+
+/*20*/ 0      , 0      , NFP|VEC|IJC, 0      , 0      , 0      , 0      , 0      ,
+/*28*/ 0      , 0      , 0      , 0      , 0      , 0      , 0      , VEC    ,
+/*30*/ VHC    , VHC    , VHC    , VHC    , VHC    , VHC    , VHC    , VHC    ,
+/*38*/ VHC    , VHC    , 0      , 0      , 0      , 0      , 0      , 0      ,
+
+/*40*/ 0      , VHC    , VHC    , VHC    , VHC    , VHC    , VHC    , 0      ,
+/*48*/ 0      , 0      , 0      , 0      , 0      , 0      , 0      , 0      ,
+/*50*/ 0      , 0      , 0      , 0      , 0      , 0      , 0      , 0      ,
+/*58*/ 0      , 0      , 0      , 0      , NFP|VEC|IJC, 0      , 0      , 0      ,
+
+/*60*/ 0      , VHC    , VEC|VHC, VHC    , VHC    , VHC    , VEC|VHC, 0      ,
+/*68*/ 0      , 0      , 0      , 0      , 0      , 0      , VEC    , 0      ,
+/*70*/ 0      , 0      , VEC    , 0      , VEC    , 0      , 0      , 0      ,
+/*78*/ 0      , 0      , 0      , 0      , 0      , 0      , 0      , 0      ,
+
+       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
+       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
+       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
+       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
+
+       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
+       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
+       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
+       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
+
+       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
+       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
+       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
+       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
+
+       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
+       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
+       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    ,
+       NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC    , NUC
+};
+
+/** process a variable length utf8 encoded codepoint.
+ *
+ *  returns:
+ *    yajl_tok_string - if valid utf8 char was parsed and offset was
+ *                      advanced
+ *    yajl_tok_eof - if end of input was hit before validation could
+ *                   complete
+ *    yajl_tok_error - if invalid utf8 was encountered
+ * 
+ *  NOTE: on error the offset will point to the first char of the
+ *  invalid utf8 */
+#define UTF8_CHECK_EOF if (*offset >= jsonTextLen) { return yajl_tok_eof; }
+
+static yajl_tok
+yajl_lex_utf8_char(yajl_lexer lexer, const unsigned char * jsonText,
+                   size_t jsonTextLen, size_t * offset,
+                   unsigned char curChar)
+{
+    if (curChar <= 0x7f) {
+        /* single byte */
+        return yajl_tok_string;
+    } else if ((curChar >> 5) == 0x6) {
+        /* two byte */ 
+        UTF8_CHECK_EOF;
+        curChar = readChar(lexer, jsonText, offset);
+        if ((curChar >> 6) == 0x2) return yajl_tok_string;
+    } else if ((curChar >> 4) == 0x0e) {
+        /* three byte */
+        UTF8_CHECK_EOF;
+        curChar = readChar(lexer, jsonText, offset);
+        if ((curChar >> 6) == 0x2) {
+            UTF8_CHECK_EOF;
+            curChar = readChar(lexer, jsonText, offset);
+            if ((curChar >> 6) == 0x2) return yajl_tok_string;
+        }
+    } else if ((curChar >> 3) == 0x1e) {
+        /* four byte */
+        UTF8_CHECK_EOF;
+        curChar = readChar(lexer, jsonText, offset);
+        if ((curChar >> 6) == 0x2) {
+            UTF8_CHECK_EOF;
+            curChar = readChar(lexer, jsonText, offset);
+            if ((curChar >> 6) == 0x2) {
+                UTF8_CHECK_EOF;
+                curChar = readChar(lexer, jsonText, offset);
+                if ((curChar >> 6) == 0x2) return yajl_tok_string;
+            }
+        }
+    } 
+
+    return yajl_tok_error;
+}
+
+/* lex a string.  input is the lexer, pointer to beginning of
+ * json text, and start of string (offset).
+ * a token is returned which has the following meanings:
+ * yajl_tok_string: lex of string was successful.  offset points to
+ *                  terminating '"'.
+ * yajl_tok_eof: end of text was encountered before we could complete
+ *               the lex.
+ * yajl_tok_error: embedded in the string were unallowable chars.  offset
+ *               points to the offending char
+ */
+#define STR_CHECK_EOF \
+if (*offset >= jsonTextLen) { \
+   tok = yajl_tok_eof; \
+   goto finish_string_lex; \
+}
+
+/** scan a string for interesting characters that might need further
+ *  review.  return the number of chars that are uninteresting and can
+ *  be skipped.
+ * (lth) hi world, any thoughts on how to make this routine faster? */
+static size_t
+yajl_string_scan(const unsigned char * buf, size_t len, int utf8check)
+{
+    unsigned char mask = IJC|NFP|(utf8check ? NUC : 0);
+    size_t skip = 0;
+    while (skip < len && !(charLookupTable[*buf] & mask))
+    {
+        skip++;
+        buf++;
+    }
+    return skip;
+}
+
+static yajl_tok
+yajl_lex_string(yajl_lexer lexer, const unsigned char * jsonText,
+                size_t jsonTextLen, size_t * offset)
+{
+    yajl_tok tok = yajl_tok_error;
+    int hasEscapes = 0;
+
+    for (;;) {
+        unsigned char curChar;
+
+        /* now jump into a faster scanning routine to skip as much
+         * of the buffers as possible */
+        {
+            const unsigned char * p;
+            size_t len;
+            
+            if ((lexer->bufInUse && yajl_buf_len(lexer->buf) &&
+                 lexer->bufOff < yajl_buf_len(lexer->buf)))
+            {
+                p = ((const unsigned char *) yajl_buf_data(lexer->buf) +
+                     (lexer->bufOff));
+                len = yajl_buf_len(lexer->buf) - lexer->bufOff;
+                lexer->bufOff += yajl_string_scan(p, len, lexer->validateUTF8);
+            }                
+            else if (*offset < jsonTextLen) 
+            {
+                p = jsonText + *offset;
+                len = jsonTextLen - *offset;
+                *offset += yajl_string_scan(p, len, lexer->validateUTF8);
+            }
+        }
+
+        STR_CHECK_EOF;
+
+        curChar = readChar(lexer, jsonText, offset);
+
+        /* quote terminates */
+        if (curChar == '"') {
+            tok = yajl_tok_string;
+            break;
+        }
+        /* backslash escapes a set of control chars, */
+        else if (curChar == '\\') {
+            hasEscapes = 1;
+            STR_CHECK_EOF;
+
+            /* special case \u */
+            curChar = readChar(lexer, jsonText, offset);
+            if (curChar == 'u') {
+                unsigned int i = 0;
+
+                for (i=0;i<4;i++) {
+                    STR_CHECK_EOF;                
+                    curChar = readChar(lexer, jsonText, offset);                
+                    if (!(charLookupTable[curChar] & VHC)) {
+                        /* back up to offending char */
+                        unreadChar(lexer, offset);
+                        lexer->error = yajl_lex_string_invalid_hex_char;
+                        goto finish_string_lex;
+                    }
+                }
+            } else if (!(charLookupTable[curChar] & VEC)) {
+                /* back up to offending char */
+                unreadChar(lexer, offset);
+                lexer->error = yajl_lex_string_invalid_escaped_char;
+                goto finish_string_lex;                
+            } 
+        }
+        /* when not validating UTF8 it's a simple table lookup to determine
+         * if the present character is invalid */
+        else if(charLookupTable[curChar] & IJC) {
+            /* back up to offending char */
+            unreadChar(lexer, offset);
+            lexer->error = yajl_lex_string_invalid_json_char;
+            goto finish_string_lex;                
+        }
+        /* when in validate UTF8 mode we need to do some extra work */
+        else if (lexer->validateUTF8) {
+            yajl_tok t = yajl_lex_utf8_char(lexer, jsonText, jsonTextLen,
+                                            offset, curChar);
+            
+            if (t == yajl_tok_eof) {
+                tok = yajl_tok_eof;
+                goto finish_string_lex;
+            } else if (t == yajl_tok_error) {
+                lexer->error = yajl_lex_string_invalid_utf8;
+                goto finish_string_lex;
+            } 
+        }
+        /* accept it, and move on */ 
+    }
+  finish_string_lex:
+    /* tell our buddy, the parser, wether he needs to process this string
+     * again */
+    if (hasEscapes && tok == yajl_tok_string) {
+        tok = yajl_tok_string_with_escapes;
+    } 
+
+    return tok;
+}
+
+#define RETURN_IF_EOF if (*offset >= jsonTextLen) return yajl_tok_eof;
+
+static yajl_tok
+yajl_lex_number(yajl_lexer lexer, const unsigned char * jsonText,
+                size_t jsonTextLen, size_t * offset)
+{
+    /** XXX: numbers are the only entities in json that we must lex
+     *       _beyond_ in order to know that they are complete.  There
+     *       is an ambiguous case for integers at EOF. */
+
+    unsigned char c;
+
+    yajl_tok tok = yajl_tok_integer;
+
+    RETURN_IF_EOF;    
+    c = readChar(lexer, jsonText, offset);
+
+    /* optional leading minus */
+    if (c == '-') {
+        RETURN_IF_EOF;    
+        c = readChar(lexer, jsonText, offset); 
+    }
+
+    /* a single zero, or a series of integers */
+    if (c == '0') {
+        RETURN_IF_EOF;    
+        c = readChar(lexer, jsonText, offset); 
+    } else if (c >= '1' && c <= '9') {
+        do {
+            RETURN_IF_EOF;    
+            c = readChar(lexer, jsonText, offset); 
+        } while (c >= '0' && c <= '9');
+    } else {
+        unreadChar(lexer, offset);
+        lexer->error = yajl_lex_missing_integer_after_minus;
+        return yajl_tok_error;
+    }
+
+    /* optional fraction (indicates this is floating point) */
+    if (c == '.') {
+        int numRd = 0;
+        
+        RETURN_IF_EOF;
+        c = readChar(lexer, jsonText, offset); 
+
+        while (c >= '0' && c <= '9') {
+            numRd++;
+            RETURN_IF_EOF;
+            c = readChar(lexer, jsonText, offset); 
+        } 
+
+        if (!numRd) {
+            unreadChar(lexer, offset);
+            lexer->error = yajl_lex_missing_integer_after_decimal;
+            return yajl_tok_error;
+        }
+        tok = yajl_tok_double;
+    }
+
+    /* optional exponent (indicates this is floating point) */
+    if (c == 'e' || c == 'E') {
+        RETURN_IF_EOF;
+        c = readChar(lexer, jsonText, offset); 
+
+        /* optional sign */
+        if (c == '+' || c == '-') {
+            RETURN_IF_EOF;
+            c = readChar(lexer, jsonText, offset); 
+        }
+
+        if (c >= '0' && c <= '9') {
+            do {
+                RETURN_IF_EOF;
+                c = readChar(lexer, jsonText, offset); 
+            } while (c >= '0' && c <= '9');
+        } else {
+            unreadChar(lexer, offset);
+            lexer->error = yajl_lex_missing_integer_after_exponent;
+            return yajl_tok_error;
+        }
+        tok = yajl_tok_double;
+    }
+    
+    /* we always go "one too far" */
+    unreadChar(lexer, offset);
+    
+    return tok;
+}
+
+static yajl_tok
+yajl_lex_comment(yajl_lexer lexer, const unsigned char * jsonText,
+                 size_t jsonTextLen, size_t * offset)
+{
+    unsigned char c;
+
+    yajl_tok tok = yajl_tok_comment;
+
+    RETURN_IF_EOF;    
+    c = readChar(lexer, jsonText, offset);
+
+    /* either slash or star expected */
+    if (c == '/') {
+        /* now we throw away until end of line */
+        do {
+            RETURN_IF_EOF;    
+            c = readChar(lexer, jsonText, offset); 
+        } while (c != '\n');
+    } else if (c == '*') {
+        /* now we throw away until end of comment */        
+        for (;;) {
+            RETURN_IF_EOF;    
+            c = readChar(lexer, jsonText, offset); 
+            if (c == '*') {
+                RETURN_IF_EOF;    
+                c = readChar(lexer, jsonText, offset);                 
+                if (c == '/') {
+                    break;
+                } else {
+                    unreadChar(lexer, offset);
+                }
+            }
+        }
+    } else {
+        lexer->error = yajl_lex_invalid_char;
+        tok = yajl_tok_error;
+    }
+    
+    return tok;
+}
+
+yajl_tok
+yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText,
+             size_t jsonTextLen, size_t * offset,
+             const unsigned char ** outBuf, size_t * outLen)
+{
+    yajl_tok tok = yajl_tok_error;
+    unsigned char c;
+    size_t startOffset = *offset;
+
+    *outBuf = NULL;
+    *outLen = 0;
+
+    for (;;) {
+        assert(*offset <= jsonTextLen);
+
+        if (*offset >= jsonTextLen) {
+            tok = yajl_tok_eof;
+            goto lexed;
+        }
+
+        c = readChar(lexer, jsonText, offset);
+
+        switch (c) {
+            case '{':
+                tok = yajl_tok_left_bracket;
+                goto lexed;
+            case '}':
+                tok = yajl_tok_right_bracket;
+                goto lexed;
+            case '[':
+                tok = yajl_tok_left_brace;
+                goto lexed;
+            case ']':
+                tok = yajl_tok_right_brace;
+                goto lexed;
+            case ',':
+                tok = yajl_tok_comma;
+                goto lexed;
+            case ':':
+                tok = yajl_tok_colon;
+                goto lexed;
+            case '\t': case '\n': case '\v': case '\f': case '\r': case ' ':
+                startOffset++;
+                break;
+            case 't': {
+                const char * want = "rue";
+                do {
+                    if (*offset >= jsonTextLen) {
+                        tok = yajl_tok_eof;
+                        goto lexed;
+                    }
+                    c = readChar(lexer, jsonText, offset);
+                    if (c != *want) {
+                        unreadChar(lexer, offset);
+                        lexer->error = yajl_lex_invalid_string;
+                        tok = yajl_tok_error;
+                        goto lexed;
+                    }
+                } while (*(++want));
+                tok = yajl_tok_bool;
+                goto lexed;
+            }
+            case 'f': {
+                const char * want = "alse";
+                do {
+                    if (*offset >= jsonTextLen) {
+                        tok = yajl_tok_eof;
+                        goto lexed;
+                    }
+                    c = readChar(lexer, jsonText, offset);
+                    if (c != *want) {
+                        unreadChar(lexer, offset);
+                        lexer->error = yajl_lex_invalid_string;
+                        tok = yajl_tok_error;
+                        goto lexed;
+                    }
+                } while (*(++want));
+                tok = yajl_tok_bool;
+                goto lexed;
+            }
+            case 'n': {
+                const char * want = "ull";
+                do {
+                    if (*offset >= jsonTextLen) {
+                        tok = yajl_tok_eof;
+                        goto lexed;
+                    }
+                    c = readChar(lexer, jsonText, offset);
+                    if (c != *want) {
+                        unreadChar(lexer, offset);
+                        lexer->error = yajl_lex_invalid_string;
+                        tok = yajl_tok_error;
+                        goto lexed;
+                    }
+                } while (*(++want));
+                tok = yajl_tok_null;
+                goto lexed;
+            }
+            case '"': {
+                tok = yajl_lex_string(lexer, (const unsigned char *) jsonText,
+                                      jsonTextLen, offset);
+                goto lexed;
+            }
+            case '-':
+            case '0': case '1': case '2': case '3': case '4': 
+            case '5': case '6': case '7': case '8': case '9': {
+                /* integer parsing wants to start from the beginning */
+                unreadChar(lexer, offset);
+                tok = yajl_lex_number(lexer, (const unsigned char *) jsonText,
+                                      jsonTextLen, offset);
+                goto lexed;
+            }
+            case '/':
+                /* hey, look, a probable comment!  If comments are disabled
+                 * it's an error. */
+                if (!lexer->allowComments) {
+                    unreadChar(lexer, offset);
+                    lexer->error = yajl_lex_unallowed_comment;
+                    tok = yajl_tok_error;
+                    goto lexed;
+                }
+                /* if comments are enabled, then we should try to lex
+                 * the thing.  possible outcomes are
+                 * - successful lex (tok_comment, which means continue),
+                 * - malformed comment opening (slash not followed by
+                 *   '*' or '/') (tok_error)
+                 * - eof hit. (tok_eof) */
+                tok = yajl_lex_comment(lexer, (const unsigned char *) jsonText,
+                                       jsonTextLen, offset);
+                if (tok == yajl_tok_comment) {
+                    /* "error" is silly, but that's the initial
+                     * state of tok.  guilty until proven innocent. */  
+                    tok = yajl_tok_error;
+                    yajl_buf_clear(lexer->buf);
+                    lexer->bufInUse = 0;
+                    startOffset = *offset; 
+                    break;
+                }
+                /* hit error or eof, bail */
+                goto lexed;
+            default:
+                lexer->error = yajl_lex_invalid_char;
+                tok = yajl_tok_error;
+                goto lexed;
+        }
+    }
+
+
+  lexed:
+    /* need to append to buffer if the buffer is in use or
+     * if it's an EOF token */
+    if (tok == yajl_tok_eof || lexer->bufInUse) {
+        if (!lexer->bufInUse) yajl_buf_clear(lexer->buf);
+        lexer->bufInUse = 1;
+        yajl_buf_append(lexer->buf, jsonText + startOffset, *offset - startOffset);
+        lexer->bufOff = 0;
+        
+        if (tok != yajl_tok_eof) {
+            *outBuf = yajl_buf_data(lexer->buf);
+            *outLen = yajl_buf_len(lexer->buf);
+            lexer->bufInUse = 0;
+        }
+    } else if (tok != yajl_tok_error) {
+        *outBuf = jsonText + startOffset;
+        *outLen = *offset - startOffset;
+    }
+
+    /* special case for strings. skip the quotes. */
+    if (tok == yajl_tok_string || tok == yajl_tok_string_with_escapes)
+    {
+        assert(*outLen >= 2);
+        (*outBuf)++;
+        *outLen -= 2; 
+    }
+
+
+#ifdef YAJL_LEXER_DEBUG
+    if (tok == yajl_tok_error) {
+        printf("lexical error: %s\n",
+               yajl_lex_error_to_string(yajl_lex_get_error(lexer)));
+    } else if (tok == yajl_tok_eof) {
+        printf("EOF hit\n");
+    } else {
+        printf("lexed %s: '", tokToStr(tok));
+        fwrite(*outBuf, 1, *outLen, stdout);
+        printf("'\n");
+    }
+#endif
+
+    return tok;
+}
+
+const char *
+yajl_lex_error_to_string(yajl_lex_error error)
+{
+    switch (error) {
+        case yajl_lex_e_ok:
+            return "ok, no error";
+        case yajl_lex_string_invalid_utf8:
+            return "invalid bytes in UTF8 string.";
+        case yajl_lex_string_invalid_escaped_char:
+            return "inside a string, '\\' occurs before a character "
+                   "which it may not.";
+        case yajl_lex_string_invalid_json_char:            
+            return "invalid character inside string.";
+        case yajl_lex_string_invalid_hex_char:
+            return "invalid (non-hex) character occurs after '\\u' inside "
+                   "string.";
+        case yajl_lex_invalid_char:
+            return "invalid char in json text.";
+        case yajl_lex_invalid_string:
+            return "invalid string in json text.";
+        case yajl_lex_missing_integer_after_exponent:
+            return "malformed number, a digit is required after the exponent.";
+        case yajl_lex_missing_integer_after_decimal:
+            return "malformed number, a digit is required after the "
+                   "decimal point.";
+        case yajl_lex_missing_integer_after_minus:
+            return "malformed number, a digit is required after the "
+                   "minus sign.";
+        case yajl_lex_unallowed_comment:
+            return "probable comment found in input text, comments are "
+                   "not enabled.";
+    }
+    return "unknown error code";
+}
+
+
+/** allows access to more specific information about the lexical
+ *  error when yajl_lex_lex returns yajl_tok_error. */
+yajl_lex_error
+yajl_lex_get_error(yajl_lexer lexer)
+{
+    if (lexer == NULL) return (yajl_lex_error) -1;
+    return lexer->error;
+}
+
+size_t yajl_lex_current_line(yajl_lexer lexer)
+{
+    return lexer->lineOff;
+}
+
+size_t yajl_lex_current_char(yajl_lexer lexer)
+{
+    return lexer->charOff;
+}
+
+yajl_tok yajl_lex_peek(yajl_lexer lexer, const unsigned char * jsonText,
+                       size_t jsonTextLen, size_t offset)
+{
+    const unsigned char * outBuf;
+    size_t outLen;
+    size_t bufLen = yajl_buf_len(lexer->buf);
+    size_t bufOff = lexer->bufOff;
+    unsigned int bufInUse = lexer->bufInUse;
+    yajl_tok tok;
+    
+    tok = yajl_lex_lex(lexer, jsonText, jsonTextLen, &offset,
+                       &outBuf, &outLen);
+
+    lexer->bufOff = bufOff;
+    lexer->bufInUse = bufInUse;
+    yajl_buf_truncate(lexer->buf, bufLen);
+    
+    return tok;
+}
diff --git a/xlators/cluster/nsr-server/src/yajl_lex.h b/xlators/cluster/nsr-server/src/yajl_lex.h
new file mode 100644
index 000000000..cbaae0c13
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/yajl_lex.h
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2007-2011, Lloyd Hilaiel <lloyd@hilaiel.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef __YAJL_LEX_H__
+#define __YAJL_LEX_H__
+
+#include "yajl/yajl_common.h"
+
+typedef enum {
+    yajl_tok_bool,         
+    yajl_tok_colon,
+    yajl_tok_comma,     
+    yajl_tok_eof,
+    yajl_tok_error,
+    yajl_tok_left_brace,     
+    yajl_tok_left_bracket,
+    yajl_tok_null,         
+    yajl_tok_right_brace,     
+    yajl_tok_right_bracket,
+
+    /* we differentiate between integers and doubles to allow the
+     * parser to interpret the number without re-scanning */
+    yajl_tok_integer, 
+    yajl_tok_double, 
+
+    /* we differentiate between strings which require further processing,
+     * and strings that do not */
+    yajl_tok_string,
+    yajl_tok_string_with_escapes,
+
+    /* comment tokens are not currently returned to the parser, ever */
+    yajl_tok_comment
+} yajl_tok;
+
+typedef struct yajl_lexer_t * yajl_lexer;
+
+yajl_lexer yajl_lex_alloc(yajl_alloc_funcs * alloc,
+                          unsigned int allowComments,
+                          unsigned int validateUTF8);
+
+void yajl_lex_free(yajl_lexer lexer);
+
+/**
+ * run/continue a lex. "offset" is an input/output parameter.
+ * It should be initialized to zero for a
+ * new chunk of target text, and upon subsetquent calls with the same
+ * target text should passed with the value of the previous invocation.
+ *
+ * the client may be interested in the value of offset when an error is
+ * returned from the lexer.  This allows the client to render useful
+n * error messages.
+ *
+ * When you pass the next chunk of data, context should be reinitialized
+ * to zero.
+ * 
+ * Finally, the output buffer is usually just a pointer into the jsonText,
+ * however in cases where the entity being lexed spans multiple chunks,
+ * the lexer will buffer the entity and the data returned will be
+ * a pointer into that buffer.
+ *
+ * This behavior is abstracted from client code except for the performance
+ * implications which require that the client choose a reasonable chunk
+ * size to get adequate performance.
+ */
+yajl_tok yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText,
+                      size_t jsonTextLen, size_t * offset,
+                      const unsigned char ** outBuf, size_t * outLen);
+
+/** have a peek at the next token, but don't move the lexer forward */
+yajl_tok yajl_lex_peek(yajl_lexer lexer, const unsigned char * jsonText,
+                       size_t jsonTextLen, size_t offset);
+
+
+typedef enum {
+    yajl_lex_e_ok = 0,
+    yajl_lex_string_invalid_utf8,
+    yajl_lex_string_invalid_escaped_char,
+    yajl_lex_string_invalid_json_char,
+    yajl_lex_string_invalid_hex_char,
+    yajl_lex_invalid_char,
+    yajl_lex_invalid_string,
+    yajl_lex_missing_integer_after_decimal,
+    yajl_lex_missing_integer_after_exponent,
+    yajl_lex_missing_integer_after_minus,
+    yajl_lex_unallowed_comment
+} yajl_lex_error;
+
+const char * yajl_lex_error_to_string(yajl_lex_error error);
+
+/** allows access to more specific information about the lexical
+ *  error when yajl_lex_lex returns yajl_tok_error. */
+yajl_lex_error yajl_lex_get_error(yajl_lexer lexer);
+
+/** get the current offset into the most recently lexed json string. */
+size_t yajl_lex_current_offset(yajl_lexer lexer);
+
+/** get the number of lines lexed by this lexer instance */
+size_t yajl_lex_current_line(yajl_lexer lexer);
+
+/** get the number of chars lexed by this lexer instance since the last
+ *  \n or \r */
+size_t yajl_lex_current_char(yajl_lexer lexer);
+
+#endif
diff --git a/xlators/cluster/nsr-server/src/yajl_parser.c b/xlators/cluster/nsr-server/src/yajl_parser.c
new file mode 100644
index 000000000..bf9ef24ef
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/yajl_parser.c
@@ -0,0 +1,492 @@
+/*
+ * Copyright (c) 2007-2011, Lloyd Hilaiel <lloyd@hilaiel.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "yajl/yajl_parse.h"
+#include "yajl_lex.h"
+#include "yajl_parser.h"
+#include "yajl_encode.h"
+#include "yajl_bytestack.h"
+
+#include <stdlib.h>
+#include <limits.h>
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+#include <assert.h>
+#include <math.h>
+
+#define MAX_VALUE_TO_MULTIPLY ((LLONG_MAX / 10) + (LLONG_MAX % 10))
+
+ /* same semantics as strtol */
+long long
+yajl_parse_integer(const unsigned char *number, unsigned int length)
+{
+    long long ret  = 0;
+    long sign = 1;
+    const unsigned char *pos = number;
+    if (*pos == '-') { pos++; sign = -1; }
+    if (*pos == '+') { pos++; }
+
+    while (pos < number + length) {
+        if ( ret > MAX_VALUE_TO_MULTIPLY ) {
+            errno = ERANGE;
+            return sign == 1 ? LLONG_MAX : LLONG_MIN;
+        }
+        ret *= 10;
+        if (LLONG_MAX - ret < (*pos - '0')) {
+            errno = ERANGE;
+            return sign == 1 ? LLONG_MAX : LLONG_MIN;
+        }
+        ret += (*pos++ - '0');
+    }
+
+    return sign * ret;
+}
+
+unsigned char *
+yajl_render_error_string(yajl_handle hand, const unsigned char * jsonText,
+                         size_t jsonTextLen, int verbose)
+{
+    size_t offset = hand->bytesConsumed;
+    unsigned char * str;
+    const char * errorType = NULL;
+    const char * errorText = NULL;
+    char text[72];
+    const char * arrow = "                     (right here) ------^\n";
+
+    if (yajl_bs_current(hand->stateStack) == yajl_state_parse_error) {
+        errorType = "parse";
+        errorText = hand->parseError;
+    } else if (yajl_bs_current(hand->stateStack) == yajl_state_lexical_error) {
+        errorType = "lexical";
+        errorText = yajl_lex_error_to_string(yajl_lex_get_error(hand->lexer));
+    } else {
+        errorType = "unknown";
+    }
+
+    {
+        size_t memneeded = 0;
+        memneeded += strlen(errorType);
+        memneeded += strlen(" error");
+        if (errorText != NULL) {
+            memneeded += strlen(": ");
+            memneeded += strlen(errorText);
+        }
+        str = (unsigned char *) YA_MALLOC(&(hand->alloc), memneeded + 2);
+        if (!str) return NULL;
+        str[0] = 0;
+        strcat((char *) str, errorType);
+        strcat((char *) str, " error");
+        if (errorText != NULL) {
+            strcat((char *) str, ": ");
+            strcat((char *) str, errorText);
+        }
+        strcat((char *) str, "\n");
+    }
+
+    /* now we append as many spaces as needed to make sure the error
+     * falls at char 41, if verbose was specified */
+    if (verbose) {
+        size_t start, end, i;
+        size_t spacesNeeded;
+
+        spacesNeeded = (offset < 30 ? 40 - offset : 10);
+        start = (offset >= 30 ? offset - 30 : 0);
+        end = (offset + 30 > jsonTextLen ? jsonTextLen : offset + 30);
+
+        for (i=0;i<spacesNeeded;i++) text[i] = ' ';
+
+        for (;start < end;start++, i++) {
+            if (jsonText[start] != '\n' && jsonText[start] != '\r')
+            {
+                text[i] = jsonText[start];
+            }
+            else
+            {
+                text[i] = ' ';
+            }
+        }
+        assert(i <= 71);
+        text[i++] = '\n';
+        text[i] = 0;
+        {
+            char * newStr = (char *)
+                YA_MALLOC(&(hand->alloc), (unsigned int)(strlen((char *) str) +
+                                                         strlen((char *) text) +
+                                                         strlen(arrow) + 1));
+            if (newStr) {
+                newStr[0] = 0;
+                strcat((char *) newStr, (char *) str);
+                strcat((char *) newStr, text);
+                strcat((char *) newStr, arrow);
+            }
+            YA_FREE(&(hand->alloc), str);
+            str = (unsigned char *) newStr;
+        }
+    }
+    return str;
+}
+
+/* check for client cancelation */
+#define _CC_CHK(x)                                                \
+    if (!(x)) {                                                   \
+        yajl_bs_set(hand->stateStack, yajl_state_parse_error);    \
+        hand->parseError =                                        \
+            "client cancelled parse via callback return value";   \
+        return yajl_status_client_canceled;                       \
+    }
+
+
+yajl_status
+yajl_do_finish(yajl_handle hand)
+{
+    yajl_status stat;
+    stat = yajl_do_parse(hand,(const unsigned char *) " ",1);
+
+    if (stat != yajl_status_ok) return stat;
+
+    switch(yajl_bs_current(hand->stateStack))
+    {
+        case yajl_state_parse_error:
+        case yajl_state_lexical_error:
+            return yajl_status_error;
+        case yajl_state_got_value:
+        case yajl_state_parse_complete:
+            return yajl_status_ok;
+        default:
+            if (!(hand->flags & yajl_allow_partial_values))
+            {
+                yajl_bs_set(hand->stateStack, yajl_state_parse_error);
+                hand->parseError = "premature EOF";
+                return yajl_status_error;
+            }
+            return yajl_status_ok;
+    }
+}
+
+yajl_status
+yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
+              size_t jsonTextLen)
+{
+    yajl_tok tok;
+    const unsigned char * buf;
+    size_t bufLen;
+    size_t * offset = &(hand->bytesConsumed);
+
+    *offset = 0;
+
+  around_again:
+    switch (yajl_bs_current(hand->stateStack)) {
+        case yajl_state_parse_complete:
+            if (hand->flags & yajl_allow_multiple_values) {
+                yajl_bs_set(hand->stateStack, yajl_state_got_value);
+                goto around_again;
+            }
+            if (!(hand->flags & yajl_allow_trailing_garbage)) {
+                if (*offset != jsonTextLen) {
+                    tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen,
+                                       offset, &buf, &bufLen);
+                    if (tok != yajl_tok_eof) {
+                        yajl_bs_set(hand->stateStack, yajl_state_parse_error);
+                        hand->parseError = "trailing garbage";
+                    }
+                    goto around_again;
+                }
+            }
+            return yajl_status_ok;
+        case yajl_state_lexical_error:
+        case yajl_state_parse_error:
+            return yajl_status_error;
+        case yajl_state_start:
+        case yajl_state_got_value:
+        case yajl_state_map_need_val:
+        case yajl_state_array_need_val:
+        case yajl_state_array_start:  {
+            /* for arrays and maps, we advance the state for this
+             * depth, then push the state of the next depth.
+             * If an error occurs during the parsing of the nesting
+             * enitity, the state at this level will not matter.
+             * a state that needs pushing will be anything other
+             * than state_start */
+
+            yajl_state stateToPush = yajl_state_start;
+
+            tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen,
+                               offset, &buf, &bufLen);
+
+            switch (tok) {
+                case yajl_tok_eof:
+                    return yajl_status_ok;
+                case yajl_tok_error:
+                    yajl_bs_set(hand->stateStack, yajl_state_lexical_error);
+                    goto around_again;
+                case yajl_tok_string:
+                    if (hand->callbacks && hand->callbacks->yajl_string) {
+                        _CC_CHK(hand->callbacks->yajl_string(hand->ctx,
+                                                             buf, bufLen));
+                    }
+                    break;
+                case yajl_tok_string_with_escapes:
+                    if (hand->callbacks && hand->callbacks->yajl_string) {
+                        yajl_buf_clear(hand->decodeBuf);
+                        yajl_string_decode(hand->decodeBuf, buf, bufLen);
+                        _CC_CHK(hand->callbacks->yajl_string(
+                                    hand->ctx, yajl_buf_data(hand->decodeBuf),
+                                    yajl_buf_len(hand->decodeBuf)));
+                    }
+                    break;
+                case yajl_tok_bool:
+                    if (hand->callbacks && hand->callbacks->yajl_boolean) {
+                        _CC_CHK(hand->callbacks->yajl_boolean(hand->ctx,
+                                                              *buf == 't'));
+                    }
+                    break;
+                case yajl_tok_null:
+                    if (hand->callbacks && hand->callbacks->yajl_null) {
+                        _CC_CHK(hand->callbacks->yajl_null(hand->ctx));
+                    }
+                    break;
+                case yajl_tok_left_bracket:
+                    if (hand->callbacks && hand->callbacks->yajl_start_map) {
+                        _CC_CHK(hand->callbacks->yajl_start_map(hand->ctx));
+                    }
+                    stateToPush = yajl_state_map_start;
+                    break;
+                case yajl_tok_left_brace:
+                    if (hand->callbacks && hand->callbacks->yajl_start_array) {
+                        _CC_CHK(hand->callbacks->yajl_start_array(hand->ctx));
+                    }
+                    stateToPush = yajl_state_array_start;
+                    break;
+                case yajl_tok_integer:
+                    if (hand->callbacks) {
+                        if (hand->callbacks->yajl_number) {
+                            _CC_CHK(hand->callbacks->yajl_number(
+                                        hand->ctx,(const char *) buf, bufLen));
+                        } else if (hand->callbacks->yajl_integer) {
+                            long long int i = 0;
+                            i = yajl_parse_integer(buf, bufLen);
+                            if ((i == LLONG_MIN || i == LLONG_MAX) &&
+                                errno == ERANGE)
+                            {
+                                yajl_bs_set(hand->stateStack,
+                                            yajl_state_parse_error);
+                                hand->parseError = "integer overflow" ;
+                                /* try to restore error offset */
+                                if (*offset >= bufLen) *offset -= bufLen;
+                                else *offset = 0;
+                                goto around_again;
+                            }
+                            _CC_CHK(hand->callbacks->yajl_integer(hand->ctx,
+                                                                  i));
+                        }
+                    }
+                    break;
+                case yajl_tok_double:
+                    if (hand->callbacks) {
+                        if (hand->callbacks->yajl_number) {
+                            _CC_CHK(hand->callbacks->yajl_number(
+                                        hand->ctx, (const char *) buf, bufLen));
+                        } else if (hand->callbacks->yajl_double) {
+                            double d = 0.0;
+                            yajl_buf_clear(hand->decodeBuf);
+                            yajl_buf_append(hand->decodeBuf, buf, bufLen);
+                            buf = yajl_buf_data(hand->decodeBuf);
+                            d = strtod((char *) buf, NULL);
+                            if ((d == HUGE_VAL || d == -HUGE_VAL) &&
+                                errno == ERANGE)
+                            {
+                                yajl_bs_set(hand->stateStack,
+                                            yajl_state_parse_error);
+                                hand->parseError = "numeric (floating point) "
+                                    "overflow";
+                                /* try to restore error offset */
+                                if (*offset >= bufLen) *offset -= bufLen;
+                                else *offset = 0;
+                                goto around_again;
+                            }
+                            _CC_CHK(hand->callbacks->yajl_double(hand->ctx,
+                                                                 d));
+                        }
+                    }
+                    break;
+                case yajl_tok_right_brace: {
+                    if (yajl_bs_current(hand->stateStack) ==
+                        yajl_state_array_start)
+                    {
+                        if (hand->callbacks &&
+                            hand->callbacks->yajl_end_array)
+                        {
+                            _CC_CHK(hand->callbacks->yajl_end_array(hand->ctx));
+                        }
+                        yajl_bs_pop(hand->stateStack);
+                        goto around_again;
+                    }
+                    /* intentional fall-through */
+                }
+                case yajl_tok_colon:
+                case yajl_tok_comma:
+                case yajl_tok_right_bracket:
+                    yajl_bs_set(hand->stateStack, yajl_state_parse_error);
+                    hand->parseError =
+                        "unallowed token at this point in JSON text";
+                    goto around_again;
+                default:
+                    yajl_bs_set(hand->stateStack, yajl_state_parse_error);
+                    hand->parseError = "invalid token, internal error";
+                    goto around_again;
+            }
+            /* got a value.  transition depends on the state we're in. */
+            {
+                yajl_state s = yajl_bs_current(hand->stateStack);
+                if (s == yajl_state_start || s == yajl_state_got_value) {
+                    yajl_bs_set(hand->stateStack, yajl_state_parse_complete);
+                } else if (s == yajl_state_map_need_val) {
+                    yajl_bs_set(hand->stateStack, yajl_state_map_got_val);
+                } else {
+                    yajl_bs_set(hand->stateStack, yajl_state_array_got_val);
+                }
+            }
+            if (stateToPush != yajl_state_start) {
+                yajl_bs_push(hand->stateStack, stateToPush);
+            }
+
+            goto around_again;
+        }
+        case yajl_state_map_start:
+        case yajl_state_map_need_key: {
+            /* only difference between these two states is that in
+             * start '}' is valid, whereas in need_key, we've parsed
+             * a comma, and a string key _must_ follow */
+            tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen,
+                               offset, &buf, &bufLen);
+            switch (tok) {
+                case yajl_tok_eof:
+                    return yajl_status_ok;
+                case yajl_tok_error:
+                    yajl_bs_set(hand->stateStack, yajl_state_lexical_error);
+                    goto around_again;
+                case yajl_tok_string_with_escapes:
+                    if (hand->callbacks && hand->callbacks->yajl_map_key) {
+                        yajl_buf_clear(hand->decodeBuf);
+                        yajl_string_decode(hand->decodeBuf, buf, bufLen);
+                        buf = yajl_buf_data(hand->decodeBuf);
+                        bufLen = yajl_buf_len(hand->decodeBuf);
+                    }
+                    /* intentional fall-through */
+                case yajl_tok_string:
+                    if (hand->callbacks && hand->callbacks->yajl_map_key) {
+                        _CC_CHK(hand->callbacks->yajl_map_key(hand->ctx, buf,
+                                                              bufLen));
+                    }
+                    yajl_bs_set(hand->stateStack, yajl_state_map_sep);
+                    goto around_again;
+                case yajl_tok_right_bracket:
+                    if (yajl_bs_current(hand->stateStack) ==
+                        yajl_state_map_start)
+                    {
+                        if (hand->callbacks && hand->callbacks->yajl_end_map) {
+                            _CC_CHK(hand->callbacks->yajl_end_map(hand->ctx));
+                        }
+                        yajl_bs_pop(hand->stateStack);
+                        goto around_again;
+                    }
+                default:
+                    yajl_bs_set(hand->stateStack, yajl_state_parse_error);
+                    hand->parseError =
+                        "invalid object key (must be a string)"; 
+                    goto around_again;
+            }
+        }
+        case yajl_state_map_sep: {
+            tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen,
+                               offset, &buf, &bufLen);
+            switch (tok) {
+                case yajl_tok_colon:
+                    yajl_bs_set(hand->stateStack, yajl_state_map_need_val);
+                    goto around_again;
+                case yajl_tok_eof:
+                    return yajl_status_ok;
+                case yajl_tok_error:
+                    yajl_bs_set(hand->stateStack, yajl_state_lexical_error);
+                    goto around_again;
+                default:
+                    yajl_bs_set(hand->stateStack, yajl_state_parse_error);
+                    hand->parseError = "object key and value must "
+                        "be separated by a colon (':')";
+                    goto around_again;
+            }
+        }
+        case yajl_state_map_got_val: {
+            tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen,
+                               offset, &buf, &bufLen);
+            switch (tok) {
+                case yajl_tok_right_bracket:
+                    if (hand->callbacks && hand->callbacks->yajl_end_map) {
+                        _CC_CHK(hand->callbacks->yajl_end_map(hand->ctx));
+                    }
+                    yajl_bs_pop(hand->stateStack);
+                    goto around_again;
+                case yajl_tok_comma:
+                    yajl_bs_set(hand->stateStack, yajl_state_map_need_key);
+                    goto around_again;
+                case yajl_tok_eof:
+                    return yajl_status_ok;
+                case yajl_tok_error:
+                    yajl_bs_set(hand->stateStack, yajl_state_lexical_error);
+                    goto around_again;
+                default:
+                    yajl_bs_set(hand->stateStack, yajl_state_parse_error);
+                    hand->parseError = "after key and value, inside map, "
+                                       "I expect ',' or '}'";
+                    /* try to restore error offset */
+                    if (*offset >= bufLen) *offset -= bufLen;
+                    else *offset = 0;
+                    goto around_again;
+            }
+        }
+        case yajl_state_array_got_val: {
+            tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen,
+                               offset, &buf, &bufLen);
+            switch (tok) {
+                case yajl_tok_right_brace:
+                    if (hand->callbacks && hand->callbacks->yajl_end_array) {
+                        _CC_CHK(hand->callbacks->yajl_end_array(hand->ctx));
+                    }
+                    yajl_bs_pop(hand->stateStack);
+                    goto around_again;
+                case yajl_tok_comma:
+                    yajl_bs_set(hand->stateStack, yajl_state_array_need_val);
+                    goto around_again;
+                case yajl_tok_eof:
+                    return yajl_status_ok;
+                case yajl_tok_error:
+                    yajl_bs_set(hand->stateStack, yajl_state_lexical_error);
+                    goto around_again;
+                default:
+                    yajl_bs_set(hand->stateStack, yajl_state_parse_error);
+                    hand->parseError =
+                        "after array element, I expect ',' or ']'";
+                    goto around_again;
+            }
+        }
+    }
+
+    abort();
+    return yajl_status_error;
+}
+
diff --git a/xlators/cluster/nsr-server/src/yajl_parser.h b/xlators/cluster/nsr-server/src/yajl_parser.h
new file mode 100644
index 000000000..53409731a
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/yajl_parser.h
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2007-2011, Lloyd Hilaiel <lloyd@hilaiel.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef __YAJL_PARSER_H__
+#define __YAJL_PARSER_H__
+
+#include "yajl/yajl_parse.h"
+#include "yajl_bytestack.h"
+#include "yajl_buf.h"
+#include "yajl_lex.h"
+
+
+typedef enum {
+    yajl_state_start = 0,
+    yajl_state_parse_complete,
+    yajl_state_parse_error,
+    yajl_state_lexical_error,
+    yajl_state_map_start,
+    yajl_state_map_sep,
+    yajl_state_map_need_val,
+    yajl_state_map_got_val,
+    yajl_state_map_need_key,
+    yajl_state_array_start,
+    yajl_state_array_got_val,
+    yajl_state_array_need_val,
+    yajl_state_got_value,
+} yajl_state;
+
+struct yajl_handle_t {
+    const yajl_callbacks * callbacks;
+    void * ctx;
+    yajl_lexer lexer;
+    const char * parseError;
+    /* the number of bytes consumed from the last client buffer,
+     * in the case of an error this will be an error offset, in the
+     * case of an error this can be used as the error offset */
+    size_t bytesConsumed;
+    /* temporary storage for decoded strings */
+    yajl_buf decodeBuf;
+    /* a stack of states.  access with yajl_state_XXX routines */
+    yajl_bytestack stateStack;
+    /* memory allocation routines */
+    yajl_alloc_funcs alloc;
+    /* bitfield */
+    unsigned int flags;
+};
+
+yajl_status
+yajl_do_parse(yajl_handle handle, const unsigned char * jsonText,
+              size_t jsonTextLen);
+
+yajl_status
+yajl_do_finish(yajl_handle handle);
+
+unsigned char *
+yajl_render_error_string(yajl_handle hand, const unsigned char * jsonText,
+                         size_t jsonTextLen, int verbose);
+
+/* A little built in integer parsing routine with the same semantics as strtol
+ * that's unaffected by LOCALE. */
+long long
+yajl_parse_integer(const unsigned char *number, unsigned int length);
+
+
+#endif
diff --git a/xlators/cluster/nsr-server/src/yajl_tree.c b/xlators/cluster/nsr-server/src/yajl_tree.c
new file mode 100644
index 000000000..1a69134e7
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/yajl_tree.c
@@ -0,0 +1,501 @@
+/*
+ * Copyright (c) 2010-2011  Florian Forster  <ff at octo.it>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <assert.h>
+
+#include "yajl/yajl_tree.h"
+#include "yajl/yajl_parse.h"
+
+#include "yajl_parser.h"
+
+#ifdef WIN32
+#define snprintf sprintf_s
+#endif
+
+#define STATUS_CONTINUE 1
+#define STATUS_ABORT    0
+
+struct stack_elem_s;
+typedef struct stack_elem_s stack_elem_t;
+struct stack_elem_s
+{
+    char * key;
+    yajl_val value;
+    stack_elem_t *next;
+};
+
+struct context_s
+{
+    stack_elem_t *stack;
+    yajl_val root;
+    char *errbuf;
+    size_t errbuf_size;
+};
+typedef struct context_s context_t;
+
+#define RETURN_ERROR(ctx,retval,...) {                                  \
+        if ((ctx)->errbuf != NULL)                                      \
+            snprintf ((ctx)->errbuf, (ctx)->errbuf_size, __VA_ARGS__);  \
+        return (retval);                                                \
+    }
+
+static yajl_val value_alloc (yajl_type type)
+{
+    yajl_val v;
+
+    v = malloc (sizeof (*v));
+    if (v == NULL) return (NULL);
+    memset (v, 0, sizeof (*v));
+    v->type = type;
+
+    return (v);
+}
+
+static void yajl_object_free (yajl_val v)
+{
+    size_t i;
+
+    if (!YAJL_IS_OBJECT(v)) return;
+
+    for (i = 0; i < v->u.object.len; i++)
+    {
+        free((char *) v->u.object.keys[i]);
+        v->u.object.keys[i] = NULL;
+        yajl_tree_free (v->u.object.values[i]);
+        v->u.object.values[i] = NULL;
+    }
+
+    free((void*) v->u.object.keys);
+    free(v->u.object.values);
+    free(v);
+}
+
+static void yajl_array_free (yajl_val v)
+{
+    size_t i;
+
+    if (!YAJL_IS_ARRAY(v)) return;
+
+    for (i = 0; i < v->u.array.len; i++)
+    {
+        yajl_tree_free (v->u.array.values[i]);
+        v->u.array.values[i] = NULL;
+    }
+
+    free(v->u.array.values);
+    free(v);
+}
+
+/*
+ * Parsing nested objects and arrays is implemented using a stack. When a new
+ * object or array starts (a curly or a square opening bracket is read), an
+ * appropriate value is pushed on the stack. When the end of the object is
+ * reached (an appropriate closing bracket has been read), the value is popped
+ * off the stack and added to the enclosing object using "context_add_value".
+ */
+static int context_push(context_t *ctx, yajl_val v)
+{
+    stack_elem_t *stack;
+
+    stack = malloc (sizeof (*stack));
+    if (stack == NULL)
+        RETURN_ERROR (ctx, ENOMEM, "Out of memory");
+    memset (stack, 0, sizeof (*stack));
+
+    assert ((ctx->stack == NULL)
+            || YAJL_IS_OBJECT (v)
+            || YAJL_IS_ARRAY (v));
+
+    stack->value = v;
+    stack->next = ctx->stack;
+    ctx->stack = stack;
+
+    return (0);
+}
+
+static yajl_val context_pop(context_t *ctx)
+{
+    stack_elem_t *stack;
+    yajl_val v;
+
+    if (ctx->stack == NULL)
+        RETURN_ERROR (ctx, NULL, "context_pop: "
+                      "Bottom of stack reached prematurely");
+
+    stack = ctx->stack;
+    ctx->stack = stack->next;
+
+    v = stack->value;
+
+    free (stack);
+
+    return (v);
+}
+
+static int object_add_keyval(context_t *ctx,
+                             yajl_val obj, char *key, yajl_val value)
+{
+    const char **tmpk;
+    yajl_val *tmpv;
+
+    /* We're checking for NULL in "context_add_value" or its callers. */
+    assert (ctx != NULL);
+    assert (obj != NULL);
+    assert (key != NULL);
+    assert (value != NULL);
+
+    /* We're assuring that "obj" is an object in "context_add_value". */
+    assert(YAJL_IS_OBJECT(obj));
+
+    tmpk = realloc((void *) obj->u.object.keys, sizeof(*(obj->u.object.keys)) * (obj->u.object.len + 1));
+    if (tmpk == NULL)
+        RETURN_ERROR(ctx, ENOMEM, "Out of memory");
+    obj->u.object.keys = tmpk;
+
+    tmpv = realloc(obj->u.object.values, sizeof (*obj->u.object.values) * (obj->u.object.len + 1));
+    if (tmpv == NULL)
+        RETURN_ERROR(ctx, ENOMEM, "Out of memory");
+    obj->u.object.values = tmpv;
+
+    obj->u.object.keys[obj->u.object.len] = key;
+    obj->u.object.values[obj->u.object.len] = value;
+    obj->u.object.len++;
+
+    return (0);
+}
+
+static int array_add_value (context_t *ctx,
+                            yajl_val array, yajl_val value)
+{
+    yajl_val *tmp;
+
+    /* We're checking for NULL pointers in "context_add_value" or its
+     * callers. */
+    assert (ctx != NULL);
+    assert (array != NULL);
+    assert (value != NULL);
+
+    /* "context_add_value" will only call us with array values. */
+    assert(YAJL_IS_ARRAY(array));
+    
+    tmp = realloc(array->u.array.values,
+                  sizeof(*(array->u.array.values)) * (array->u.array.len + 1));
+    if (tmp == NULL)
+        RETURN_ERROR(ctx, ENOMEM, "Out of memory");
+    array->u.array.values = tmp;
+    array->u.array.values[array->u.array.len] = value;
+    array->u.array.len++;
+
+    return 0;
+}
+
+/*
+ * Add a value to the value on top of the stack or the "root" member in the
+ * context if the end of the parsing process is reached.
+ */
+static int context_add_value (context_t *ctx, yajl_val v)
+{
+    /* We're checking for NULL values in all the calling functions. */
+    assert (ctx != NULL);
+    assert (v != NULL);
+
+    /*
+     * There are three valid states in which this function may be called:
+     *   - There is no value on the stack => This is the only value. This is the
+     *     last step done when parsing a document. We assign the value to the
+     *     "root" member and return.
+     *   - The value on the stack is an object. In this case store the key on the
+     *     stack or, if the key has already been read, add key and value to the
+     *     object.
+     *   - The value on the stack is an array. In this case simply add the value
+     *     and return.
+     */
+    if (ctx->stack == NULL)
+    {
+        assert (ctx->root == NULL);
+        ctx->root = v;
+        return (0);
+    }
+    else if (YAJL_IS_OBJECT (ctx->stack->value))
+    {
+        if (ctx->stack->key == NULL)
+        {
+            if (!YAJL_IS_STRING (v))
+                RETURN_ERROR (ctx, EINVAL, "context_add_value: "
+                              "Object key is not a string (%#04x)",
+                              v->type);
+
+            ctx->stack->key = v->u.string;
+            v->u.string = NULL;
+            free(v);
+            return (0);
+        }
+        else /* if (ctx->key != NULL) */
+        {
+            char * key;
+
+            key = ctx->stack->key;
+            ctx->stack->key = NULL;
+            return (object_add_keyval (ctx, ctx->stack->value, key, v));
+        }
+    }
+    else if (YAJL_IS_ARRAY (ctx->stack->value))
+    {
+        return (array_add_value (ctx, ctx->stack->value, v));
+    }
+    else
+    {
+        RETURN_ERROR (ctx, EINVAL, "context_add_value: Cannot add value to "
+                      "a value of type %#04x (not a composite type)",
+                      ctx->stack->value->type);
+    }
+}
+
+static int handle_string (void *ctx,
+                          const unsigned char *string, size_t string_length)
+{
+    yajl_val v;
+
+    v = value_alloc (yajl_t_string);
+    if (v == NULL)
+        RETURN_ERROR ((context_t *) ctx, STATUS_ABORT, "Out of memory");
+
+    v->u.string = malloc (string_length + 1);
+    if (v->u.string == NULL)
+    {
+        free (v);
+        RETURN_ERROR ((context_t *) ctx, STATUS_ABORT, "Out of memory");
+    }
+    memcpy(v->u.string, string, string_length);
+    v->u.string[string_length] = 0;
+
+    return ((context_add_value (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT);
+}
+
+static int handle_number (void *ctx, const char *string, size_t string_length)
+{
+    yajl_val v;
+    char *endptr;
+
+    v = value_alloc(yajl_t_number);
+    if (v == NULL)
+        RETURN_ERROR((context_t *) ctx, STATUS_ABORT, "Out of memory");
+
+    v->u.number.r = malloc(string_length + 1);
+    if (v->u.number.r == NULL)
+    {
+        free(v);
+        RETURN_ERROR((context_t *) ctx, STATUS_ABORT, "Out of memory");
+    }
+    memcpy(v->u.number.r, string, string_length);
+    v->u.number.r[string_length] = 0;
+
+    v->u.number.flags = 0;
+
+    endptr = NULL;
+    errno = 0;
+    v->u.number.i = yajl_parse_integer((const unsigned char *) v->u.number.r,
+                                       strlen(v->u.number.r));
+    if ((errno == 0) && (endptr != NULL) && (*endptr == 0))
+        v->u.number.flags |= YAJL_NUMBER_INT_VALID;
+
+    endptr = NULL;
+    errno = 0;
+    v->u.number.d = strtod(v->u.number.r, &endptr);
+    if ((errno == 0) && (endptr != NULL) && (*endptr == 0))
+        v->u.number.flags |= YAJL_NUMBER_DOUBLE_VALID;
+
+    return ((context_add_value(ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT);
+}
+
+static int handle_start_map (void *ctx)
+{
+    yajl_val v;
+
+    v = value_alloc(yajl_t_object);
+    if (v == NULL)
+        RETURN_ERROR ((context_t *) ctx, STATUS_ABORT, "Out of memory");
+
+    v->u.object.keys = NULL;
+    v->u.object.values = NULL;
+    v->u.object.len = 0;
+
+    return ((context_push (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT);
+}
+
+static int handle_end_map (void *ctx)
+{
+    yajl_val v;
+
+    v = context_pop (ctx);
+    if (v == NULL)
+        return (STATUS_ABORT);
+
+    return ((context_add_value (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT);
+}
+
+static int handle_start_array (void *ctx)
+{
+    yajl_val v;
+
+    v = value_alloc(yajl_t_array);
+    if (v == NULL)
+        RETURN_ERROR ((context_t *) ctx, STATUS_ABORT, "Out of memory");
+
+    v->u.array.values = NULL;
+    v->u.array.len = 0;
+
+    return ((context_push (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT);
+}
+
+static int handle_end_array (void *ctx)
+{
+    yajl_val v;
+
+    v = context_pop (ctx);
+    if (v == NULL)
+        return (STATUS_ABORT);
+
+    return ((context_add_value (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT);
+}
+
+static int handle_boolean (void *ctx, int boolean_value)
+{
+    yajl_val v;
+
+    v = value_alloc (boolean_value ? yajl_t_true : yajl_t_false);
+    if (v == NULL)
+        RETURN_ERROR ((context_t *) ctx, STATUS_ABORT, "Out of memory");
+
+    return ((context_add_value (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT);
+}
+
+static int handle_null (void *ctx)
+{
+    yajl_val v;
+
+    v = value_alloc (yajl_t_null);
+    if (v == NULL)
+        RETURN_ERROR ((context_t *) ctx, STATUS_ABORT, "Out of memory");
+
+    return ((context_add_value (ctx, v) == 0) ? STATUS_CONTINUE : STATUS_ABORT);
+}
+
+/*
+ * Public functions
+ */
+yajl_val yajl_tree_parse (const char *input,
+                          char *error_buffer, size_t error_buffer_size)
+{
+    static const yajl_callbacks callbacks =
+        {
+            /* null        = */ handle_null,
+            /* boolean     = */ handle_boolean,
+            /* integer     = */ NULL,
+            /* double      = */ NULL,
+            /* number      = */ handle_number,
+            /* string      = */ handle_string,
+            /* start map   = */ handle_start_map,
+            /* map key     = */ handle_string,
+            /* end map     = */ handle_end_map,
+            /* start array = */ handle_start_array,
+            /* end array   = */ handle_end_array
+        };
+
+    yajl_handle handle;
+    yajl_status status;
+	context_t ctx = { NULL, NULL, NULL, 0 };
+
+	ctx.errbuf = error_buffer;
+	ctx.errbuf_size = error_buffer_size;
+
+    if (error_buffer != NULL)
+        memset (error_buffer, 0, error_buffer_size);
+
+    handle = yajl_alloc (&callbacks, NULL, &ctx);
+    yajl_config(handle, yajl_allow_comments, 1);
+
+    status = yajl_parse(handle,
+                        (unsigned char *) input,
+                        strlen (input));
+    status = yajl_complete_parse (handle);
+    if (status != yajl_status_ok) {
+        if (error_buffer != NULL && error_buffer_size > 0) {
+            snprintf(
+                error_buffer, error_buffer_size, "%s",
+                (char *) yajl_get_error(handle, 1,
+                                        (const unsigned char *) input,
+                                        strlen(input)));
+        }
+        yajl_free (handle);
+        return NULL;
+    }
+
+    yajl_free (handle);
+    return (ctx.root);
+}
+
+yajl_val yajl_tree_get(yajl_val n, const char ** path, yajl_type type)
+{
+    if (!path) return NULL;
+    while (n && *path) {
+        unsigned int i;
+
+        if (n->type != yajl_t_object) return NULL;
+        for (i = 0; i < n->u.object.len; i++) {
+            if (!strcmp(*path, n->u.object.keys[i])) {
+                n = n->u.object.values[i];
+                break;
+            }
+        }
+        if (i == n->u.object.len) return NULL;
+        path++;
+    }
+    if (n && type != yajl_t_any && type != n->type) n = NULL;
+    return n;
+}
+
+void yajl_tree_free (yajl_val v)
+{
+    if (v == NULL) return;
+
+    if (YAJL_IS_STRING(v))
+    {
+        free(v->u.string);
+        free(v);
+    }
+    else if (YAJL_IS_NUMBER(v))
+    {
+        free(v->u.number.r);
+        free(v);
+    }
+    else if (YAJL_GET_OBJECT(v))
+    {
+        yajl_object_free(v);
+    }
+    else if (YAJL_GET_ARRAY(v))
+    {
+        yajl_array_free(v);
+    }
+    else /* if (yajl_t_true or yajl_t_false or yajl_t_null) */
+    {
+        free(v);
+    }
+}
diff --git a/xlators/cluster/nsr-server/src/yajl_version.c b/xlators/cluster/nsr-server/src/yajl_version.c
new file mode 100644
index 000000000..0671da722
--- /dev/null
+++ b/xlators/cluster/nsr-server/src/yajl_version.c
@@ -0,0 +1,7 @@
+#include <yajl/yajl_version.h>
+
+int yajl_version(void)
+{
+	return YAJL_VERSION;
+}
+
diff --git a/xlators/cluster/stripe/src/Makefile.am b/xlators/cluster/stripe/src/Makefile.am
index 6d4fae4d2..4268d6f03 100644
--- a/xlators/cluster/stripe/src/Makefile.am
+++ b/xlators/cluster/stripe/src/Makefile.am
@@ -1,16 +1,19 @@
-
 xlator_LTLIBRARIES = stripe.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/cluster
 
-stripe_la_LDFLAGS = -module -avoidversion
+stripe_la_LDFLAGS = -module -avoid-version
+
+stripe_la_SOURCES = stripe.c stripe-helpers.c \
+	$(top_builddir)/xlators/lib/src/libxlator.c
 
-stripe_la_SOURCES = stripe.c
 stripe_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = stripe.h stripe-mem-types.h
+noinst_HEADERS = stripe.h stripe-mem-types.h $(top_builddir)/xlators/lib/src/libxlator.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/xlators/lib/src
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/cluster/stripe/src/stripe-helpers.c b/xlators/cluster/stripe/src/stripe-helpers.c
new file mode 100644
index 000000000..3c12809d6
--- /dev/null
+++ b/xlators/cluster/stripe/src/stripe-helpers.c
@@ -0,0 +1,677 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#include <fnmatch.h>
+
+#include "stripe.h"
+#include "byte-order.h"
+#include "mem-types.h"
+#include "logging.h"
+
+void
+stripe_local_wipe (stripe_local_t *local)
+{
+        if (!local)
+                goto out;
+
+        loc_wipe (&local->loc);
+        loc_wipe (&local->loc2);
+
+        if (local->fd)
+                fd_unref (local->fd);
+
+        if (local->inode)
+                inode_unref (local->inode);
+
+        if (local->xattr)
+                dict_unref (local->xattr);
+
+        if (local->xdata)
+                dict_unref (local->xdata);
+
+out:
+        return;
+}
+
+
+
+int
+stripe_aggregate (dict_t *this, char *key, data_t *value, void *data)
+{
+        dict_t  *dst  = NULL;
+        int64_t *ptr  = 0, *size = NULL;
+        int32_t  ret  = -1;
+
+        dst = data;
+
+        if (strcmp (key, GF_XATTR_QUOTA_SIZE_KEY) == 0) {
+                ret = dict_get_bin (dst, key, (void **)&size);
+                if (ret < 0) {
+                        size = GF_CALLOC (1, sizeof (int64_t),
+                                          gf_common_mt_char);
+                        if (size == NULL) {
+                                gf_log ("stripe", GF_LOG_WARNING,
+                                        "memory allocation failed");
+                                goto out;
+                        }
+                        ret = dict_set_bin (dst, key, size, sizeof (int64_t));
+                        if (ret < 0) {
+                                gf_log ("stripe", GF_LOG_WARNING,
+                                        "stripe aggregate dict set failed");
+                                GF_FREE (size);
+                                goto out;
+                        }
+                }
+
+                ptr = data_to_bin (value);
+                if (ptr == NULL) {
+                        gf_log ("stripe", GF_LOG_WARNING, "data to bin failed");
+                        goto out;
+                }
+
+                *size = hton64 (ntoh64 (*size) + ntoh64 (*ptr));
+        } else if (strcmp (key, GF_CONTENT_KEY)) {
+                /* No need to aggregate 'CONTENT' data */
+                ret = dict_set (dst, key, value);
+                if (ret)
+                        gf_log ("stripe", GF_LOG_WARNING, "xattr dict set failed");
+        }
+
+out:
+        return 0;
+}
+
+
+void
+stripe_aggregate_xattr (dict_t *dst, dict_t *src)
+{
+        if ((dst == NULL) || (src == NULL)) {
+                goto out;
+        }
+
+        dict_foreach (src, stripe_aggregate, dst);
+out:
+        return;
+}
+
+
+int32_t
+stripe_xattr_aggregate (char *buffer, stripe_local_t *local, int32_t *total)
+{
+        int32_t              i     = 0;
+        int32_t              ret   = -1;
+        int32_t              len   = 0;
+        char                *sbuf  = NULL;
+        stripe_xattr_sort_t *xattr = NULL;
+
+        if (!buffer || !local || !local->xattr_list)
+                goto out;
+
+        sbuf = buffer;
+
+        for (i = 0; i < local->nallocs; i++) {
+                xattr = local->xattr_list + i;
+                len = xattr->xattr_len;
+
+                if (len && xattr && xattr->xattr_value) {
+                        memcpy (buffer, xattr->xattr_value, len);
+                        buffer += len;
+                        *buffer++ = ' ';
+                }
+        }
+
+        *--buffer = '\0';
+        if (total)
+                *total = buffer - sbuf;
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+int32_t
+stripe_free_xattr_str (stripe_local_t *local)
+{
+        int32_t              i     = 0;
+        int32_t              ret   = -1;
+        stripe_xattr_sort_t *xattr = NULL;
+
+        if (!local || !local->xattr_list)
+                goto out;
+
+        for (i = 0; i < local->nallocs; i++) {
+                xattr = local->xattr_list + i;
+
+                if (xattr && xattr->xattr_value)
+                        GF_FREE (xattr->xattr_value);
+        }
+
+        ret = 0;
+ out:
+        return ret;
+}
+
+
+int32_t
+stripe_fill_lockinfo_xattr (xlator_t *this, stripe_local_t *local,
+                            void **xattr_serz)
+{
+        int32_t              ret   = -1, i = 0, len = 0;
+        dict_t              *tmp1  = NULL, *tmp2 = NULL;
+        char                *buf   = NULL;
+        stripe_xattr_sort_t *xattr = NULL;
+
+        if (xattr_serz == NULL) {
+                goto out;
+        }
+
+        tmp2 = dict_new ();
+
+        if (tmp2 == NULL) {
+                goto out;
+        }
+
+        for (i = 0; i < local->nallocs; i++) {
+                xattr = local->xattr_list + i;
+                len = xattr->xattr_len;
+
+                if (len && xattr && xattr->xattr_value) {
+                        ret = dict_reset (tmp2);
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "dict_reset failed (%s)",
+                                        strerror (-ret));
+                        }
+
+                        ret = dict_unserialize (xattr->xattr_value,
+                                                xattr->xattr_len,
+                                                &tmp2);
+                        if (ret < 0) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "dict_unserialize failed (%s)",
+                                        strerror (-ret));
+                                ret = -1;
+                                goto out;
+                        }
+
+                        tmp1 = dict_copy (tmp2, tmp1);
+                        if (tmp1 == NULL) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "dict_copy failed (%s)",
+                                        strerror (-ret));
+                                ret = -1;
+                                goto out;
+                        }
+                }
+        }
+
+        len = dict_serialized_length (tmp1);
+        if (len > 0) {
+                buf = GF_CALLOC (1, len, gf_common_mt_dict_t);
+                if (buf == NULL) {
+                        ret = -1;
+                        goto out;
+                }
+
+                ret = dict_serialize (tmp1, buf);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "dict_serialize failed (%s)", strerror (-ret));
+                        GF_FREE(buf);
+                        ret = -1;
+                        goto out;
+                }
+
+                *xattr_serz = buf;
+        }
+
+        ret = 0;
+out:
+        if (tmp1 != NULL) {
+                dict_unref (tmp1);
+        }
+
+        if (tmp2 != NULL) {
+                dict_unref (tmp2);
+        }
+
+        return ret;
+}
+
+
+int32_t
+stripe_fill_pathinfo_xattr (xlator_t *this, stripe_local_t *local,
+                            char **xattr_serz)
+{
+        int      ret             = -1;
+        int32_t  padding         = 0;
+        int32_t  tlen            = 0;
+        char stripe_size_str[20] = {0,};
+        char    *pathinfo_serz   = NULL;
+
+        if (!local) {
+                gf_log (this->name, GF_LOG_ERROR, "Possible NULL deref");
+                goto out;
+        }
+
+        (void) snprintf (stripe_size_str, 20, "%"PRId64,
+                         (long long) (local->fctx) ? local->fctx->stripe_size : 0);
+
+        /* extra bytes for decorations (brackets and <>'s) */
+        padding = strlen (this->name) + strlen (STRIPE_PATHINFO_HEADER)
+                + strlen (stripe_size_str) + 7;
+        local->xattr_total_len += (padding + 2);
+
+        pathinfo_serz = GF_CALLOC (local->xattr_total_len, sizeof (char),
+                                   gf_common_mt_char);
+        if (!pathinfo_serz)
+                goto out;
+
+        /* xlator info */
+        (void) sprintf (pathinfo_serz, "(<"STRIPE_PATHINFO_HEADER"%s:[%s]> ",
+                        this->name, stripe_size_str);
+
+        ret = stripe_xattr_aggregate (pathinfo_serz + padding, local, &tlen);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Cannot aggregate pathinfo list");
+                GF_FREE(pathinfo_serz);
+                goto out;
+        }
+
+        *(pathinfo_serz + padding + tlen) = ')';
+        *(pathinfo_serz + padding + tlen + 1) = '\0';
+
+        *xattr_serz = pathinfo_serz;
+
+        ret = 0;
+ out:
+        return ret;
+}
+
+/**
+ * stripe_get_matching_bs - Get the matching block size for the given path.
+ */
+int32_t
+stripe_get_matching_bs (const char *path, stripe_private_t *priv)
+{
+        struct stripe_options *trav       = NULL;
+        uint64_t               block_size = 0;
+
+        GF_VALIDATE_OR_GOTO ("stripe", priv, out);
+        GF_VALIDATE_OR_GOTO ("stripe", path, out);
+
+        LOCK (&priv->lock);
+        {
+                block_size = priv->block_size;
+                trav = priv->pattern;
+                while (trav) {
+                        if (!fnmatch (trav->path_pattern, path, FNM_NOESCAPE)) {
+                                block_size = trav->block_size;
+                                break;
+                        }
+                        trav = trav->next;
+                }
+        }
+        UNLOCK (&priv->lock);
+
+out:
+        return block_size;
+}
+
+int32_t
+stripe_ctx_handle (xlator_t *this, call_frame_t *prev, stripe_local_t *local,
+                   dict_t *dict)
+{
+        char            key[256]       = {0,};
+        data_t         *data            = NULL;
+        int32_t         index           = 0;
+        stripe_private_t *priv          = NULL;
+
+        priv = this->private;
+
+
+        if (!local->fctx) {
+                local->fctx =  GF_CALLOC (1, sizeof (stripe_fd_ctx_t),
+                                         gf_stripe_mt_stripe_fd_ctx_t);
+                if (!local->fctx) {
+                        local->op_errno = ENOMEM;
+                        local->op_ret = -1;
+                        goto out;
+                }
+
+                local->fctx->static_array = 0;
+        }
+        /* Stripe block size */
+        sprintf (key, "trusted.%s.stripe-size", this->name);
+        data = dict_get (dict, key);
+        if (!data) {
+                local->xattr_self_heal_needed = 1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get stripe-size");
+                goto out;
+        } else {
+                if (!local->fctx->stripe_size) {
+                        local->fctx->stripe_size =
+                                     data_to_int64 (data);
+                }
+
+                if (local->fctx->stripe_size != data_to_int64 (data)) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "stripe-size mismatch in blocks");
+                        local->xattr_self_heal_needed = 1;
+                }
+        }
+
+        /* Stripe count */
+        sprintf (key, "trusted.%s.stripe-count", this->name);
+        data = dict_get (dict, key);
+
+        if (!data) {
+                local->xattr_self_heal_needed = 1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get stripe-count");
+                goto out;
+        }
+        if (!local->fctx->xl_array) {
+                local->fctx->stripe_count = data_to_int32 (data);
+                if (!local->fctx->stripe_count) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error with stripe-count xattr");
+                        local->op_ret   = -1;
+                        local->op_errno = EIO;
+                        goto out;
+                }
+
+                local->fctx->xl_array = GF_CALLOC (local->fctx->stripe_count,
+                                                   sizeof (xlator_t *),
+                                                   gf_stripe_mt_xlator_t);
+
+                if (!local->fctx->xl_array) {
+                        local->op_errno = ENOMEM;
+                        local->op_ret   = -1;
+                        goto out;
+                }
+        }
+        if (local->fctx->stripe_count != data_to_int32 (data)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error with stripe-count xattr (%d != %d)",
+                        local->fctx->stripe_count, data_to_int32 (data));
+                local->op_ret   = -1;
+                local->op_errno = EIO;
+                goto out;
+        }
+
+        /* index */
+        sprintf (key, "trusted.%s.stripe-index", this->name);
+        data = dict_get (dict, key);
+        if (!data) {
+                local->xattr_self_heal_needed = 1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Failed to get stripe-index");
+                goto out;
+        }
+        index = data_to_int32 (data);
+        if (index > priv->child_count) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error with stripe-index xattr (%d)", index);
+                local->op_ret   = -1;
+                local->op_errno = EIO;
+                goto out;
+        }
+        if (local->fctx->xl_array) {
+                if (!local->fctx->xl_array[index])
+                        local->fctx->xl_array[index] = prev->this;
+        }
+
+	sprintf(key, "trusted.%s.stripe-coalesce", this->name);
+	data = dict_get(dict, key);
+	if (!data) {
+		/*
+		 * The file was probably created prior to coalesce support.
+		 * Assume non-coalesce mode for this file to maintain backwards
+		 * compatibility.
+		 */
+		gf_log(this->name, GF_LOG_DEBUG, "missing stripe-coalesce "
+			"attr, assume non-coalesce mode");
+		local->fctx->stripe_coalesce = 0;
+	} else {
+		local->fctx->stripe_coalesce = data_to_int32(data);
+	}
+
+
+out:
+        return 0;
+}
+
+int32_t
+stripe_xattr_request_build (xlator_t *this, dict_t *dict, uint64_t stripe_size,
+                            uint32_t stripe_count, uint32_t stripe_index,
+			    uint32_t stripe_coalesce)
+{
+        char            key[256]       = {0,};
+        int32_t         ret             = -1;
+
+        sprintf (key, "trusted.%s.stripe-size", this->name);
+        ret = dict_set_int64 (dict, key, stripe_size);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set %s in xattr_req dict", key);
+                goto out;
+        }
+
+        sprintf (key, "trusted.%s.stripe-count", this->name);
+        ret = dict_set_int32 (dict, key, stripe_count);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set %s in xattr_req dict", key);
+                goto out;
+        }
+
+        sprintf (key, "trusted.%s.stripe-index", this->name);
+        ret = dict_set_int32 (dict, key, stripe_index);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to set %s in xattr_req dict", key);
+                goto out;
+        }
+
+	sprintf(key, "trusted.%s.stripe-coalesce", this->name);
+	ret = dict_set_int32(dict, key, stripe_coalesce);
+	if (ret) {
+		gf_log(this->name, GF_LOG_WARNING,
+			"failed to set %s in xattr_req_dict", key);
+		goto out;
+	}
+out:
+        return ret;
+}
+
+
+static int
+set_default_block_size (stripe_private_t *priv, char *num)
+{
+
+        int             ret = -1;
+        GF_VALIDATE_OR_GOTO ("stripe", THIS, out);
+        GF_VALIDATE_OR_GOTO (THIS->name, priv, out);
+        GF_VALIDATE_OR_GOTO (THIS->name, num, out);
+
+
+        if (gf_string2bytesize_uint64 (num, &priv->block_size) != 0) {
+                gf_log (THIS->name, GF_LOG_ERROR,
+                        "invalid number format \"%s\"", num);
+                goto out;
+        }
+
+        ret = 0;
+
+ out:
+        return ret;
+
+}
+
+
+int
+set_stripe_block_size (xlator_t *this, stripe_private_t *priv, char *data)
+{
+        int                    ret = -1;
+        char                  *tmp_str = NULL;
+        char                  *tmp_str1 = NULL;
+        char                  *dup_str = NULL;
+        char                  *stripe_str = NULL;
+        char                  *pattern = NULL;
+        char                  *num = NULL;
+        struct stripe_options *temp_stripeopt = NULL;
+        struct stripe_options *stripe_opt = NULL;
+
+        if (!this || !priv || !data)
+                goto out;
+
+        /* Get the pattern for striping.
+           "option block-size *avi:10MB" etc */
+        stripe_str = strtok_r (data, ",", &tmp_str);
+        while (stripe_str) {
+                dup_str = gf_strdup (stripe_str);
+                stripe_opt = GF_CALLOC (1, sizeof (struct stripe_options),
+                                        gf_stripe_mt_stripe_options);
+                if (!stripe_opt) {
+                        goto out;
+                }
+
+                pattern = strtok_r (dup_str, ":", &tmp_str1);
+                num = strtok_r (NULL, ":", &tmp_str1);
+                if (!num) {
+                        num = pattern;
+                        pattern = "*";
+                        ret = set_default_block_size (priv, num);
+                        if (ret)
+                                goto out;
+                }
+                if (gf_string2bytesize_uint64 (num, &stripe_opt->block_size) != 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "invalid number format \"%s\"", num);
+                        goto out;
+                }
+
+                if (stripe_opt->block_size < STRIPE_MIN_BLOCK_SIZE) {
+                        gf_log (this->name, GF_LOG_ERROR, "Invalid Block-size: "
+                                "%s. Should be atleast %llu bytes", num,
+                                STRIPE_MIN_BLOCK_SIZE);
+                        goto out;
+                }
+                if (stripe_opt->block_size % 512) {
+                        gf_log (this->name, GF_LOG_ERROR, "Block-size: %s should"
+                                " be a multiple of 512 bytes", num);
+                        goto out;
+                }
+
+                memcpy (stripe_opt->path_pattern, pattern, strlen (pattern));
+
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "block-size : pattern %s : size %"PRId64,
+                        stripe_opt->path_pattern, stripe_opt->block_size);
+
+                if (priv->pattern)
+                        temp_stripeopt = NULL;
+                else
+                        temp_stripeopt = priv->pattern;
+
+                stripe_opt->next = temp_stripeopt;
+
+                priv->pattern = stripe_opt;
+                stripe_opt = NULL;
+
+                GF_FREE (dup_str);
+                dup_str = NULL;
+
+                stripe_str = strtok_r (NULL, ",", &tmp_str);
+        }
+
+        ret = 0;
+out:
+
+        GF_FREE (dup_str);
+
+        GF_FREE (stripe_opt);
+
+        return ret;
+}
+
+int32_t
+stripe_iatt_merge (struct iatt *from, struct iatt *to)
+{
+        if (to->ia_size < from->ia_size)
+                to->ia_size = from->ia_size;
+        if (to->ia_mtime < from->ia_mtime)
+                to->ia_mtime = from->ia_mtime;
+        if (to->ia_ctime < from->ia_ctime)
+                to->ia_ctime = from->ia_ctime;
+        if (to->ia_atime < from->ia_atime)
+                to->ia_atime = from->ia_atime;
+        return 0;
+}
+
+off_t
+coalesced_offset(off_t offset, uint64_t stripe_size, int stripe_count)
+{
+	size_t line_size = 0;
+	uint64_t stripe_num = 0;
+	off_t coalesced_offset = 0;
+
+	line_size = stripe_size * stripe_count;
+	stripe_num = offset / line_size;
+
+	coalesced_offset = (stripe_num * stripe_size) +
+		(offset % stripe_size);
+
+	return coalesced_offset;
+}
+
+off_t
+uncoalesced_size(off_t size, uint64_t stripe_size, int stripe_count,
+		int stripe_index)
+{
+        uint64_t nr_full_stripe_chunks = 0, mod = 0;
+
+        if (!size)
+                return size;
+
+	/*
+	 * Estimate the number of fully written stripes from the
+	 * local file size. Each stripe_size chunk corresponds to
+	 * a stripe.
+	 */
+        nr_full_stripe_chunks = (size / stripe_size) * stripe_count;
+        mod = size % stripe_size;
+
+        if (!mod) {
+                /*
+		 * There is no remainder, thus we could have overestimated
+		 * the size of the file in terms of chunks. Trim the number
+		 * of chunks by the following stripe members and leave it
+		 * up to those nodes to respond with a larger size (if
+		 * necessary).
+		 */
+                nr_full_stripe_chunks -= stripe_count -
+                        (stripe_index + 1);
+                size = nr_full_stripe_chunks * stripe_size;
+        } else {
+		/*
+		 * There is a remainder and thus we own the last chunk of the
+		 * file. Add the preceding stripe members of the final stripe
+		 * along with the remainder to calculate the exact size.
+		 */
+                nr_full_stripe_chunks += stripe_index;
+                size = nr_full_stripe_chunks * stripe_size + mod;
+        }
+
+        return size;
+}
diff --git a/xlators/cluster/stripe/src/stripe-mem-types.h b/xlators/cluster/stripe/src/stripe-mem-types.h
index 9479fdc80..e9ac9cf46 100644
--- a/xlators/cluster/stripe/src/stripe-mem-types.h
+++ b/xlators/cluster/stripe/src/stripe-mem-types.h
@@ -1,21 +1,11 @@
-
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
 
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -25,15 +15,16 @@
 #include "mem-types.h"
 
 enum gf_stripe_mem_types_ {
-        gf_stripe_mt_stripe_local_t = gf_common_mt_end + 1,
-        gf_stripe_mt_iovec,
-        gf_stripe_mt_readv_replies,
+        gf_stripe_mt_iovec = gf_common_mt_end + 1,
+        gf_stripe_mt_stripe_replies,
         gf_stripe_mt_stripe_fd_ctx_t,
         gf_stripe_mt_char,
         gf_stripe_mt_int8_t,
+        gf_stripe_mt_int32_t,
         gf_stripe_mt_xlator_t,
         gf_stripe_mt_stripe_private_t,
         gf_stripe_mt_stripe_options,
+        gf_stripe_mt_xattr_sort_t,
         gf_stripe_mt_end
 };
 #endif
diff --git a/xlators/cluster/stripe/src/stripe.c b/xlators/cluster/stripe/src/stripe.c
index d37f38ca6..0ebea8168 100644
--- a/xlators/cluster/stripe/src/stripe.c
+++ b/xlators/cluster/stripe/src/stripe.c
@@ -1,25 +1,16 @@
 /*
-  Copyright (c) 2007-2009 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 /**
  * xlators/cluster/stripe:
- *    Stripe translator, stripes the data accross its child nodes,
+ *    Stripe translator, stripes the data across its child nodes,
  *    as per the options given in the volfile. The striping works
  *    fairly simple. It writes files at different offset as per
  *    calculation. So, 'ls -l' output at the real posix level will
@@ -32,65 +23,19 @@
  *  very much necessary, or else, use it in combination with AFR, to have a
  *  backup copy.
  */
+#include <fnmatch.h>
 
 #include "stripe.h"
+#include "libxlator.h"
+#include "byte-order.h"
+#include "statedump.h"
 
-void
-stripe_local_wipe (stripe_local_t *local)
-{
-        if (!local)
-                goto out;
-
-        if (local->loc.path)
-                loc_wipe (&local->loc);
-        if (local->loc2.path)
-                loc_wipe (&local->loc2);
-out:
-        return;
-}
-
-/**
- * stripe_get_matching_bs - Get the matching block size for the given path.
- */
-int32_t
-stripe_get_matching_bs (const char *path, struct stripe_options *opts,
-                        uint64_t default_bs)
-{
-        struct stripe_options *trav       = NULL;
-        char                  *pathname   = NULL;
-        uint64_t               block_size = 0;
-
-        block_size = default_bs;
-
-        if (!path || !opts)
-                goto out;
-
-        /* FIXME: is a strdup really necessary? */
-        pathname = gf_strdup (path);
-        if (!pathname)
-                goto out;
-
-        trav = opts;
-        while (trav) {
-                if (!fnmatch (trav->path_pattern, pathname, FNM_NOESCAPE)) {
-                        block_size = trav->block_size;
-                        break;
-                }
-                trav = trav->next;
-        }
-
-        GF_FREE (pathname);
-
-out:
-        return block_size;
-}
-
-
+struct volume_options options[];
 
 int32_t
 stripe_sh_chown_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
-                     struct iatt *preop, struct iatt *postop)
+                     struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
         int             callcnt = -1;
         stripe_local_t *local   = NULL;
@@ -109,8 +54,7 @@ stripe_sh_chown_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         UNLOCK (&frame->lock);
 
         if (!callcnt) {
-                stripe_local_wipe (local);
-                STACK_DESTROY (frame->root);
+                STRIPE_STACK_DESTROY (frame);
         }
 out:
         return 0;
@@ -120,7 +64,7 @@ int32_t
 stripe_sh_make_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                           int32_t op_ret, int32_t op_errno, inode_t *inode,
                           struct iatt *buf, struct iatt *preparent,
-                          struct iatt *postparent)
+                          struct iatt *postparent, dict_t *xdata)
 {
         stripe_local_t *local = NULL;
         call_frame_t    *prev = NULL;
@@ -135,7 +79,7 @@ stripe_sh_make_entry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         STACK_WIND (frame, stripe_sh_chown_cbk, prev->this,
                     prev->this->fops->setattr, &local->loc,
-                    &local->stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID));
+                    &local->stbuf, (GF_SET_ATTR_UID | GF_SET_ATTR_GID), NULL);
 
 out:
         return 0;
@@ -149,7 +93,7 @@ stripe_entry_self_heal (call_frame_t *frame, xlator_t *this,
         call_frame_t     *rframe = NULL;
         stripe_local_t   *rlocal = NULL;
         stripe_private_t *priv   = NULL;
-        dict_t           *dict   = NULL;
+        dict_t           *xdata  = NULL;
         int               ret    = 0;
 
         if (!local || !this || !frame) {
@@ -167,8 +111,7 @@ stripe_entry_self_heal (call_frame_t *frame, xlator_t *this,
         if (!rframe) {
                 goto out;
         }
-        rlocal = GF_CALLOC (1, sizeof (stripe_local_t),
-                            gf_stripe_mt_stripe_local_t);
+        rlocal = mem_get0 (this->local_pool);
         if (!rlocal) {
                 goto out;
         }
@@ -177,14 +120,14 @@ stripe_entry_self_heal (call_frame_t *frame, xlator_t *this,
         loc_copy (&rlocal->loc, &local->loc);
         memcpy (&rlocal->stbuf, &local->stbuf, sizeof (struct iatt));
 
-        dict = dict_new ();
-        if (!dict)
+        xdata = dict_new ();
+        if (!xdata)
                 goto out;
 
-        ret = dict_set_static_bin (dict, "gfid-req", local->stbuf.ia_gfid, 16);
+        ret = dict_set_static_bin (xdata, "gfid-req", local->stbuf.ia_gfid, 16);
         if (ret)
                 gf_log (this->name, GF_LOG_WARNING,
-                        "failed to set gfid-req");
+                        "%s: failed to set gfid-req", local->loc.path);
 
         while (trav) {
                 if (IA_ISREG (local->stbuf.ia_type)) {
@@ -192,36 +135,43 @@ stripe_entry_self_heal (call_frame_t *frame, xlator_t *this,
                                     trav->xlator, trav->xlator->fops->mknod,
                                     &local->loc,
                                     st_mode_from_ia (local->stbuf.ia_prot,
-                                                     local->stbuf.ia_type), 0,
-                                    dict);
+                                                     local->stbuf.ia_type),
+                                    0, 0, xdata);
                 }
                 if (IA_ISDIR (local->stbuf.ia_type)) {
                         STACK_WIND (rframe, stripe_sh_make_entry_cbk,
                                     trav->xlator, trav->xlator->fops->mkdir,
-                                    &local->loc, st_mode_from_ia (local->stbuf.ia_prot,
-                                                                  local->stbuf.ia_type),
-                                    dict);
+                                    &local->loc,
+                                    st_mode_from_ia (local->stbuf.ia_prot,
+                                                     local->stbuf.ia_type),
+                                    0, xdata);
                 }
                 trav = trav->next;
         }
 
+        if (xdata)
+                dict_unref (xdata);
+        return 0;
+
 out:
-        if (dict)
-                dict_unref (dict);
+        if (rframe)
+                STRIPE_STACK_DESTROY (rframe);
+        if (xdata)
+                dict_unref (xdata);
 
         return 0;
 }
 
+
 int32_t
 stripe_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, inode_t *inode,
-                   struct iatt *buf, dict_t *dict, struct iatt *postparent)
+                   struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
-        int32_t         callcnt = 0;
-        dict_t         *tmp_dict = NULL;
-        inode_t        *tmp_inode = NULL;
-        stripe_local_t *local = NULL;
-        call_frame_t   *prev = NULL;
+        int32_t         callcnt     = 0;
+        stripe_local_t *local       = NULL;
+        call_frame_t   *prev        = NULL;
+        int             ret         = 0;
 
         if (!this || !frame || !frame->local || !cookie) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -243,7 +193,7 @@ stripe_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                         strerror (op_errno));
                         if (local->op_errno != ESTALE)
                                 local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
+                        if (((op_errno != ENOENT) && (op_errno != ENOTCONN)) ||
                             (prev->this == FIRST_CHILD (this)))
                                 local->failed = 1;
                         if (op_errno == ENOENT)
@@ -252,51 +202,81 @@ stripe_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
                 if (op_ret >= 0) {
                         local->op_ret = 0;
+                        if (IA_ISREG (buf->ia_type)) {
+                                ret = stripe_ctx_handle (this, prev, local,
+                                                         xdata);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                 "Error getting fctx info from"
+                                                 " dict");
+                        }
 
                         if (FIRST_CHILD(this) == prev->this) {
                                 local->stbuf      = *buf;
                                 local->postparent = *postparent;
                                 local->inode = inode_ref (inode);
-                                local->dict = dict_ref (dict);
+                                if (xdata)
+                                        local->xdata = dict_ref (xdata);
+                                if (local->xattr) {
+                                        stripe_aggregate_xattr (local->xdata,
+                                                                local->xattr);
+                                        dict_unref (local->xattr);
+                                        local->xattr = NULL;
+                                }
                         }
+
+                        if (!local->xdata && !local->xattr) {
+                                local->xattr = dict_ref (xdata);
+                        } else if (local->xdata) {
+                                stripe_aggregate_xattr (local->xdata, xdata);
+                        } else if (local->xattr) {
+                                stripe_aggregate_xattr (local->xattr, xdata);
+                        }
+
                         local->stbuf_blocks      += buf->ia_blocks;
                         local->postparent_blocks += postparent->ia_blocks;
 
+			correct_file_size(buf, local->fctx, prev);
+
                         if (local->stbuf_size < buf->ia_size)
                                 local->stbuf_size = buf->ia_size;
                         if (local->postparent_size < postparent->ia_size)
                                 local->postparent_size = postparent->ia_size;
+
+                        if (uuid_is_null (local->ia_gfid))
+                                uuid_copy (local->ia_gfid, buf->ia_gfid);
+
+                        /* Make sure the gfid on all the nodes are same */
+                        if (uuid_compare (local->ia_gfid, buf->ia_gfid)) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "%s: gfid different on subvolume %s",
+                                        local->loc.path, prev->this->name);
+                        }
                 }
         }
         UNLOCK (&frame->lock);
 
         if (!callcnt) {
-                if (local->op_ret == 0 && local->entry_self_heal_needed)
+                if (local->op_ret == 0 && local->entry_self_heal_needed &&
+                    !uuid_is_null (local->loc.inode->gfid))
                         stripe_entry_self_heal (frame, this, local);
 
                 if (local->failed)
                         local->op_ret = -1;
 
-                tmp_dict  = local->dict;
-                tmp_inode = local->inode;
-
                 if (local->op_ret != -1) {
                         local->stbuf.ia_blocks      = local->stbuf_blocks;
                         local->stbuf.ia_size        = local->stbuf_size;
                         local->postparent.ia_blocks = local->postparent_blocks;
                         local->postparent.ia_size   = local->postparent_size;
+                        inode_ctx_put (local->inode, this,
+                                       (uint64_t) (long)local->fctx);
                 }
 
-                stripe_local_wipe (local);
-                STACK_UNWIND_STRICT (lookup, frame, local->op_ret,
+                STRIPE_STACK_UNWIND (lookup, frame, local->op_ret,
                                      local->op_errno, local->inode,
-                                     &local->stbuf, local->dict,
+                                     &local->stbuf, local->xdata,
                                      &local->postparent);
-
-                if (tmp_inode)
-                        inode_unref (tmp_inode);
-                if (tmp_dict)
-                        dict_unref (tmp_dict);
         }
 out:
         return 0;
@@ -304,12 +284,15 @@ out:
 
 int32_t
 stripe_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               dict_t *xattr_req)
+               dict_t *xdata)
 {
-        stripe_local_t   *local = NULL;
-        xlator_list_t    *trav = NULL;
-        stripe_private_t *priv = NULL;
+        stripe_local_t   *local    = NULL;
+        xlator_list_t    *trav     = NULL;
+        stripe_private_t *priv     = NULL;
         int32_t           op_errno = EINVAL;
+        int64_t           filesize = 0;
+        int               ret      = 0;
+        uint64_t          tmpctx   = 0;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -321,8 +304,7 @@ stripe_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
         trav = this->children;
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -331,26 +313,60 @@ stripe_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
         frame->local = local;
         loc_copy (&local->loc, loc);
 
+        inode_ctx_get (local->inode, this, &tmpctx);
+        if (tmpctx)
+                local->fctx = (stripe_fd_ctx_t*) (long)tmpctx;
+
+        /* quick-read friendly changes */
+        if (xdata && dict_get (xdata, GF_CONTENT_KEY)) {
+                ret = dict_get_int64 (xdata, GF_CONTENT_KEY, &filesize);
+                if (!ret && (filesize > priv->block_size))
+                        dict_del (xdata, GF_CONTENT_KEY);
+        }
+
+        /* get stripe-size xattr on lookup. This would be required for
+         * open/read/write/pathinfo calls. Hence we send down the request
+         * even when type == IA_INVAL */
+
+	/*
+	 * We aren't guaranteed to have xdata here. We need the format info for
+	 * the file, so allocate xdata if necessary.
+	 */
+	if (!xdata)
+		xdata = dict_new();
+	else
+		xdata = dict_ref(xdata);
+
+        if (xdata && (IA_ISREG (loc->inode->ia_type) ||
+            (loc->inode->ia_type == IA_INVAL))) {
+                ret = stripe_xattr_request_build (this, xdata, 8, 4, 4, 0);
+                if (ret)
+                        gf_log (this->name , GF_LOG_ERROR, "Failed to build"
+                                " xattr request for %s", loc->path);
+
+        }
+
         /* Everytime in stripe lookup, all child nodes
            should be looked up */
         local->call_count = priv->child_count;
         while (trav) {
                 STACK_WIND (frame, stripe_lookup_cbk, trav->xlator,
-                            trav->xlator->fops->lookup,
-                            loc, xattr_req);
+                            trav->xlator->fops->lookup, loc, xdata);
                 trav = trav->next;
         }
 
+	dict_unref(xdata);
+
         return 0;
 err:
-        STACK_UNWIND_STRICT (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        STRIPE_STACK_UNWIND (lookup, frame, -1, op_errno, NULL, NULL, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 stripe_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -385,6 +401,9 @@ stripe_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         }
 
                         local->stbuf_blocks += buf->ia_blocks;
+
+			correct_file_size(buf, local->fctx, prev);
+
                         if (local->stbuf_size < buf->ia_size)
                                 local->stbuf_size = buf->ia_size;
                 }
@@ -400,20 +419,20 @@ stripe_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->stbuf.ia_blocks = local->stbuf_blocks;
                 }
 
-                stripe_local_wipe (local);
-                STACK_UNWIND_STRICT (stat, frame, local->op_ret,
-                                     local->op_errno, &local->stbuf);
+                STRIPE_STACK_UNWIND (stat, frame, local->op_ret,
+                                     local->op_errno, &local->stbuf, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+stripe_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
+	stripe_fd_ctx_t  *fctx = NULL;
         int32_t           op_errno = EINVAL;
 
         VALIDATE_OR_GOTO (frame, err);
@@ -431,8 +450,7 @@ stripe_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -441,23 +459,30 @@ stripe_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
         frame->local = local;
         local->call_count = priv->child_count;
 
+	if (IA_ISREG(loc->inode->ia_type)) {
+		inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
+		if (!fctx)
+			goto err;
+		local->fctx = fctx;
+	}
+
         while (trav) {
                 STACK_WIND (frame, stripe_stat_cbk, trav->xlator,
-                            trav->xlator->fops->stat, loc);
+                            trav->xlator->fops->stat, loc, NULL);
                 trav = trav->next;
         }
 
         return 0;
 
 err:
-        STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (stat, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 stripe_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct statvfs *stbuf)
+                   int32_t op_ret, int32_t op_errno, struct statvfs *stbuf, dict_t *xdata)
 {
         stripe_local_t *local = NULL;
         int32_t         callcnt = 0;
@@ -494,15 +519,15 @@ stripe_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         UNLOCK (&frame->lock);
 
         if (!callcnt) {
-                STACK_UNWIND_STRICT (statfs, frame, local->op_ret,
-                                     local->op_errno, &local->statvfs_buf);
+                STRIPE_STACK_UNWIND (statfs, frame, local->op_ret,
+                                     local->op_errno, &local->statvfs_buf, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+stripe_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         xlator_list_t    *trav = NULL;
@@ -517,8 +542,7 @@ stripe_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
         priv = this->private;
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -530,13 +554,13 @@ stripe_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
         local->call_count = priv->child_count;
         while (trav) {
                 STACK_WIND (frame, stripe_statfs_cbk, trav->xlator,
-                            trav->xlator->fops->statfs, loc);
+                            trav->xlator->fops->statfs, loc, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STACK_UNWIND_STRICT (statfs, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (statfs, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
@@ -545,7 +569,7 @@ err:
 int32_t
 stripe_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf)
+                     struct iatt *postbuf, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -583,6 +607,9 @@ stripe_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->prebuf_blocks  += prebuf->ia_blocks;
                         local->postbuf_blocks += postbuf->ia_blocks;
 
+			correct_file_size(prebuf, local->fctx, prev);
+			correct_file_size(postbuf, local->fctx, prev);
+
                         if (local->prebuf_size < prebuf->ia_size)
                                 local->prebuf_size = prebuf->ia_size;
 
@@ -603,22 +630,23 @@ stripe_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->post_buf.ia_size   = local->postbuf_size;
                 }
 
-                stripe_local_wipe (local);
-                STACK_UNWIND_STRICT (truncate, frame, local->op_ret,
+                STRIPE_STACK_UNWIND (truncate, frame, local->op_ret,
                                      local->op_errno, &local->pre_buf,
-                                     &local->post_buf);
+                                     &local->post_buf, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
+stripe_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata)
 {
-        xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
+	stripe_fd_ctx_t  *fctx = NULL;
         int32_t           op_errno = EINVAL;
+	int		  i, eof_idx;
+	off_t		  dest_offset, tmp_offset;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -627,7 +655,6 @@ stripe_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
         VALIDATE_OR_GOTO (loc->inode, err);
 
         priv = this->private;
-        trav = this->children;
 
         if (priv->first_child_down) {
                 op_errno = ENOTCONN;
@@ -635,8 +662,7 @@ stripe_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -645,15 +671,55 @@ stripe_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
         frame->local = local;
         local->call_count = priv->child_count;
 
-        while (trav) {
-                STACK_WIND (frame, stripe_truncate_cbk, trav->xlator,
-                            trav->xlator->fops->truncate, loc, offset);
-                trav = trav->next;
-        }
+	inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
+	if (!fctx) {
+		gf_log(this->name, GF_LOG_ERROR, "no stripe context");
+		op_errno = EINVAL;
+		goto err;
+	}
+
+	local->fctx = fctx;
+	eof_idx = (offset / fctx->stripe_size) % fctx->stripe_count;
+
+	for (i = 0; i < fctx->stripe_count; i++) {
+		if (!fctx->xl_array[i]) {
+			gf_log(this->name, GF_LOG_ERROR,
+				"no xlator at index %d", i);
+			op_errno = EINVAL;
+			goto err;
+		}
+
+		if (fctx->stripe_coalesce) {
+			/*
+	 		 * The node that owns EOF is truncated to the exact
+	 		 * coalesced offset. Nodes prior to this index should
+	 		 * be rounded up to the size of the complete stripe,
+	 		 * while nodes after this index should be rounded down
+			 * to the size of the previous stripe.
+			 */
+			if (i < eof_idx)
+				tmp_offset = roof(offset, fctx->stripe_size *
+						fctx->stripe_count);
+			else if (i > eof_idx)
+				tmp_offset = floor(offset, fctx->stripe_size *
+						fctx->stripe_count);
+			else
+				tmp_offset = offset;
+
+			dest_offset = coalesced_offset(tmp_offset,
+					fctx->stripe_size, fctx->stripe_count);
+		} else {
+			dest_offset = offset;
+		}
+
+		STACK_WIND(frame, stripe_truncate_cbk, fctx->xl_array[i],
+			fctx->xl_array[i]->fops->truncate, loc, dest_offset,
+			NULL);
+	}
 
         return 0;
 err:
-        STACK_UNWIND_STRICT (truncate, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (truncate, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -661,7 +727,7 @@ err:
 int32_t
 stripe_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
-                    struct iatt *preop, struct iatt *postop)
+                    struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -700,6 +766,9 @@ stripe_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->prebuf_blocks  += preop->ia_blocks;
                         local->postbuf_blocks += postop->ia_blocks;
 
+			correct_file_size(preop, local->fctx, prev);
+			correct_file_size(postop, local->fctx, prev);
+
                         if (local->prebuf_size < preop->ia_size)
                                 local->prebuf_size = preop->ia_size;
                         if (local->postbuf_size < postop->ia_size)
@@ -719,10 +788,9 @@ stripe_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->post_buf.ia_size   = local->postbuf_size;
                 }
 
-                stripe_local_wipe (local);
-                STACK_UNWIND_STRICT (setattr, frame, local->op_ret,
+                STRIPE_STACK_UNWIND (setattr, frame, local->op_ret,
                                      local->op_errno, &local->pre_buf,
-                                     &local->post_buf);
+                                     &local->post_buf, NULL);
         }
 out:
         return 0;
@@ -731,11 +799,12 @@ out:
 
 int32_t
 stripe_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                struct iatt *stbuf, int32_t valid)
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
+	stripe_fd_ctx_t	 *fctx = NULL;
         int32_t           op_errno = EINVAL;
 
         VALIDATE_OR_GOTO (frame, err);
@@ -753,33 +822,47 @@ stripe_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
         }
         local->op_ret = -1;
         frame->local = local;
-        local->call_count = priv->child_count;
+        if (!IA_ISDIR (loc->inode->ia_type) &&
+            !IA_ISREG (loc->inode->ia_type)) {
+                local->call_count = 1;
+                STACK_WIND (frame, stripe_setattr_cbk, FIRST_CHILD (this),
+                            FIRST_CHILD (this)->fops->setattr,
+                            loc, stbuf, valid, NULL);
+                return 0;
+        }
 
+	if (IA_ISREG(loc->inode->ia_type)) {
+		inode_ctx_get(loc->inode, this, (uint64_t *) &fctx);
+		if (!fctx)
+			goto err;
+		local->fctx = fctx;
+	}
+
+        local->call_count = priv->child_count;
         while (trav) {
                 STACK_WIND (frame, stripe_setattr_cbk,
                             trav->xlator, trav->xlator->fops->setattr,
-                            loc, stbuf, valid);
+                            loc, stbuf, valid, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STACK_UNWIND_STRICT (setattr, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (setattr, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 stripe_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                 struct iatt *stbuf, int32_t valid)
+                 struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
@@ -795,8 +878,7 @@ stripe_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
         trav = this->children;
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -807,13 +889,13 @@ stripe_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         while (trav) {
                 STACK_WIND (frame, stripe_setattr_cbk, trav->xlator,
-                            trav->xlator->fops->fsetattr, fd, stbuf, valid);
+                            trav->xlator->fops->fsetattr, fd, stbuf, valid, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STACK_UNWIND_STRICT (fsetattr, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (fsetattr, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -821,7 +903,8 @@ int32_t
 stripe_stack_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
                          struct iatt *preoldparent, struct iatt *postoldparent,
-                         struct iatt *prenewparent, struct iatt *postnewparent)
+                         struct iatt *prenewparent, struct iatt *postnewparent,
+                         dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -858,6 +941,8 @@ stripe_stack_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->pre_buf.ia_blocks    += prenewparent->ia_blocks;
                         local->post_buf.ia_blocks   += postnewparent->ia_blocks;
 
+			correct_file_size(buf, local->fctx, prev);
+
                         if (local->stbuf.ia_size < buf->ia_size)
                                 local->stbuf.ia_size =  buf->ia_size;
 
@@ -880,11 +965,10 @@ stripe_stack_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (local->failed)
                         local->op_ret = -1;
 
-                stripe_local_wipe (local);
-                STACK_UNWIND_STRICT (rename, frame, local->op_ret, local->op_errno,
+                STRIPE_STACK_UNWIND (rename, frame, local->op_ret, local->op_errno,
                                      &local->stbuf, &local->preparent,
                                      &local->postparent,  &local->pre_buf,
-                                     &local->post_buf);
+                                     &local->post_buf, NULL);
         }
 out:
         return 0;
@@ -894,7 +978,8 @@ int32_t
 stripe_first_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int32_t op_ret, int32_t op_errno, struct iatt *buf,
                          struct iatt *preoldparent, struct iatt *postoldparent,
-                         struct iatt *prenewparent, struct iatt *postnewparent)
+                         struct iatt *prenewparent, struct iatt *postnewparent,
+                         dict_t *xdata)
 {
         stripe_local_t *local = NULL;
         xlator_list_t  *trav = NULL;
@@ -925,25 +1010,25 @@ stripe_first_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         while (trav) {
                 STACK_WIND (frame, stripe_stack_rename_cbk,
                             trav->xlator, trav->xlator->fops->rename,
-                            &local->loc, &local->loc2);
+                            &local->loc, &local->loc2, NULL);
                 trav = trav->next;
         }
         return 0;
 
 unwind:
-        stripe_local_wipe (local);
-        STACK_UNWIND_STRICT (rename, frame, -1, op_errno, buf, preoldparent,
-                             postoldparent, prenewparent, postnewparent);
+        STRIPE_STACK_UNWIND (rename, frame, -1, op_errno, buf, preoldparent,
+                             postoldparent, prenewparent, postnewparent, NULL);
         return 0;
 }
 
 int32_t
 stripe_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-               loc_t *newloc)
+               loc_t *newloc, dict_t *xdata)
 {
         stripe_private_t *priv = NULL;
         stripe_local_t   *local = NULL;
         xlator_list_t    *trav = NULL;
+	stripe_fd_ctx_t	 *fctx = NULL;
         int32_t           op_errno = EINVAL;
 
         VALIDATE_OR_GOTO (frame, err);
@@ -963,36 +1048,79 @@ stripe_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
         }
+
+        frame->local = local;
+
         local->op_ret = -1;
         loc_copy (&local->loc, oldloc);
         loc_copy (&local->loc2, newloc);
 
         local->call_count = priv->child_count;
 
-        frame->local = local;
+	if (IA_ISREG(oldloc->inode->ia_type)) {
+		inode_ctx_get(oldloc->inode, this, (uint64_t *) &fctx);
+		if (!fctx)
+			goto err;
+		local->fctx = fctx;
+	}
 
         STACK_WIND (frame, stripe_first_rename_cbk, trav->xlator,
-                    trav->xlator->fops->rename, oldloc, newloc);
+                    trav->xlator->fops->rename, oldloc, newloc, NULL);
 
         return 0;
 err:
-        STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL);
+        STRIPE_STACK_UNWIND (rename, frame, -1, op_errno, NULL, NULL, NULL,
+                             NULL, NULL, NULL);
+        return 0;
+}
+int32_t
+stripe_first_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                         struct iatt *postparent, dict_t *xdata)
+{
+        stripe_local_t *local   = NULL;
+        call_frame_t   *prev = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG, "%s returned %s",
+                        prev->this->name, strerror (op_errno));
+                goto out;
+        }
+        local->op_ret = 0;
+        local->preparent  = *preparent;
+        local->postparent = *postparent;
+        local->preparent_blocks  += preparent->ia_blocks;
+        local->postparent_blocks += postparent->ia_blocks;
+
+        STRIPE_STACK_UNWIND(unlink, frame, local->op_ret, local->op_errno,
+                            &local->preparent, &local->postparent, xdata);
+        return 0;
+out:
+        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
+
         return 0;
 }
 
 
 
+
 int32_t
 stripe_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local   = NULL;
@@ -1014,50 +1142,33 @@ stripe_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         gf_log (this->name, GF_LOG_DEBUG, "%s returned %s",
                                 prev->this->name, strerror (op_errno));
                         local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
+                        if (op_errno != ENOENT) {
                                 local->failed = 1;
-                }
-                if (op_ret >= 0) {
-                        local->op_ret = op_ret;
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->preparent  = *preparent;
-                                local->postparent = *postparent;
+                                local->op_ret = op_ret;
                         }
-                        local->preparent_blocks  += preparent->ia_blocks;
-                        local->postparent_blocks += postparent->ia_blocks;
-
-                        if (local->preparent_size < preparent->ia_size)
-                                local->preparent_size = preparent->ia_size;
-
-                        if (local->postparent_size < postparent->ia_size)
-                                local->postparent_size = postparent->ia_size;
                 }
         }
         UNLOCK (&frame->lock);
 
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->preparent.ia_blocks  = local->preparent_blocks;
-                        local->preparent.ia_size    = local->preparent_size;
-                        local->postparent.ia_blocks = local->postparent_blocks;
-                        local->postparent.ia_size   = local->postparent_size;
+        if (callcnt == 1) {
+                if (local->failed) {
+                        op_errno = local->op_errno;
+                        goto out;
                 }
-
-                stripe_local_wipe (local);
-                STACK_UNWIND_STRICT (unlink, frame, local->op_ret,
-                                     local->op_errno, &local->preparent,
-                                     &local->postparent);
+                STACK_WIND(frame, stripe_first_unlink_cbk, FIRST_CHILD (this),
+                           FIRST_CHILD (this)->fops->unlink, &local->loc,
+                           local->xflag, local->xdata);
         }
+        return 0;
 out:
+        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
+
         return 0;
 }
 
 int32_t
-stripe_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+stripe_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               int xflag, dict_t *xdata)
 {
         xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
@@ -1085,26 +1196,32 @@ stripe_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
         }
         local->op_ret = -1;
+        loc_copy (&local->loc, loc);
+        local->xflag = xflag;
+
+        if (xdata)
+            local->xdata = dict_ref (xdata);
+
         frame->local = local;
         local->call_count = priv->child_count;
+        trav = trav->next; /* Skip the first child */
 
         while (trav) {
                 STACK_WIND (frame, stripe_unlink_cbk,
                             trav->xlator, trav->xlator->fops->unlink,
-                            loc);
+                            loc, xflag, xdata);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (unlink, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1112,10 +1229,8 @@ err:
 int32_t
 stripe_first_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno,struct iatt *preparent,
-                        struct iatt *postparent)
-
+                        struct iatt *postparent, dict_t *xdata)
 {
-        xlator_list_t *trav = NULL;
         stripe_local_t *local = NULL;
 
         if (!this || !frame || !frame->local) {
@@ -1128,11 +1243,10 @@ stripe_first_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 goto err;
         }
 
-        trav = this->children;
         local = frame->local;
+        local->op_ret = 0;
 
         local->call_count--; /* First child successful */
-        trav = trav->next; /* Skip first child */
 
         local->preparent  = *preparent;
         local->postparent = *postparent;
@@ -1141,22 +1255,60 @@ stripe_first_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         local->preparent_blocks  += preparent->ia_blocks;
         local->postparent_blocks += postparent->ia_blocks;
 
-        while (trav) {
-                STACK_WIND (frame, stripe_unlink_cbk, trav->xlator,
-                            trav->xlator->fops->rmdir, &local->loc,
-                            local->flags);
-                trav = trav->next;
-        }
-
+        STRIPE_STACK_UNWIND (rmdir, frame, local->op_ret, local->op_errno,
+                             &local->preparent, &local->postparent, xdata);
         return 0;
 err:
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (rmdir, frame, op_ret, op_errno, NULL, NULL, NULL);
         return 0;
 
 }
 
 int32_t
-stripe_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+stripe_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local   = NULL;
+        call_frame_t   *prev = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->call_count;
+
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_DEBUG, "%s returned %s",
+                                prev->this->name, strerror (op_errno));
+                        if (op_errno != ENOENT)
+                                local->failed = 1;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if (callcnt == 1) {
+                if (local->failed)
+                        goto out;
+                STACK_WIND (frame, stripe_first_rmdir_cbk, FIRST_CHILD (this),
+                            FIRST_CHILD (this)->fops->rmdir, &local->loc,
+                            local->flags, NULL);
+        }
+        return 0;
+out:
+        STRIPE_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int32_t
+stripe_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags, dict_t *xdata)
 {
         xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
@@ -1179,8 +1331,7 @@ stripe_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -1190,13 +1341,17 @@ stripe_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
         loc_copy (&local->loc, loc);
         local->flags = flags;
         local->call_count = priv->child_count;
+        trav = trav->next; /* skip the first child */
 
-        STACK_WIND (frame, stripe_first_rmdir_cbk,  trav->xlator,
-                    trav->xlator->fops->rmdir, loc, flags);
+        while (trav) {
+                STACK_WIND (frame, stripe_rmdir_cbk,  trav->xlator,
+                            trav->xlator->fops->rmdir, loc, flags, NULL);
+                trav = trav->next;
+        }
 
         return 0;
 err:
-        STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (rmdir, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1205,7 +1360,7 @@ int32_t
 stripe_mknod_ifreg_fail_unlink_cbk (call_frame_t *frame, void *cookie,
                                     xlator_t *this, int32_t op_ret,
                                     int32_t op_errno, struct iatt *preparent,
-                                    struct iatt *postparent)
+                                    struct iatt *postparent, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -1224,10 +1379,9 @@ stripe_mknod_ifreg_fail_unlink_cbk (call_frame_t *frame, void *cookie,
         UNLOCK (&frame->lock);
 
         if (!callcnt) {
-                stripe_local_wipe (local);
-                STACK_UNWIND_STRICT (mknod, frame, local->op_ret, local->op_errno,
+                STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
                                      local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent);
+                                     &local->preparent, &local->postparent, NULL);
         }
 out:
         return 0;
@@ -1239,7 +1393,7 @@ out:
 int32_t
 stripe_mknod_ifreg_setxattr_cbk (call_frame_t *frame, void *cookie,
                                  xlator_t *this, int32_t op_ret,
-                                 int32_t op_errno)
+                                 int32_t op_errno, dict_t *xdata)
 {
         int32_t           callcnt = 0;
         stripe_local_t   *local = NULL;
@@ -1278,16 +1432,15 @@ stripe_mknod_ifreg_setxattr_cbk (call_frame_t *frame, void *cookie,
                                             stripe_mknod_ifreg_fail_unlink_cbk,
                                             trav->xlator,
                                             trav->xlator->fops->unlink,
-                                            &local->loc);
+                                            &local->loc, 0, NULL);
                                 trav = trav->next;
                         }
                         return 0;
                 }
 
-                stripe_local_wipe (local);
-                STACK_UNWIND_STRICT (mknod, frame, local->op_ret, local->op_errno,
+                STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
                                      local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent);
+                                     &local->preparent, &local->postparent, NULL);
         }
 out:
         return 0;
@@ -1297,13 +1450,13 @@ int32_t
 stripe_mknod_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno, inode_t *inode,
                         struct iatt *buf, struct iatt *preparent,
-                        struct iatt *postparent)
+                        struct iatt *postparent, dict_t *xdata)
 {
-        int               ret = 0;
         int32_t           callcnt = 0;
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
         call_frame_t     *prev = NULL;
+        xlator_list_t    *trav = NULL;
 
         if (!this || !frame || !frame->local || !cookie) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -1311,7 +1464,7 @@ stripe_mknod_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         prev  = cookie;
-	priv  = this->private;
+        priv  = this->private;
         local = frame->local;
 
         LOCK (&frame->lock);
@@ -1327,20 +1480,24 @@ stripe_mknod_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                 local->failed = 1;
                         local->op_errno = op_errno;
                 }
-
                 if (op_ret >= 0) {
                         local->op_ret = op_ret;
 
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->stbuf      = *buf;
-                                local->preparent  = *preparent;
-                                local->postparent = *postparent;
-                        }
+                        /* Can be used as a mechanism to understand if mknod
+                           was successful in at least one place */
+                        if (uuid_is_null (local->ia_gfid))
+                                uuid_copy (local->ia_gfid, buf->ia_gfid);
+
+			if (stripe_ctx_handle(this, prev, local, xdata))
+				gf_log(this->name, GF_LOG_ERROR,
+					"Error getting fctx info from dict");
 
                         local->stbuf_blocks += buf->ia_blocks;
                         local->preparent_blocks  += preparent->ia_blocks;
                         local->postparent_blocks += postparent->ia_blocks;
 
+			correct_file_size(buf, local->fctx, prev);
+
                         if (local->stbuf_size < buf->ia_size)
                                 local->stbuf_size = buf->ia_size;
                         if (local->preparent_size < preparent->ia_size)
@@ -1355,6 +1512,23 @@ stripe_mknod_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (local->failed)
                         local->op_ret = -1;
 
+                if ((local->op_ret == -1) && !uuid_is_null (local->ia_gfid)) {
+                        /* ia_gfid set means, at least on one node 'mknod'
+                           is successful */
+                        local->call_count = priv->child_count;
+                        trav = this->children;
+                        while (trav) {
+                                STACK_WIND (frame,
+                                            stripe_mknod_ifreg_fail_unlink_cbk,
+                                            trav->xlator,
+                                            trav->xlator->fops->unlink,
+                                            &local->loc, 0, NULL);
+                                trav = trav->next;
+                        }
+                        return 0;
+                }
+
+
                 if (local->op_ret != -1) {
                         local->preparent.ia_blocks  = local->preparent_blocks;
                         local->preparent.ia_size    = local->preparent_size;
@@ -1362,73 +1536,109 @@ stripe_mknod_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->postparent.ia_size   = local->postparent_size;
                         local->stbuf.ia_size        = local->stbuf_size;
                         local->stbuf.ia_blocks      = local->stbuf_blocks;
+                        inode_ctx_put (local->inode, this,
+                                       (uint64_t)(long) local->fctx);
+
                 }
+                STRIPE_STACK_UNWIND (mknod, frame, local->op_ret, local->op_errno,
+                                     local->inode, &local->stbuf,
+                                     &local->preparent, &local->postparent, NULL);
+        }
+out:
+        return 0;
+}
 
-                if ((local->op_ret != -1) && priv->xattr_supported) {
-                        /* Send a setxattr request to nodes where the
-                           files are created */
-                        int32_t  i              = 0;
-                        char     size_key[256]  = {0,};
-                        char     index_key[256] = {0,};
-                        char     count_key[256] = {0,};
-                        dict_t  *dict           = NULL;
 
-                        sprintf (size_key,
-                                 "trusted.%s.stripe-size", this->name);
-                        sprintf (count_key,
-                                 "trusted.%s.stripe-count", this->name);
-                        sprintf (index_key,
-                                 "trusted.%s.stripe-index", this->name);
+int32_t
+stripe_mknod_first_ifreg_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, inode_t *inode,
+                        struct iatt *buf, struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv = NULL;
+        call_frame_t     *prev = NULL;
+        xlator_list_t    *trav = NULL;
+        int               i    = 1;
+        dict_t           *dict           = NULL;
+        int               ret            = 0;
+        int               need_unref     = 0;
 
-                        local->call_count = priv->child_count;
-                        memcpy (local->loc.inode->gfid, local->stbuf.ia_gfid, 16);
-                        for (i = 0; i < priv->child_count; i++) {
-                                dict = get_new_dict ();
-                                if (!dict) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "failed to allocate dict");
-                                }
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
 
-                                dict_ref (dict);
-                                /* TODO: check return value */
-                                ret = dict_set_int64 (dict, size_key,
-                                                      local->stripe_size);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "%s: set stripe-size failed",
-                                                local->loc.path);
-                                ret = dict_set_int32 (dict, count_key,
-                                                      priv->child_count);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "%s: set child_count failed",
-                                                local->loc.path);
-                                ret = dict_set_int32 (dict, index_key, i);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "%s: set stripe-index failed",
-                                                local->loc.path);
+        prev  = cookie;
+        priv  = this->private;
+        local = frame->local;
+        trav = this->children;
 
-                                STACK_WIND (frame,
-                                            stripe_mknod_ifreg_setxattr_cbk,
-                                            priv->xl_array[i],
-                                            priv->xl_array[i]->fops->setxattr,
-                                            &local->loc, dict, 0);
+        local->call_count--;
 
-                                dict_unref (dict);
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG, "%s returned error %s",
+                        prev->this->name, strerror (op_errno));
+                local->failed = 1;
+                local->op_errno = op_errno;
+                goto out;
+        }
+
+        local->op_ret = op_ret;
+
+        local->stbuf      = *buf;
+        local->preparent  = *preparent;
+        local->postparent = *postparent;
+
+        if (uuid_is_null (local->ia_gfid))
+                uuid_copy (local->ia_gfid, buf->ia_gfid);
+        local->preparent.ia_blocks  = local->preparent_blocks;
+        local->preparent.ia_size    = local->preparent_size;
+        local->postparent.ia_blocks = local->postparent_blocks;
+        local->postparent.ia_size   = local->postparent_size;
+        local->stbuf.ia_size        = local->stbuf_size;
+        local->stbuf.ia_blocks      = local->stbuf_blocks;
+
+        trav = trav->next;
+        while (trav) {
+                if (priv->xattr_supported) {
+                        dict = dict_new ();
+                        if (!dict) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "failed to allocate dict %s", local->loc.path);
                         }
-                        return 0;
+                        need_unref = 1;
+
+                        dict_copy (local->xattr, dict);
+
+                        ret = stripe_xattr_request_build (this, dict,
+                                                          local->stripe_size,
+                                                          priv->child_count, i,
+							  priv->coalesce);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to build xattr request");
+
+                } else {
+                        dict = local->xattr;
                 }
 
-                /* Create itself has failed.. so return
-                   without setxattring */
-                stripe_local_wipe (local);
-                STACK_UNWIND_STRICT (mknod, frame, local->op_ret, local->op_errno,
-                                     local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent);
+                STACK_WIND (frame, stripe_mknod_ifreg_cbk,
+                            trav->xlator, trav->xlator->fops->mknod,
+                            &local->loc, local->mode, local->rdev, 0, dict);
+                trav = trav->next;
+                i++;
+
+                if (dict && need_unref)
+                        dict_unref (dict);
         }
-out:
+
         return 0;
+
+out:
+
+       STRIPE_STACK_UNWIND (mknod, frame, op_ret, op_errno, NULL, NULL, NULL, NULL, NULL);
+       return 0;
 }
 
 
@@ -1436,22 +1646,25 @@ int32_t
 stripe_single_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                          int32_t op_ret, int32_t op_errno, inode_t *inode,
                          struct iatt *buf, struct iatt *preparent,
-                         struct iatt *postparent)
+                         struct iatt *postparent, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+        STRIPE_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 stripe_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dev_t rdev, dict_t *params)
+              dev_t rdev, mode_t umask, dict_t *xdata)
 {
-        stripe_private_t *priv = NULL;
-        stripe_local_t   *local = NULL;
-        xlator_list_t    *trav = NULL;
-        int32_t           op_errno = EINVAL;
+        stripe_private_t *priv           = NULL;
+        stripe_local_t   *local          = NULL;
+        int32_t           op_errno       = EINVAL;
+        int32_t           i              = 0;
+        dict_t           *dict           = NULL;
+        int               ret            = 0;
+        int               need_unref     = 0;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -1460,7 +1673,6 @@ stripe_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
         VALIDATE_OR_GOTO (loc->inode, err);
 
         priv = this->private;
-        trav = this->children;
 
         if (priv->first_child_down) {
                 op_errno = ENOTCONN;
@@ -1479,43 +1691,63 @@ stripe_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
                 }
 
                 /* Initialization */
-                local = GF_CALLOC (1, sizeof (stripe_local_t),
-                                   gf_stripe_mt_stripe_local_t);
+                local = mem_get0 (this->local_pool);
                 if (!local) {
                         op_errno = ENOMEM;
                         goto err;
                 }
                 local->op_ret = -1;
                 local->op_errno = ENOTCONN;
-                local->stripe_size = stripe_get_matching_bs (loc->path,
-                                                             priv->pattern,
-                                                             priv->block_size);
+                local->stripe_size = stripe_get_matching_bs (loc->path, priv);
                 frame->local = local;
-                local->inode = loc->inode;
+                local->inode = inode_ref (loc->inode);
                 loc_copy (&local->loc, loc);
+                local->xattr = dict_copy_with_ref (xdata, NULL);
+                local->mode = mode;
+                local->umask = umask;
+                local->rdev = rdev;
 
                 /* Everytime in stripe lookup, all child nodes should
                    be looked up */
                 local->call_count = priv->child_count;
 
-                while (trav) {
-                        STACK_WIND (frame, stripe_mknod_ifreg_cbk,
-                                    trav->xlator, trav->xlator->fops->mknod,
-                                    loc, mode, rdev, params);
-                        trav = trav->next;
+                if (priv->xattr_supported) {
+                        dict = dict_new ();
+                        if (!dict) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "failed to allocate dict %s", loc->path);
+                        }
+                        need_unref = 1;
+
+                        dict_copy (xdata, dict);
+
+                        ret = stripe_xattr_request_build (this, dict,
+                                                          local->stripe_size,
+                                                          priv->child_count,
+                                                          i, priv->coalesce);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "failed to build xattr request");
+                } else {
+                        dict = xdata;
                 }
 
-                /* This case is handled, no need to continue further. */
+                STACK_WIND (frame, stripe_mknod_first_ifreg_cbk,
+                            FIRST_CHILD (this), FIRST_CHILD (this)->fops->mknod,
+                            loc, mode, rdev, umask, dict);
+
+                        if (dict && need_unref)
+                                dict_unref (dict);
                 return 0;
         }
 
         STACK_WIND (frame, stripe_single_mknod_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, rdev, params);
+                    loc, mode, rdev, umask, xdata);
 
         return 0;
 err:
-        STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        STRIPE_STACK_UNWIND (mknod, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1524,11 +1756,10 @@ int32_t
 stripe_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, inode_t *inode,
                   struct iatt *buf, struct iatt *preparent,
-                  struct iatt *postparent)
+                  struct iatt *postparent, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t  *local   = NULL;
-        inode_t         *local_inode = NULL;
         call_frame_t    *prev = NULL;
 
         if (!this || !frame || !frame->local || !cookie) {
@@ -1556,12 +1787,6 @@ stripe_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (op_ret >= 0) {
                         local->op_ret = 0;
 
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->inode      = inode_ref (inode);
-                                local->stbuf      = *buf;
-                                local->postparent = *postparent;
-                                local->preparent  = *preparent;
-                        }
                         local->stbuf_blocks      += buf->ia_blocks;
                         local->preparent_blocks  += preparent->ia_blocks;
                         local->postparent_blocks += postparent->ia_blocks;
@@ -1577,12 +1802,7 @@ stripe_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         UNLOCK (&frame->lock);
 
         if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                local_inode = local->inode;
-
-                if (local->op_ret != -1) {
+                if (local->failed != -1) {
                         local->preparent.ia_blocks  = local->preparent_blocks;
                         local->preparent.ia_size    = local->preparent_size;
                         local->postparent.ia_blocks = local->postparent_blocks;
@@ -1590,22 +1810,79 @@ stripe_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->stbuf.ia_size        = local->stbuf_size;
                         local->stbuf.ia_blocks      = local->stbuf_blocks;
                 }
-                STACK_UNWIND_STRICT (mkdir, frame, local->op_ret,
+                STRIPE_STACK_UNWIND (mkdir, frame, local->op_ret,
                                      local->op_errno, local->inode,
                                      &local->stbuf, &local->preparent,
-                                     &local->postparent);
+                                     &local->postparent, NULL);
+        }
+out:
+        return 0;
+}
+
 
-                if (local_inode)
-                        inode_unref (local_inode);
+int32_t
+stripe_first_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, inode_t *inode,
+                        struct iatt *buf, struct iatt *preparent,
+                        struct iatt *postparent, dict_t *xdata)
+{
+        stripe_local_t  *local   = NULL;
+        call_frame_t    *prev = NULL;
+        xlator_list_t        *trav = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
         }
+
+        prev  = cookie;
+        local = frame->local;
+        trav = this->children;
+
+        local->call_count--; /* first child is successful */
+        trav = trav->next;   /* skip first child */
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG, "%s returned error %s",
+                        prev->this->name, strerror (op_errno));
+                local->op_errno = op_errno;
+                goto out;
+        }
+
+        local->op_ret = 0;
+
+        local->inode      = inode_ref (inode);
+        local->stbuf      = *buf;
+        local->postparent = *postparent;
+        local->preparent  = *preparent;
+
+        local->stbuf_blocks      += buf->ia_blocks;
+        local->preparent_blocks  += preparent->ia_blocks;
+        local->postparent_blocks += postparent->ia_blocks;
+
+        local->stbuf_size = buf->ia_size;
+        local->preparent_size = preparent->ia_size;
+        local->postparent_size = postparent->ia_size;
+
+        while (trav) {
+                STACK_WIND (frame, stripe_mkdir_cbk, trav->xlator,
+                            trav->xlator->fops->mkdir, &local->loc, local->mode,
+                            local->umask, local->xdata);
+                trav = trav->next;
+        }
+        return 0;
 out:
+        STRIPE_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL,
+                             NULL, NULL);
+
         return 0;
+
 }
 
 
 int
 stripe_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-              dict_t *params)
+              mode_t umask, dict_t *xdata)
 {
         stripe_private_t *priv = NULL;
         stripe_local_t   *local = NULL;
@@ -1627,27 +1904,27 @@ stripe_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
         }
         local->op_ret = -1;
         local->call_count = priv->child_count;
+        if (xdata)
+                local->xdata = dict_ref (xdata);
+        local->mode  = mode;
+        local->umask = umask;
+        loc_copy (&local->loc, loc);
         frame->local = local;
 
         /* Everytime in stripe lookup, all child nodes should be looked up */
-        while (trav) {
-                STACK_WIND (frame, stripe_mkdir_cbk,
-                            trav->xlator, trav->xlator->fops->mkdir,
-                            loc, mode, params);
-                trav = trav->next;
-        }
+        STACK_WIND (frame, stripe_first_mkdir_cbk, trav->xlator,
+                    trav->xlator->fops->mkdir, loc, mode, umask, xdata);
 
         return 0;
 err:
-        STACK_UNWIND_STRICT (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        STRIPE_STACK_UNWIND (mkdir, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1656,12 +1933,12 @@ int32_t
 stripe_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, inode_t *inode,
                  struct iatt *buf, struct iatt *preparent,
-                 struct iatt *postparent)
+                 struct iatt *postparent, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t  *local   = NULL;
-        inode_t         *local_inode = NULL;
         call_frame_t    *prev = NULL;
+	stripe_fd_ctx_t *fctx = NULL;
 
         if (!this || !frame || !frame->local || !cookie) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -1688,6 +1965,16 @@ stripe_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (op_ret >= 0) {
                         local->op_ret = 0;
 
+			if (IA_ISREG(inode->ia_type)) {
+				inode_ctx_get(inode, this, (uint64_t *) &fctx);
+				if (!fctx) {
+					gf_log(this->name, GF_LOG_ERROR,
+						"failed to get stripe context");
+					op_ret = -1;
+					op_errno = EINVAL;
+				}
+			}
+
                         if (FIRST_CHILD(this) == prev->this) {
                                 local->inode      = inode_ref (inode);
                                 local->stbuf      = *buf;
@@ -1698,6 +1985,8 @@ stripe_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->preparent_blocks  += preparent->ia_blocks;
                         local->postparent_blocks += postparent->ia_blocks;
 
+			correct_file_size(buf, fctx, prev);
+
                         if (local->stbuf_size < buf->ia_size)
                                 local->stbuf_size = buf->ia_size;
                         if (local->preparent_size < preparent->ia_size)
@@ -1712,8 +2001,6 @@ stripe_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (local->failed)
                         local->op_ret = -1;
 
-                local_inode = local->inode;
-
                 if (local->op_ret != -1) {
                         local->preparent.ia_blocks  = local->preparent_blocks;
                         local->preparent.ia_size    = local->preparent_size;
@@ -1722,20 +2009,17 @@ stripe_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->stbuf.ia_size        = local->stbuf_size;
                         local->stbuf.ia_blocks      = local->stbuf_blocks;
                 }
-                STACK_UNWIND_STRICT (link, frame, local->op_ret,
+                STRIPE_STACK_UNWIND (link, frame, local->op_ret,
                                      local->op_errno, local->inode,
                                      &local->stbuf, &local->preparent,
-                                     &local->postparent);
-
-                if (local_inode)
-                        inode_unref (local_inode);
+                                     &local->postparent, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
+stripe_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
         xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
@@ -1758,8 +2042,7 @@ stripe_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -1773,13 +2056,13 @@ stripe_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
         while (trav) {
                 STACK_WIND (frame, stripe_link_cbk,
                             trav->xlator, trav->xlator->fops->link,
-                            oldloc, newloc);
+                            oldloc, newloc, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STACK_UNWIND_STRICT (link, frame, -1, op_errno, NULL, NULL, NULL, NULL);
+        STRIPE_STACK_UNWIND (link, frame, -1, op_errno, NULL, NULL, NULL, NULL, NULL);
         return 0;
 }
 
@@ -1787,12 +2070,10 @@ int32_t
 stripe_create_fail_unlink_cbk (call_frame_t *frame, void *cookie,
                                xlator_t *this, int32_t op_ret,
                                int32_t op_errno, struct iatt *preparent,
-                               struct iatt *postparent)
+                               struct iatt *postparent, dict_t *xdata)
 {
         int32_t         callcnt = 0;
-        fd_t           *lfd = NULL;
         stripe_local_t *local = NULL;
-        inode_t        *local_inode = NULL;
 
         if (!this || !frame || !frame->local) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -1808,18 +2089,9 @@ stripe_create_fail_unlink_cbk (call_frame_t *frame, void *cookie,
         UNLOCK (&frame->lock);
 
         if (!callcnt) {
-                local_inode = local->inode;
-                lfd = local->fd;
-
-                stripe_local_wipe (local);
-                STACK_UNWIND_STRICT (create, frame, local->op_ret, local->op_errno,
+                STRIPE_STACK_UNWIND (create, frame, local->op_ret, local->op_errno,
                                      local->fd, local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent);
-
-                if (local_inode)
-                        inode_unref (local_inode);
-                if (lfd)
-                        fd_unref (lfd);
+                                     &local->preparent, &local->postparent, NULL);
         }
 out:
         return 0;
@@ -1827,16 +2099,16 @@ out:
 
 
 int32_t
-stripe_create_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                            int32_t op_ret, int32_t op_errno)
+stripe_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd,
+                   inode_t *inode, struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
 {
-        inode_t          *local_inode = NULL;
-        fd_t             *lfd = NULL;
+        int32_t           callcnt = 0;
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
+        call_frame_t     *prev = NULL;
         xlator_list_t    *trav = NULL;
-        int32_t           callcnt = 0;
-        call_frame_t   *prev = NULL;
 
         if (!this || !frame || !frame->local || !cookie) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -1844,7 +2116,7 @@ stripe_create_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         prev  = cookie;
-	priv  = this->private;
+        priv  = this->private;
         local = frame->local;
 
         LOCK (&frame->lock);
@@ -1855,13 +2127,40 @@ stripe_create_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         gf_log (this->name, GF_LOG_DEBUG,
                                 "%s returned error %s",
                                 prev->this->name, strerror (op_errno));
-                        local->op_ret = -1;
+                        local->failed = 1;
                         local->op_errno = op_errno;
                 }
+
+                if (op_ret >= 0) {
+			if (IA_ISREG(buf->ia_type)) {
+				if (stripe_ctx_handle(this, prev, local, xdata))
+					gf_log(this->name, GF_LOG_ERROR,
+						"Error getting fctx info from "
+						"dict");
+			}
+
+                        local->op_ret = op_ret;
+
+                        local->stbuf_blocks += buf->ia_blocks;
+                        local->preparent_blocks  += preparent->ia_blocks;
+                        local->postparent_blocks += postparent->ia_blocks;
+
+			correct_file_size(buf, local->fctx, prev);
+
+                        if (local->stbuf_size < buf->ia_size)
+                                local->stbuf_size = buf->ia_size;
+                        if (local->preparent_size < preparent->ia_size)
+                                local->preparent_size = preparent->ia_size;
+                        if (local->postparent_size < postparent->ia_size)
+                                local->postparent_size = postparent->ia_size;
+                }
         }
         UNLOCK (&frame->lock);
 
         if (!callcnt) {
+                if (local->failed)
+                        local->op_ret = -1;
+
                 if (local->op_ret == -1) {
                         local->call_count = priv->child_count;
                         trav = this->children;
@@ -1870,44 +2169,57 @@ stripe_create_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                             stripe_create_fail_unlink_cbk,
                                             trav->xlator,
                                             trav->xlator->fops->unlink,
-                                            &local->loc);
+                                            &local->loc, 0, NULL);
                                 trav = trav->next;
                         }
 
                         return 0;
                 }
 
-                lfd = local->fd;
-                local_inode = local->inode;
+                if (local->op_ret >= 0) {
+                        local->preparent.ia_blocks  = local->preparent_blocks;
+                        local->preparent.ia_size    = local->preparent_size;
+                        local->postparent.ia_blocks = local->postparent_blocks;
+                        local->postparent.ia_size   = local->postparent_size;
+                        local->stbuf.ia_size        = local->stbuf_size;
+                        local->stbuf.ia_blocks      = local->stbuf_blocks;
 
-                stripe_local_wipe (local);
-                STACK_UNWIND_STRICT (create, frame, local->op_ret, local->op_errno,
-                                     local->fd, local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent);
+			stripe_copy_xl_array(local->fctx->xl_array,
+				             priv->xl_array,
+					     local->fctx->stripe_count);
+			inode_ctx_put(local->inode, this,
+					(uint64_t) local->fctx);
+                }
 
-                if (local_inode)
-                        inode_unref (local_inode);
-                if (lfd)
-                        fd_unref (lfd);
+                /* Create itself has failed.. so return
+                   without setxattring */
+                STRIPE_STACK_UNWIND (create, frame, local->op_ret,
+                                     local->op_errno, local->fd,
+                                     local->inode, &local->stbuf,
+                                     &local->preparent, &local->postparent, NULL);
         }
+
 out:
         return 0;
 }
 
+
+
 int32_t
-stripe_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+stripe_first_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, fd_t *fd,
                    inode_t *inode, struct iatt *buf, struct iatt *preparent,
-                   struct iatt *postparent)
+                   struct iatt *postparent, dict_t *xdata)
 {
-        int32_t           callcnt = 0;
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
-        fd_t             *lfd = NULL;
-        stripe_fd_ctx_t  *fctx = NULL;
-        inode_t          *local_inode = NULL;
         call_frame_t     *prev = NULL;
-        int               ret = 0;
+        xlator_list_t    *trav = NULL;
+        int               i    = 1;
+        dict_t           *dict = NULL;
+        loc_t            *loc  = NULL;
+        int32_t           need_unref = 0;
+        int32_t           ret  = -1;
 
         if (!this || !frame || !frame->local || !cookie) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -1917,150 +2229,89 @@ stripe_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         prev  = cookie;
         priv  = this->private;
         local = frame->local;
+        trav = this->children;
+        loc = &local->loc;
 
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                        local->op_errno = op_errno;
-                }
-
-                if (op_ret >= 0) {
-                        local->op_ret = op_ret;
-                        /* Get the mapping in inode private */
-                        /* Get the stat buf right */
-                        if (FIRST_CHILD(this) == prev->this) {
-                                local->stbuf      = *buf;
-                                local->preparent  = *preparent;
-                                local->postparent = *postparent;
-                        }
-
-                        local->stbuf_blocks += buf->ia_blocks;
-                        local->preparent_blocks  += preparent->ia_blocks;
-                        local->postparent_blocks += postparent->ia_blocks;
+        --local->call_count;
 
-                        if (local->stbuf_size < buf->ia_size)
-                                local->stbuf_size = buf->ia_size;
-                        if (local->preparent_size < preparent->ia_size)
-                                local->preparent_size = preparent->ia_size;
-                        if (local->postparent_size < postparent->ia_size)
-                                local->postparent_size = postparent->ia_size;
-                }
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG, "%s returned error %s",
+                        prev->this->name, strerror (op_errno));
+                 local->failed = 1;
+                 local->op_errno = op_errno;
         }
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret != -1) {
-                        local->preparent.ia_blocks  = local->preparent_blocks;
-                        local->preparent.ia_size    = local->preparent_size;
-                        local->postparent.ia_blocks = local->postparent_blocks;
-                        local->postparent.ia_size   = local->postparent_size;
-                        local->stbuf.ia_size        = local->stbuf_size;
-                        local->stbuf.ia_blocks      = local->stbuf_blocks;
-                }
-
-                /* */
-                if (local->op_ret >= 0) {
-                        fctx = GF_CALLOC (1, sizeof (stripe_fd_ctx_t),
-                                          gf_stripe_mt_stripe_fd_ctx_t);
-                        if (!fctx) {
-                                local->op_ret = -1;
-                                local->op_errno = ENOMEM;
-                                goto unwind;
-                        }
 
-                        fctx->stripe_size  = local->stripe_size;
-                        fctx->stripe_count = priv->child_count;
-                        fctx->static_array = 1;
-                        fctx->xl_array = priv->xl_array;
-                        fd_ctx_set (local->fd, this,
-                                    (uint64_t)(long)fctx);
-                }
-
-                if ((local->op_ret != -1) &&
-                    local->stripe_size && priv->xattr_supported) {
-                        /* Send a setxattr request to nodes where
-                           the files are created */
-                        int32_t        i = 0;
-                        char           size_key[256] = {0,};
-                        char           index_key[256] = {0,};
-                        char           count_key[256] = {0,};
-                        dict_t        *dict = NULL;
-
-                        sprintf (size_key,
-                                 "trusted.%s.stripe-size", this->name);
-                        sprintf (count_key,
-                                 "trusted.%s.stripe-count", this->name);
-                        sprintf (index_key,
-                                 "trusted.%s.stripe-index", this->name);
+        local->op_ret = 0;
+        /* Get the mapping in inode private */
+        /* Get the stat buf right */
+        local->stbuf      = *buf;
+        local->preparent  = *preparent;
+        local->postparent = *postparent;
 
-                        local->call_count = priv->child_count;
-                        memcpy (local->loc.inode->gfid, local->stbuf.ia_gfid, 16);
-                        for (i = 0; i < priv->child_count; i++) {
-                                dict = get_new_dict ();
-                                if (!dict) {
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "error allocating dict");
-                                }
-                                dict_ref (dict);
+        local->stbuf_blocks += buf->ia_blocks;
+        local->preparent_blocks  += preparent->ia_blocks;
+        local->postparent_blocks += postparent->ia_blocks;
 
-                                /* TODO: check return values */
-                                ret = dict_set_int64 (dict, size_key,
-                                                      local->stripe_size);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "%s: set stripe-size failed",
-                                                local->loc.path);
+        if (local->stbuf_size < buf->ia_size)
+              local->stbuf_size = buf->ia_size;
+        if (local->preparent_size < preparent->ia_size)
+              local->preparent_size = preparent->ia_size;
+        if (local->postparent_size < postparent->ia_size)
+              local->postparent_size = postparent->ia_size;
 
-                                ret = dict_set_int32 (dict, count_key,
-                                                      priv->child_count);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "%s: set stripe-size failed",
-                                                local->loc.path);
+        if (local->failed)
+                local->op_ret = -1;
 
-                                ret = dict_set_int32 (dict, index_key, i);
-                                if (ret)
-                                        gf_log (this->name, GF_LOG_ERROR,
-                                                "%s: set stripe-size failed",
-                                                local->loc.path);
+        if (local->op_ret == -1) {
+                local->call_count = 1;
+                STACK_WIND (frame, stripe_create_fail_unlink_cbk,
+                            FIRST_CHILD (this), FIRST_CHILD (this)->fops->unlink,
+                            &local->loc, 0, NULL);
+                return 0;
+        }
 
-                                STACK_WIND (frame, stripe_create_setxattr_cbk,
-                                            priv->xl_array[i],
-                                            priv->xl_array[i]->fops->setxattr,
-                                            &local->loc, dict, 0);
+        if (local->op_ret >= 0) {
+                local->preparent.ia_blocks  = local->preparent_blocks;
+                local->preparent.ia_size    = local->preparent_size;
+                local->postparent.ia_blocks = local->postparent_blocks;
+                local->postparent.ia_size   = local->postparent_size;
+                local->stbuf.ia_size        = local->stbuf_size;
+                local->stbuf.ia_blocks      = local->stbuf_blocks;
+        }
 
-                                dict_unref (dict);
+        /* Send a setxattr request to nodes where the
+           files are created */
+        trav = trav->next;
+        while (trav) {
+                if (priv->xattr_supported) {
+                        dict = dict_new ();
+                        if (!dict) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "failed to allocate dict %s", loc->path);
                         }
-                        return 0;
-                }
+                        need_unref = 1;
 
-unwind:
-                /* Create itself has failed.. so return
-                   without setxattring */
-                lfd = local->fd;
-                local_inode = local->inode;
+                        dict_copy (local->xattr, dict);
 
-                stripe_local_wipe (local);
-                STACK_UNWIND_STRICT (create, frame, local->op_ret,
-                                     local->op_errno, local->fd,
-                                     local->inode, &local->stbuf,
-                                     &local->preparent, &local->postparent);
+                        ret = stripe_xattr_request_build (this, dict,
+                                                          local->stripe_size,
+                                                          priv->child_count,
+                                                          i, priv->coalesce);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "failed to build xattr request");
+                } else {
+                        dict = local->xattr;
+                }
 
-                if (local_inode)
-                        inode_unref (local_inode);
-                if (lfd)
-                        fd_unref (lfd);
+                STACK_WIND (frame, stripe_create_cbk, trav->xlator,
+                            trav->xlator->fops->create, &local->loc,
+                            local->flags, local->mode, local->umask, local->fd,
+                            dict);
+                trav = trav->next;
+                if (need_unref && dict)
+                        dict_unref (dict);
+                i++;
         }
 
 out:
@@ -2068,6 +2319,7 @@ out:
 }
 
 
+
 /**
  * stripe_create - If a block-size is specified for the 'name', create the
  *    file in all the child nodes. If not, create it in only first child.
@@ -2076,12 +2328,15 @@ out:
  */
 int32_t
 stripe_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
+               int32_t flags, mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
 {
         stripe_private_t *priv = NULL;
         stripe_local_t   *local = NULL;
-        xlator_list_t    *trav = NULL;
         int32_t           op_errno = EINVAL;
+        int               ret            = 0;
+        int               need_unref     = 0;
+        int               i              = 0;
+        dict_t           *dict           = NULL;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -2102,46 +2357,71 @@ stripe_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
         }
         local->op_ret = -1;
         local->op_errno = ENOTCONN;
-        local->stripe_size = stripe_get_matching_bs (loc->path,
-                                                     priv->pattern,
-                                                     priv->block_size);
+        local->stripe_size = stripe_get_matching_bs (loc->path, priv);
         frame->local = local;
         local->inode = inode_ref (loc->inode);
         loc_copy (&local->loc, loc);
         local->fd = fd_ref (fd);
+        local->flags = flags;
+        local->mode = mode;
+        local->umask = umask;
+        if (xdata)
+                local->xattr = dict_ref (xdata);
 
         local->call_count = priv->child_count;
+        /* Send a setxattr request to nodes where the
+           files are created */
 
-        trav = this->children;
-        while (trav) {
-                STACK_WIND (frame, stripe_create_cbk, trav->xlator,
-                            trav->xlator->fops->create, loc, flags,
-                            mode, fd, params);
-                trav = trav->next;
+        if (priv->xattr_supported) {
+                dict = dict_new ();
+                if (!dict) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to allocate dict %s", loc->path);
+                }
+                need_unref = 1;
+
+                dict_copy (xdata, dict);
+
+                ret = stripe_xattr_request_build (this, dict,
+                                                  local->stripe_size,
+                                                  priv->child_count,
+                                                  i, priv->coalesce);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to build xattr request");
+        } else {
+                        dict = xdata;
         }
 
+
+        STACK_WIND (frame, stripe_first_create_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->create, loc, flags, mode,
+                    umask, fd, dict);
+
+        if (need_unref && dict)
+                dict_unref (dict);
+
+
         return 0;
 err:
-        STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL, NULL, NULL,
-                             NULL, NULL);
+        STRIPE_STACK_UNWIND (create, frame, -1, op_errno, NULL, NULL, NULL,
+                             NULL, NULL, xdata);
         return 0;
 }
 
 int32_t
 stripe_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, fd_t *fd)
+                 int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
-        fd_t           *lfd = NULL;
         call_frame_t   *prev = NULL;
 
         if (!this || !frame || !frame->local || !cookie) {
@@ -2176,228 +2456,20 @@ stripe_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (local->failed)
                         local->op_ret = -1;
 
-                if (local->op_ret == -1) {
-                        if (local->fctx) {
-                                if (!local->fctx->static_array)
-                                        GF_FREE (local->fctx->xl_array);
-                                GF_FREE (local->fctx);
-                        }
-                } else {
-                        fd_ctx_set (local->fd, this,
-                                    (uint64_t)(long)local->fctx);
-                }
-
-                lfd = local->fd;
-
-                stripe_local_wipe (local);
-                STACK_UNWIND_STRICT (open, frame, local->op_ret,
-                                     local->op_errno, local->fd);
-                if (lfd)
-                        fd_unref (lfd);
-
+                STRIPE_STACK_UNWIND (open, frame, local->op_ret,
+                                     local->op_errno, local->fd, xdata);
         }
 out:
         return 0;
 }
 
 
-int32_t
-stripe_open_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno, dict_t *dict)
-{
-        int32_t           index = 0;
-        int32_t           callcnt = 0;
-        char              key[256] = {0,};
-        stripe_local_t   *local = NULL;
-        xlator_list_t    *trav = NULL;
-        stripe_private_t *priv = NULL;
-        data_t           *data = NULL;
-        call_frame_t     *prev = NULL;
-        fd_t             *lfd = NULL;
-
-        if (!this || !frame || !frame->local || !cookie) {
-                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
-                goto out;
-        }
-
-        prev  = (call_frame_t *)cookie;
-        priv  = this->private;
-        local = frame->local;
-
-        LOCK (&frame->lock);
-        {
-                callcnt = --local->call_count;
-
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_ret = -1;
-                        if (local->op_errno != EIO)
-                                local->op_errno = op_errno;
-                        if ((op_errno != ENOENT) ||
-                            (prev->this == FIRST_CHILD (this)))
-                                local->failed = 1;
-                        goto unlock;
-                }
-
-                if (!dict)
-                        goto unlock;
-
-                if (!local->fctx) {
-                        local->fctx =  GF_CALLOC (1, sizeof (stripe_fd_ctx_t),
-                                                  gf_stripe_mt_stripe_fd_ctx_t);
-                        if (!local->fctx) {
-                                local->op_errno = ENOMEM;
-                                local->op_ret = -1;
-                                goto unlock;
-                        }
-
-                        local->fctx->static_array = 0;
-                }
-                /* Stripe block size */
-                sprintf (key, "trusted.%s.stripe-size", this->name);
-                data = dict_get (dict, key);
-                if (!data) {
-                        local->xattr_self_heal_needed = 1;
-                } else {
-                        if (!local->fctx->stripe_size) {
-                                local->fctx->stripe_size =
-                                        data_to_int64 (data);
-                        }
-
-                        if (local->fctx->stripe_size != data_to_int64 (data)) {
-                                gf_log (this->name, GF_LOG_WARNING,
-                                        "stripe-size mismatch in blocks");
-                                local->xattr_self_heal_needed = 1;
-                        }
-                }
-                /* Stripe count */
-                sprintf (key, "trusted.%s.stripe-count", this->name);
-                data = dict_get (dict, key);
-                if (!data) {
-                        local->xattr_self_heal_needed = 1;
-                        goto unlock;
-                }
-                if (!local->fctx->xl_array) {
-                        local->fctx->stripe_count = data_to_int32 (data);
-                        if (!local->fctx->stripe_count) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "error with stripe-count xattr");
-                                local->op_ret   = -1;
-                                local->op_errno = EIO;
-                                goto unlock;
-                        }
-
-                        local->fctx->xl_array =
-                                GF_CALLOC (local->fctx->stripe_count,
-                                           sizeof (xlator_t *),
-                                           gf_stripe_mt_xlator_t);
-                        if (!local->fctx->xl_array) {
-                                local->op_errno = ENOMEM;
-                                local->op_ret   = -1;
-                                goto unlock;
-                        }
-                }
-                if (local->fctx->stripe_count != data_to_int32 (data)) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "error with stripe-count xattr (%d != %d)",
-                                local->fctx->stripe_count, data_to_int32 (data));
-                        local->op_ret   = -1;
-                        local->op_errno = EIO;
-                        goto unlock;
-                }
-
-                /* index */
-                sprintf (key, "trusted.%s.stripe-index", this->name);
-                data = dict_get (dict, key);
-                if (!data) {
-                        local->xattr_self_heal_needed = 1;
-                        goto unlock;
-                }
-                index = data_to_int32 (data);
-                if (index > priv->child_count) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "error with stripe-index xattr (%d)", index);
-                        local->op_ret   = -1;
-                        local->op_errno = EIO;
-                        goto unlock;
-                }
-                if (local->fctx->xl_array) {
-                        if (local->fctx->xl_array[index]) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "duplicate entry @ index (%d)", index);
-                                local->op_ret   = -1;
-                                local->op_errno = EIO;
-                                goto unlock;
-                        }
-                        local->fctx->xl_array[index] = prev->this;
-                }
-                local->entry_count++;
-                local->op_ret = 0;
-        }
-unlock:
-        UNLOCK (&frame->lock);
-
-        if (!callcnt) {
-                /* TODO: if self-heal flag is set, do it */
-                if (local->xattr_self_heal_needed) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s: stripe info need to be healed",
-                                local->loc.path);
-                }
-
-                if (local->failed)
-                        local->op_ret = -1;
-
-                if (local->op_ret)
-                        goto err;
-
-                if (local->entry_count != local->fctx->stripe_count) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "entry-count (%d) != stripe-count (%d)",
-                                local->entry_count, local->fctx->stripe_count);
-                        local->op_ret = -1;
-                        local->op_errno = EIO;
-                        goto err;
-                }
-                if (!local->fctx->stripe_size) {
-                        gf_log (this->name, GF_LOG_ERROR, "stripe size not set");
-                        local->op_ret = -1;
-                        local->op_errno = EIO;
-                        goto err;
-                }
-
-                local->call_count = local->fctx->stripe_count;
-
-                trav = this->children;
-                while (trav) {
-                        STACK_WIND (frame, stripe_open_cbk, trav->xlator,
-                                    trav->xlator->fops->open, &local->loc,
-                                    local->flags, local->fd, 0);
-                        trav = trav->next;
-                }
-        }
-
-        return 0;
-err:
-        lfd = local->fd;
-
-        stripe_local_wipe (local);
-        STACK_UNWIND_STRICT (open, frame, local->op_ret, local->op_errno,
-                             local->fd);
-        if (lfd)
-                fd_unref (lfd);
-out:
-        return 0;
-}
-
 /**
  * stripe_open -
  */
 int32_t
 stripe_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             int32_t flags, fd_t *fd, int32_t wbflags)
+             int32_t flags, fd_t *fd, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
@@ -2419,8 +2491,7 @@ stripe_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -2436,52 +2507,28 @@ stripe_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
         /* Striped files */
         local->flags = flags;
         local->call_count = priv->child_count;
-        local->stripe_size = stripe_get_matching_bs (loc->path,
-                                                     priv->pattern,
-                                                     priv->block_size);
-
-        if (priv->xattr_supported) {
-                while (trav) {
-                        STACK_WIND (frame, stripe_open_getxattr_cbk,
-                                    trav->xlator, trav->xlator->fops->getxattr,
-                                    loc, NULL);
-                        trav = trav->next;
-                }
-                return 0;
-        }
-        local->fctx =  GF_CALLOC (1, sizeof (stripe_fd_ctx_t),
-                                  gf_stripe_mt_stripe_fd_ctx_t);
-        if (!local->fctx) {
-                op_errno = ENOMEM;
-                goto err;
-        }
-
-        local->fctx->static_array = 1;
-        local->fctx->stripe_size  = local->stripe_size;
-        local->fctx->stripe_count = priv->child_count;
-        local->fctx->xl_array     = priv->xl_array;
+        local->stripe_size = stripe_get_matching_bs (loc->path, priv);
 
         while (trav) {
                 STACK_WIND (frame, stripe_open_cbk, trav->xlator,
                             trav->xlator->fops->open,
                             &local->loc, local->flags, local->fd,
-                            wbflags);
+                            xdata);
                 trav = trav->next;
         }
         return 0;
 err:
-        STACK_UNWIND_STRICT (open, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (open, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 stripe_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, fd_t *fd)
+                    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
-        fd_t           *local_fd = NULL;
         call_frame_t   *prev = NULL;
 
         if (!this || !frame || !frame->local || !cookie) {
@@ -2510,11 +2557,8 @@ stripe_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         UNLOCK (&frame->lock);
 
         if (!callcnt) {
-                local_fd = local->fd;
-                STACK_UNWIND_STRICT (opendir, frame, local->op_ret,
-                                     local->op_errno, local->fd);
-                if (local_fd)
-                        fd_unref (local_fd);
+                STRIPE_STACK_UNWIND (opendir, frame, local->op_ret,
+                                     local->op_errno, local->fd, NULL);
         }
 out:
         return 0;
@@ -2522,7 +2566,7 @@ out:
 
 
 int32_t
-stripe_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+stripe_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, dict_t *xdata)
 {
         xlator_list_t    *trav = NULL;
         stripe_local_t   *local = NULL;
@@ -2544,8 +2588,7 @@ stripe_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
         }
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -2556,19 +2599,19 @@ stripe_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
 
         while (trav) {
                 STACK_WIND (frame, stripe_opendir_cbk, trav->xlator,
-                            trav->xlator->fops->opendir, loc, fd);
+                            trav->xlator->fops->opendir, loc, fd, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STACK_UNWIND_STRICT (opendir, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (opendir, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 int32_t
 stripe_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-               int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+               int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -2607,8 +2650,8 @@ stripe_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         if (!callcnt) {
                 if (local->failed)
                         local->op_ret = -1;
-                STACK_UNWIND_STRICT (lk, frame, local->op_ret,
-                                     local->op_errno, &local->lock);
+                STRIPE_STACK_UNWIND (lk, frame, local->op_ret,
+                                     local->op_errno, &local->lock, NULL);
         }
 out:
         return 0;
@@ -2616,7 +2659,7 @@ out:
 
 int32_t
 stripe_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-           struct gf_flock *lock)
+           struct gf_flock *lock, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         xlator_list_t    *trav = NULL;
@@ -2632,8 +2675,7 @@ stripe_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
         priv = this->private;
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -2644,20 +2686,20 @@ stripe_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
 
         while (trav) {
                 STACK_WIND (frame, stripe_lk_cbk, trav->xlator,
-                            trav->xlator->fops->lk, fd, cmd, lock);
+                            trav->xlator->fops->lk, fd, cmd, lock, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STACK_UNWIND_STRICT (lk, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (lk, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 stripe_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local   = NULL;
@@ -2693,16 +2735,15 @@ stripe_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (local->failed)
                         local->op_ret = -1;
 
-                stripe_local_wipe (local);
-                STACK_UNWIND_STRICT (flush, frame, local->op_ret,
-                                     local->op_errno);
+                STRIPE_STACK_UNWIND (flush, frame, local->op_ret,
+                                     local->op_errno, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+stripe_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
@@ -2722,8 +2763,7 @@ stripe_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
                 goto err;
         }
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -2734,13 +2774,13 @@ stripe_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
 
         while (trav) {
                 STACK_WIND (frame, stripe_flush_cbk,  trav->xlator,
-                            trav->xlator->fops->flush, fd);
+                            trav->xlator->fops->flush, fd, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STACK_UNWIND_STRICT (flush, frame, -1, op_errno);
+        STRIPE_STACK_UNWIND (flush, frame, -1, op_errno, NULL);
         return 0;
 }
 
@@ -2749,7 +2789,7 @@ err:
 int32_t
 stripe_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                  struct iatt *postbuf)
+                  struct iatt *postbuf, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local   = NULL;
@@ -2785,6 +2825,9 @@ stripe_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->prebuf_blocks  += prebuf->ia_blocks;
                         local->postbuf_blocks += postbuf->ia_blocks;
 
+			correct_file_size(prebuf, local->fctx, prev);
+			correct_file_size(postbuf, local->fctx, prev);
+
                         if (local->prebuf_size < prebuf->ia_size)
                                 local->prebuf_size = prebuf->ia_size;
 
@@ -2805,21 +2848,21 @@ stripe_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->post_buf.ia_size   = local->postbuf_size;
                 }
 
-                stripe_local_wipe (local);
-                STACK_UNWIND_STRICT (fsync, frame, local->op_ret,
+                STRIPE_STACK_UNWIND (fsync, frame, local->op_ret,
                                      local->op_errno, &local->pre_buf,
-                                     &local->post_buf);
+                                     &local->post_buf, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+stripe_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
         xlator_list_t    *trav = NULL;
+	stripe_fd_ctx_t  *fctx = NULL;
         int32_t           op_errno = 1;
 
         VALIDATE_OR_GOTO (frame, err);
@@ -2831,31 +2874,38 @@ stripe_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
         trav = this->children;
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
         }
-        local->op_ret = -1;
+
         frame->local = local;
+
+	inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
+	if (!fctx) {
+		op_errno = EINVAL;
+		goto err;
+	}
+	local->fctx = fctx;
+        local->op_ret = -1;
         local->call_count = priv->child_count;
 
         while (trav) {
                 STACK_WIND (frame, stripe_fsync_cbk, trav->xlator,
-                            trav->xlator->fops->fsync, fd, flags);
+                            trav->xlator->fops->fsync, fd, flags, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (fsync, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
 int32_t
 stripe_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                  int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
@@ -2890,6 +2940,9 @@ stripe_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                 local->stbuf = *buf;
 
                         local->stbuf_blocks += buf->ia_blocks;
+
+			correct_file_size(buf, local->fctx, prev);
+
                         if (local->stbuf_size < buf->ia_size)
                                 local->stbuf_size = buf->ia_size;
                 }
@@ -2905,9 +2958,8 @@ stripe_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         local->stbuf.ia_blocks = local->stbuf_blocks;
                 }
 
-                stripe_local_wipe (local);
-                STACK_UNWIND_STRICT (fstat, frame, local->op_ret,
-                                     local->op_errno, &local->stbuf);
+                STRIPE_STACK_UNWIND (fstat, frame, local->op_ret,
+                                     local->op_errno, &local->stbuf, NULL);
         }
 
 out:
@@ -2917,11 +2969,12 @@ out:
 int32_t
 stripe_fstat (call_frame_t *frame,
               xlator_t *this,
-              fd_t *fd)
+              fd_t *fd, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
         xlator_list_t    *trav = NULL;
+	stripe_fd_ctx_t  *fctx = NULL;
         int32_t           op_errno = 1;
 
         VALIDATE_OR_GOTO (frame, err);
@@ -2933,8 +2986,7 @@ stripe_fstat (call_frame_t *frame,
         trav = this->children;
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -2943,26 +2995,35 @@ stripe_fstat (call_frame_t *frame,
         frame->local = local;
         local->call_count = priv->child_count;
 
+	if (IA_ISREG(fd->inode->ia_type)) {
+		inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
+		if (!fctx)
+			goto err;
+		local->fctx = fctx;
+	}
+
         while (trav) {
                 STACK_WIND (frame, stripe_fstat_cbk, trav->xlator,
-                            trav->xlator->fops->fstat, fd);
+                            trav->xlator->fops->fstat, fd, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STACK_UNWIND_STRICT (fstat, frame, -1, op_errno, NULL);
+        STRIPE_STACK_UNWIND (fstat, frame, -1, op_errno, NULL, NULL);
         return 0;
 }
 
 
 int32_t
-stripe_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
+stripe_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
-        xlator_list_t    *trav = NULL;
-        int32_t           op_errno = 1;
+	stripe_fd_ctx_t  *fctx = NULL;
+	int		  i, eof_idx;
+	off_t		  dest_offset, tmp_offset;
+        int32_t		  op_errno = 1;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
@@ -2970,11 +3031,9 @@ stripe_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
         VALIDATE_OR_GOTO (fd->inode, err);
 
         priv = this->private;
-        trav = this->children;
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -2983,22 +3042,60 @@ stripe_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset)
         frame->local = local;
         local->call_count = priv->child_count;
 
-        while (trav) {
-                STACK_WIND (frame, stripe_truncate_cbk, trav->xlator,
-                            trav->xlator->fops->ftruncate, fd, offset);
-                trav = trav->next;
-        }
+	inode_ctx_get(fd->inode, this, (uint64_t *) &fctx);
+	if (!fctx) {
+		gf_log(this->name, GF_LOG_ERROR, "no stripe context");
+		op_errno = EINVAL;
+		goto err;
+	}
+	if (!fctx->stripe_count) {
+		gf_log(this->name, GF_LOG_ERROR, "no stripe count");
+		op_errno = EINVAL;
+		goto err;
+	}
+
+	local->fctx = fctx;
+	eof_idx = (offset / fctx->stripe_size) % fctx->stripe_count;
+
+	for (i = 0; i < fctx->stripe_count; i++) {
+		if (!fctx->xl_array[i]) {
+			gf_log(this->name, GF_LOG_ERROR, "no xlator at index "
+				"%d", i);
+			op_errno = EINVAL;
+			goto err;
+		}
+
+		if (fctx->stripe_coalesce) {
+			if (i < eof_idx)
+				tmp_offset = roof(offset, fctx->stripe_size *
+						fctx->stripe_count);
+			else if (i > eof_idx)
+				tmp_offset = floor(offset, fctx->stripe_size *
+						fctx->stripe_count);
+			else
+				tmp_offset = offset;
+
+			dest_offset = coalesced_offset(tmp_offset,
+				fctx->stripe_size, fctx->stripe_count);
+		} else {
+			dest_offset = offset;
+		}
+
+		STACK_WIND(frame, stripe_truncate_cbk, fctx->xl_array[i],
+			fctx->xl_array[i]->fops->ftruncate, fd, dest_offset,
+			NULL);
+	}
 
         return 0;
 err:
-        STACK_UNWIND_STRICT (ftruncate, frame, -1, op_errno, NULL, NULL);
+        STRIPE_STACK_UNWIND (ftruncate, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
 
 int32_t
 stripe_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local   = NULL;
@@ -3034,16 +3131,15 @@ stripe_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 if (local->failed)
                         local->op_ret = -1;
 
-                stripe_local_wipe (local);
-                STACK_UNWIND_STRICT (fsyncdir, frame, local->op_ret,
-                                     local->op_errno);
+                STRIPE_STACK_UNWIND (fsyncdir, frame, local->op_ret,
+                                     local->op_errno, NULL);
         }
 out:
         return 0;
 }
 
 int32_t
-stripe_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+stripe_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
 {
         stripe_local_t   *local = NULL;
         stripe_private_t *priv = NULL;
@@ -3059,8 +3155,7 @@ stripe_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
         trav = this->children;
 
         /* Initialization */
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -3071,20 +3166,20 @@ stripe_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
 
         while (trav) {
                 STACK_WIND (frame, stripe_fsyncdir_cbk, trav->xlator,
-                            trav->xlator->fops->fsyncdir, fd, flags);
+                            trav->xlator->fops->fsyncdir, fd, flags, NULL);
                 trav = trav->next;
         }
 
         return 0;
 err:
-        STACK_UNWIND_STRICT (fsyncdir, frame, -1, op_errno);
+        STRIPE_STACK_UNWIND (fsyncdir, frame, -1, op_errno, NULL);
         return 0;
 }
 
 
 int32_t
 stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                        int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
         int32_t         i = 0;
         int32_t         callcnt = 0;
@@ -3094,6 +3189,7 @@ stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         struct iatt     tmp_stbuf = {0,};
         struct iobref  *tmp_iobref = NULL;
         struct iobuf   *iobuf = NULL;
+	call_frame_t   *prev = NULL;
 
         if (!this || !frame || !frame->local) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -3101,13 +3197,16 @@ stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         }
 
         local = frame->local;
+	prev = cookie;
 
         LOCK (&frame->lock);
         {
                 callcnt = --local->call_count;
-                if (op_ret != -1)
+                if (op_ret != -1) {
+			correct_file_size(buf, local->fctx, prev);
                         if (local->stbuf_size < buf->ia_size)
                                 local->stbuf_size = buf->ia_size;
+		}
         }
         UNLOCK (&frame->lock);
 
@@ -3136,7 +3235,8 @@ stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                 vec[count].iov_len  =
                                         (local->replies[i].requested_size -
                                          local->replies[i].op_ret);
-                                iobuf = iobuf_get (this->ctx->iobuf_pool);
+                                iobuf = iobuf_get2 (this->ctx->iobuf_pool,
+                                                    vec[count].iov_len);
                                 if (!iobuf) {
                                         gf_log (this->name, GF_LOG_ERROR,
                                                 "Out of memory.");
@@ -3145,9 +3245,11 @@ stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                                         goto done;
                                 }
                                 memset (iobuf->ptr, 0, vec[count].iov_len);
-                                iobref_add (local->iobref, iobuf);
                                 vec[count].iov_base = iobuf->ptr;
 
+                                iobref_add (local->iobref, iobuf);
+                                iobuf_unref(iobuf);
+
                                 op_ret += vec[count].iov_len;
                                 count++;
                         }
@@ -3164,13 +3266,11 @@ stripe_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         done:
                 GF_FREE (local->replies);
                 tmp_iobref = local->iobref;
-                fd_unref (local->fd);
-                STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vec,
-                                     count, &tmp_stbuf, tmp_iobref);
+                STRIPE_STACK_UNWIND (readv, frame, op_ret, op_errno, vec,
+                                     count, &tmp_stbuf, tmp_iobref, NULL);
 
                 iobref_unref (tmp_iobref);
-                if (vec)
-                        GF_FREE (vec);
+                GF_FREE (vec);
         }
 out:
         return 0;
@@ -3183,7 +3283,7 @@ out:
 int32_t
 stripe_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                  int32_t count, struct iatt *stbuf, struct iobref *iobref)
+                  int32_t count, struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
         int32_t         index = 0;
         int32_t         callcnt = 0;
@@ -3194,8 +3294,10 @@ stripe_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         stripe_local_t *local = NULL;
         struct iovec   *final_vec = NULL;
         struct iatt     tmp_stbuf = {0,};
+        struct iatt    *tmp_stbuf_p = NULL; //need it for a warning
         struct iobref  *tmp_iobref = NULL;
         stripe_fd_ctx_t  *fctx = NULL;
+	call_frame_t	*prev = NULL;
 
         if (!this || !frame || !frame->local || !cookie) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -3204,6 +3306,7 @@ stripe_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         local  = frame->local;
         index  = local->node_index;
+	prev = cookie;
         mframe = local->orig_frame;
         if (!mframe)
                 goto out;
@@ -3224,6 +3327,12 @@ stripe_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         mlocal->replies[index].count  = count;
                         mlocal->replies[index].vector = iov_dup (vector, count);
 
+			correct_file_size(stbuf, fctx, prev);
+
+                        if (local->stbuf_size < stbuf->ia_size)
+                                local->stbuf_size = stbuf->ia_size;
+                        local->stbuf_blocks += stbuf->ia_blocks;
+
                         if (!mlocal->iobref)
                                 mlocal->iobref = iobref_new ();
                         iobref_merge (mlocal->iobref, iobref);
@@ -3280,18 +3389,21 @@ stripe_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  * cause any bugs at higher levels */
                 memcpy (&tmp_stbuf, &mlocal->replies[0].stbuf,
                         sizeof (struct iatt));
+                tmp_stbuf.ia_size = local->stbuf_size;
+                tmp_stbuf.ia_blocks = local->stbuf_blocks;
 
         done:
                 /* */
                 GF_FREE (mlocal->replies);
                 tmp_iobref = mlocal->iobref;
-                fd_unref (mlocal->fd);
-                STACK_UNWIND_STRICT (readv, mframe, op_ret, op_errno, final_vec,
-                                     final_count, &tmp_stbuf, tmp_iobref);
+                /* work around for nfs truncated read. Bug 3774 */
+                tmp_stbuf_p = &tmp_stbuf;
+                WIPE (tmp_stbuf_p);
+                STRIPE_STACK_UNWIND (readv, mframe, op_ret, op_errno, final_vec,
+                                     final_count, &tmp_stbuf, tmp_iobref, NULL);
 
                 iobref_unref (tmp_iobref);
-                if (final_vec)
-                        GF_FREE (final_vec);
+                GF_FREE (final_vec);
         }
 
         goto out;
@@ -3303,11 +3415,11 @@ check_size:
                 STACK_WIND (mframe, stripe_readv_fstat_cbk,
                             (fctx->xl_array[index]),
                             (fctx->xl_array[index])->fops->fstat,
-                            mlocal->fd);
+                            mlocal->fd, NULL);
         }
 
 out:
-        STACK_DESTROY (frame->root);
+        STRIPE_STACK_DESTROY (frame);
 end:
         return 0;
 }
@@ -3315,7 +3427,7 @@ end:
 
 int32_t
 stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
-              size_t size, off_t offset)
+              size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
         int32_t           op_errno = EINVAL;
         int32_t           idx = 0;
@@ -3328,6 +3440,7 @@ stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
         uint64_t          stripe_size = 0;
         off_t             rounded_start = 0;
         off_t             frame_offset = offset;
+	off_t		  dest_offset = 0;
         stripe_local_t   *local = NULL;
         call_frame_t     *rframe = NULL;
         stripe_local_t   *rlocal = NULL;
@@ -3338,7 +3451,7 @@ stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
         VALIDATE_OR_GOTO (fd, err);
         VALIDATE_OR_GOTO (fd->inode, err);
 
-        fd_ctx_get (fd, this, &tmp_fctx);
+        inode_ctx_get (fd->inode, this, &tmp_fctx);
         if (!tmp_fctx) {
                 op_errno = EBADFD;
                 goto err;
@@ -3346,6 +3459,8 @@ stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
         fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
         stripe_size = fctx->stripe_size;
 
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
         if (!stripe_size) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         "Wrong stripe size for the file");
@@ -3360,8 +3475,7 @@ stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
         rounded_end = roof (offset+size, stripe_size);
         num_stripe = (rounded_end- rounded_start)/stripe_size;
 
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
@@ -3369,8 +3483,8 @@ stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
         frame->local = local;
 
         /* This is where all the vectors should be copied. */
-        local->replies = GF_CALLOC (num_stripe, sizeof (struct readv_replies),
-                                    gf_stripe_mt_readv_replies);
+        local->replies = GF_CALLOC (num_stripe, sizeof (struct stripe_replies),
+                                    gf_stripe_mt_stripe_replies);
         if (!local->replies) {
                 op_errno = ENOMEM;
                 goto err;
@@ -3385,8 +3499,7 @@ stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
         for (index = off_index; index < (num_stripe + off_index); index++) {
                 rframe = copy_frame (frame);
-                rlocal = GF_CALLOC (1, sizeof (stripe_local_t),
-                                    gf_stripe_mt_stripe_local_t);
+                rlocal = mem_get0 (this->local_pool);
                 if (!rlocal) {
                         op_errno = ENOMEM;
                         goto err;
@@ -3400,19 +3513,26 @@ stripe_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 rlocal->readv_size = frame_size;
                 rframe->local = rlocal;
                 idx = (index % fctx->stripe_count);
+
+		if (fctx->stripe_coalesce)
+			dest_offset = coalesced_offset(frame_offset,
+				stripe_size, fctx->stripe_count);
+		else
+			dest_offset = frame_offset;
+
                 STACK_WIND (rframe, stripe_readv_cbk, fctx->xl_array[idx],
                             fctx->xl_array[idx]->fops->readv,
-                            fd, frame_size, frame_offset);
+                            fd, frame_size, dest_offset, flags, xdata);
 
                 frame_offset += frame_size;
         }
 
         return 0;
 err:
-        if (local && local->fd)
-                fd_unref (local->fd);
+        if (rframe)
+                STRIPE_STACK_DESTROY (rframe);
 
-        STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0, NULL, NULL);
+        STRIPE_STACK_UNWIND (readv, frame, -1, op_errno, NULL, 0, NULL, NULL, NULL);
         return 0;
 }
 
@@ -3420,11 +3540,15 @@ err:
 int32_t
 stripe_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                   struct iatt *postbuf)
+                   struct iatt *postbuf, dict_t *xdata)
 {
         int32_t         callcnt = 0;
         stripe_local_t *local = NULL;
+	stripe_local_t *mlocal = NULL;
         call_frame_t   *prev = NULL;
+	call_frame_t   *mframe = NULL;
+	struct stripe_replies *reply = NULL;
+	int32_t		i = 0;
 
         if (!this || !frame || !frame->local || !cookie) {
                 gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
@@ -3433,39 +3557,82 @@ stripe_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         prev  = cookie;
         local = frame->local;
+	mframe = local->orig_frame;
+	mlocal = mframe->local;
 
         LOCK(&frame->lock);
         {
-                callcnt = ++local->call_count;
+                callcnt = ++mlocal->call_count;
+
+		mlocal->replies[local->node_index].op_ret = op_ret;
+		mlocal->replies[local->node_index].op_errno = op_errno;
 
-                if (op_ret == -1) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "%s returned error %s",
-                                prev->this->name, strerror (op_errno));
-                        local->op_errno = op_errno;
-                        local->op_ret = -1;
-                }
                 if (op_ret >= 0) {
-                        local->op_ret += op_ret;
-                        local->post_buf = *postbuf;
-                        local->pre_buf = *prebuf;
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
+
+			mlocal->prebuf_blocks  += prebuf->ia_blocks;
+			mlocal->postbuf_blocks += postbuf->ia_blocks;
+
+			correct_file_size(prebuf, mlocal->fctx, prev);
+			correct_file_size(postbuf, mlocal->fctx, prev);
+
+			if (mlocal->prebuf_size < prebuf->ia_size)
+				mlocal->prebuf_size = prebuf->ia_size;
+			if (mlocal->postbuf_size < postbuf->ia_size)
+				mlocal->postbuf_size = postbuf->ia_size;
                 }
         }
         UNLOCK (&frame->lock);
 
-        if ((callcnt == local->wind_count) && local->unwind) {
-                STACK_UNWIND_STRICT (writev, frame, local->op_ret,
-                                     local->op_errno, &local->pre_buf,
-                                     &local->post_buf);
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+		mlocal->post_buf.ia_size = mlocal->postbuf_size;
+		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+		/*
+		 * Only return the number of consecutively written bytes up until
+		 * the first error. Only return an error if it occurs first.
+		 *
+		 * When a short write occurs, the application should retry at the
+		 * appropriate offset, at which point we'll potentially pass back
+		 * the error.
+		 */
+		for (i = 0, reply = mlocal->replies; i < mlocal->wind_count;
+			i++, reply++) {
+			if (reply->op_ret == -1) {
+				gf_log(this->name, GF_LOG_DEBUG, "reply %d "
+					"returned error %s", i,
+					strerror(reply->op_errno));
+				if (!mlocal->op_ret) {
+					mlocal->op_ret = -1;
+					mlocal->op_errno = reply->op_errno;
+				}
+				break;
+			}
+
+			mlocal->op_ret += reply->op_ret;
+
+			if (reply->op_ret < reply->requested_size)
+				break;
+		}
+
+		GF_FREE(mlocal->replies);
+
+                STRIPE_STACK_UNWIND (writev, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
         }
 out:
+	STRIPE_STACK_DESTROY(frame);
         return 0;
 }
 
 int32_t
 stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
                struct iovec *vector, int32_t count, off_t offset,
-               struct iobref *iobref)
+               uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         struct iovec     *tmp_vec = NULL;
         stripe_local_t   *local = NULL;
@@ -3479,13 +3646,19 @@ stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         off_t             fill_size = 0;
         uint64_t          stripe_size = 0;
         uint64_t          tmp_fctx = 0;
+	off_t		  dest_offset = 0;
+	off_t		  rounded_start = 0;
+	off_t		  rounded_end = 0;
+	int32_t		  total_chunks = 0;
+	call_frame_t	  *wframe = NULL;
+	stripe_local_t	  *wlocal = NULL;
 
         VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
         VALIDATE_OR_GOTO (fd->inode, err);
 
-        fd_ctx_get (fd, this, &tmp_fctx);
+        inode_ctx_get (fd->inode, this, &tmp_fctx);
         if (!tmp_fctx) {
                 op_errno = EINVAL;
                 goto err;
@@ -3493,22 +3666,51 @@ stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
         stripe_size = fctx->stripe_size;
 
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
         /* File has to be stripped across the child nodes */
         for (idx = 0; idx< count; idx ++) {
                 total_size += vector[idx].iov_len;
         }
         remaining_size = total_size;
 
-        local = GF_CALLOC (1, sizeof (stripe_local_t),
-                           gf_stripe_mt_stripe_local_t);
+        local = mem_get0 (this->local_pool);
         if (!local) {
                 op_errno = ENOMEM;
                 goto err;
         }
         frame->local = local;
         local->stripe_size = stripe_size;
+	local->fctx = fctx;
+
+        if (!stripe_size) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Wrong stripe size for the file");
+                op_errno = EINVAL;
+                goto err;
+        }
+
+	rounded_start = floor(offset, stripe_size);
+	rounded_end = roof(offset + total_size, stripe_size);
+	total_chunks = (rounded_end - rounded_start) / stripe_size;
+	local->replies = GF_CALLOC(total_chunks, sizeof(struct stripe_replies),
+				gf_stripe_mt_stripe_replies);
+	if (!local->replies) {
+		op_errno = ENOMEM;
+		goto err;
+	}
 
+	total_chunks = 0;
         while (1) {
+		wframe = copy_frame(frame);
+		wlocal = mem_get0(this->local_pool);
+		if (!wlocal) {
+			op_errno = ENOMEM;
+			goto err;
+		}
+		wlocal->orig_frame = frame;
+		wframe->local = wlocal;
+
                 /* Send striped chunk of the vector to child
                    nodes appropriately. */
                 idx = (((offset + offset_offset) /
@@ -3536,32 +3738,554 @@ stripe_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 if (remaining_size == 0)
                         local->unwind = 1;
 
-                STACK_WIND (frame, stripe_writev_cbk, fctx->xl_array[idx],
+		/*
+		 * Store off the request index (with respect to the chunk of the
+		 * initial offset) and the size of the request. This is required
+		 * in the callback to calculate an appropriate return value in
+		 * the event of a write failure in one or more requests.
+		 */
+		wlocal->node_index = total_chunks;
+		local->replies[total_chunks].requested_size = fill_size;
+
+		dest_offset = offset + offset_offset;
+		if (fctx->stripe_coalesce)
+			dest_offset = coalesced_offset(dest_offset,
+					local->stripe_size, fctx->stripe_count);
+
+                STACK_WIND (wframe, stripe_writev_cbk, fctx->xl_array[idx],
                             fctx->xl_array[idx]->fops->writev, fd, tmp_vec,
-                            tmp_count, offset + offset_offset, iobref);
+                            tmp_count, dest_offset, flags, iobref,
+                            xdata);
+
                 GF_FREE (tmp_vec);
                 offset_offset += fill_size;
+		total_chunks++;
                 if (remaining_size == 0)
                         break;
         }
 
         return 0;
 err:
-        STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL);
+	if (wframe)
+		STRIPE_STACK_DESTROY(wframe);
+
+        STRIPE_STACK_UNWIND (writev, frame, -1, op_errno, NULL, NULL, NULL);
         return 0;
 }
 
 
 int32_t
-stripe_release (xlator_t *this, fd_t *fd)
+stripe_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
 {
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+	stripe_local_t *mlocal = NULL;
+        call_frame_t   *prev = NULL;
+	call_frame_t   *mframe = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+	mframe = local->orig_frame;
+	mlocal = mframe->local;
+
+        LOCK(&frame->lock);
+        {
+                callcnt = ++mlocal->call_count;
+
+                if (op_ret == 0) {
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
+
+			mlocal->prebuf_blocks  += prebuf->ia_blocks;
+			mlocal->postbuf_blocks += postbuf->ia_blocks;
+
+			correct_file_size(prebuf, mlocal->fctx, prev);
+			correct_file_size(postbuf, mlocal->fctx, prev);
+
+			if (mlocal->prebuf_size < prebuf->ia_size)
+				mlocal->prebuf_size = prebuf->ia_size;
+			if (mlocal->postbuf_size < postbuf->ia_size)
+				mlocal->postbuf_size = postbuf->ia_size;
+                }
+
+		/* return the first failure */
+		if (mlocal->op_ret == 0) {
+			mlocal->op_ret = op_ret;
+			mlocal->op_errno = op_errno;
+		}
+        }
+        UNLOCK (&frame->lock);
+
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+		mlocal->post_buf.ia_size = mlocal->postbuf_size;
+		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+                STRIPE_STACK_UNWIND (fallocate, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
+        }
+out:
+	STRIPE_STACK_DESTROY(frame);
+        return 0;
+}
+
+int32_t
+stripe_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		 off_t offset, size_t len, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_fd_ctx_t  *fctx = NULL;
+        int32_t           op_errno = 1;
+        int32_t           idx = 0;
+        int32_t           offset_offset = 0;
+        int32_t           remaining_size = 0;
+        off_t             fill_size = 0;
+        uint64_t          stripe_size = 0;
         uint64_t          tmp_fctx = 0;
+	off_t		  dest_offset = 0;
+	call_frame_t	  *fframe = NULL;
+	stripe_local_t	  *flocal = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        inode_ctx_get (fd->inode, this, &tmp_fctx);
+        if (!tmp_fctx) {
+                op_errno = EINVAL;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        frame->local = local;
+        local->stripe_size = stripe_size;
+	local->fctx = fctx;
+
+        if (!stripe_size) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Wrong stripe size for the file");
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        while (1) {
+		fframe = copy_frame(frame);
+		flocal = mem_get0(this->local_pool);
+		if (!flocal) {
+			op_errno = ENOMEM;
+			goto err;
+		}
+		flocal->orig_frame = frame;
+		fframe->local = flocal;
+
+		/* send fallocate request to the associated child node */
+                idx = (((offset + offset_offset) /
+                        local->stripe_size) % fctx->stripe_count);
+
+                fill_size = (local->stripe_size -
+                             ((offset + offset_offset) % local->stripe_size));
+                if (fill_size > remaining_size)
+                        fill_size = remaining_size;
+
+                remaining_size -= fill_size;
+
+                local->wind_count++;
+                if (remaining_size == 0)
+                        local->unwind = 1;
+
+		dest_offset = offset + offset_offset;
+		if (fctx->stripe_coalesce)
+			dest_offset = coalesced_offset(dest_offset,
+					local->stripe_size, fctx->stripe_count);
+
+		/*
+		 * TODO: Create a separate handler for coalesce mode that sends a
+		 * single fallocate per-child (since the ranges are linear).
+		 */
+		STACK_WIND(fframe, stripe_fallocate_cbk, fctx->xl_array[idx],
+			   fctx->xl_array[idx]->fops->fallocate, fd, mode,
+			   dest_offset, fill_size, xdata);
+
+                offset_offset += fill_size;
+                if (remaining_size == 0)
+                        break;
+        }
+
+        return 0;
+err:
+	if (fframe)
+		STRIPE_STACK_DESTROY(fframe);
+
+        STRIPE_STACK_UNWIND (fallocate, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+
+int32_t
+stripe_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        int32_t         callcnt = 0;
+        stripe_local_t *local = NULL;
+	stripe_local_t *mlocal = NULL;
+        call_frame_t   *prev = NULL;
+	call_frame_t   *mframe = NULL;
+
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+	mframe = local->orig_frame;
+	mlocal = mframe->local;
+
+        LOCK(&frame->lock);
+        {
+                callcnt = ++mlocal->call_count;
+
+                if (op_ret == 0) {
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
+
+			mlocal->prebuf_blocks  += prebuf->ia_blocks;
+			mlocal->postbuf_blocks += postbuf->ia_blocks;
+
+			correct_file_size(prebuf, mlocal->fctx, prev);
+			correct_file_size(postbuf, mlocal->fctx, prev);
+
+			if (mlocal->prebuf_size < prebuf->ia_size)
+				mlocal->prebuf_size = prebuf->ia_size;
+			if (mlocal->postbuf_size < postbuf->ia_size)
+				mlocal->postbuf_size = postbuf->ia_size;
+                }
+
+		/* return the first failure */
+		if (mlocal->op_ret == 0) {
+			mlocal->op_ret = op_ret;
+			mlocal->op_errno = op_errno;
+		}
+        }
+        UNLOCK (&frame->lock);
+
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+		mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+		mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+		mlocal->post_buf.ia_size = mlocal->postbuf_size;
+		mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+                STRIPE_STACK_UNWIND (discard, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
+        }
+out:
+	STRIPE_STACK_DESTROY(frame);
+        return 0;
+}
+
+int32_t
+stripe_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+	       size_t len, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
         stripe_fd_ctx_t  *fctx = NULL;
+        int32_t           op_errno = 1;
+        int32_t           idx = 0;
+        int32_t           offset_offset = 0;
+        int32_t           remaining_size = 0;
+        off_t             fill_size = 0;
+        uint64_t          stripe_size = 0;
+        uint64_t          tmp_fctx = 0;
+	off_t		  dest_offset = 0;
+	call_frame_t	  *fframe = NULL;
+	stripe_local_t	  *flocal = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        inode_ctx_get (fd->inode, this, &tmp_fctx);
+        if (!tmp_fctx) {
+                op_errno = EINVAL;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        frame->local = local;
+        local->stripe_size = stripe_size;
+	local->fctx = fctx;
+
+        if (!stripe_size) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Wrong stripe size for the file");
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        while (1) {
+		fframe = copy_frame(frame);
+		flocal = mem_get0(this->local_pool);
+		if (!flocal) {
+			op_errno = ENOMEM;
+			goto err;
+		}
+		flocal->orig_frame = frame;
+		fframe->local = flocal;
+
+		/* send discard request to the associated child node */
+                idx = (((offset + offset_offset) /
+                        local->stripe_size) % fctx->stripe_count);
+
+                fill_size = (local->stripe_size -
+                             ((offset + offset_offset) % local->stripe_size));
+                if (fill_size > remaining_size)
+                        fill_size = remaining_size;
+
+                remaining_size -= fill_size;
+
+                local->wind_count++;
+                if (remaining_size == 0)
+                        local->unwind = 1;
+
+		dest_offset = offset + offset_offset;
+		if (fctx->stripe_coalesce)
+			dest_offset = coalesced_offset(dest_offset,
+					local->stripe_size, fctx->stripe_count);
+
+		/*
+		 * TODO: Create a separate handler for coalesce mode that sends a
+		 * single discard per-child (since the ranges are linear).
+		 */
+		STACK_WIND(fframe, stripe_discard_cbk, fctx->xl_array[idx],
+			   fctx->xl_array[idx]->fops->discard, fd, dest_offset,
+			   fill_size, xdata);
+
+                offset_offset += fill_size;
+                if (remaining_size == 0)
+                        break;
+        }
+
+        return 0;
+err:
+	if (fframe)
+		STRIPE_STACK_DESTROY(fframe);
+
+        STRIPE_STACK_UNWIND (discard, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
+
+int32_t
+stripe_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        int32_t         callcnt    = 0;
+        stripe_local_t *local      = NULL;
+        stripe_local_t *mlocal     = NULL;
+        call_frame_t   *prev       = NULL;
+        call_frame_t   *mframe     = NULL;
+
+        GF_ASSERT (frame);
+
+        if (!this || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+
+        prev  = cookie;
+        local = frame->local;
+        mframe = local->orig_frame;
+        mlocal = mframe->local;
+
+        LOCK(&frame->lock);
+        {
+                callcnt = ++mlocal->call_count;
+
+                if (op_ret == 0) {
+                        mlocal->post_buf = *postbuf;
+                        mlocal->pre_buf = *prebuf;
 
+                        mlocal->prebuf_blocks  += prebuf->ia_blocks;
+                        mlocal->postbuf_blocks += postbuf->ia_blocks;
+
+                        correct_file_size(prebuf, mlocal->fctx, prev);
+                        correct_file_size(postbuf, mlocal->fctx, prev);
+
+                        if (mlocal->prebuf_size < prebuf->ia_size)
+                                mlocal->prebuf_size = prebuf->ia_size;
+                        if (mlocal->postbuf_size < postbuf->ia_size)
+                                mlocal->postbuf_size = postbuf->ia_size;
+                }
+
+                /* return the first failure */
+                if (mlocal->op_ret == 0) {
+                        mlocal->op_ret = op_ret;
+                        mlocal->op_errno = op_errno;
+                }
+        }
+        UNLOCK (&frame->lock);
+
+        if ((callcnt == mlocal->wind_count) && mlocal->unwind) {
+                mlocal->pre_buf.ia_size = mlocal->prebuf_size;
+                mlocal->pre_buf.ia_blocks = mlocal->prebuf_blocks;
+                mlocal->post_buf.ia_size = mlocal->postbuf_size;
+                mlocal->post_buf.ia_blocks = mlocal->postbuf_blocks;
+
+                STRIPE_STACK_UNWIND (zerofill, mframe, mlocal->op_ret,
+                                     mlocal->op_errno, &mlocal->pre_buf,
+                                     &mlocal->post_buf, NULL);
+        }
+out:
+        STRIPE_STACK_DESTROY(frame);
+        return 0;
+}
+
+int32_t
+stripe_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+               off_t len, dict_t *xdata)
+{
+        stripe_local_t   *local            = NULL;
+        stripe_fd_ctx_t  *fctx             = NULL;
+        int32_t           op_errno         = 1;
+        int32_t           idx              = 0;
+        int32_t           offset_offset    = 0;
+        int32_t           remaining_size   = 0;
+        off_t             fill_size        = 0;
+        uint64_t          stripe_size      = 0;
+        uint64_t          tmp_fctx         = 0;
+        off_t             dest_offset      = 0;
+        call_frame_t     *fframe           = NULL;
+        stripe_local_t   *flocal           = NULL;
+
+        VALIDATE_OR_GOTO (frame, err);
         VALIDATE_OR_GOTO (this, err);
         VALIDATE_OR_GOTO (fd, err);
+        VALIDATE_OR_GOTO (fd->inode, err);
+
+        inode_ctx_get (fd->inode, this, &tmp_fctx);
+        if (!tmp_fctx) {
+                op_errno = EINVAL;
+                goto err;
+        }
+        fctx = (stripe_fd_ctx_t *)(long)tmp_fctx;
+        stripe_size = fctx->stripe_size;
+
+        STRIPE_VALIDATE_FCTX (fctx, err);
+
+        remaining_size = len;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        frame->local = local;
+        local->stripe_size = stripe_size;
+        local->fctx = fctx;
+
+        if (!stripe_size) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Wrong stripe size for the file");
+                op_errno = EINVAL;
+                goto err;
+        }
+
+        while (1) {
+                fframe = copy_frame(frame);
+                flocal = mem_get0(this->local_pool);
+                if (!flocal) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
+                flocal->orig_frame = frame;
+                fframe->local = flocal;
+
+                idx = (((offset + offset_offset) /
+                        local->stripe_size) % fctx->stripe_count);
+
+                fill_size = (local->stripe_size -
+                             ((offset + offset_offset) % local->stripe_size));
+                if (fill_size > remaining_size)
+                        fill_size = remaining_size;
+
+                remaining_size -= fill_size;
+
+                local->wind_count++;
+                if (remaining_size == 0)
+                        local->unwind = 1;
+
+                dest_offset = offset + offset_offset;
+                if (fctx->stripe_coalesce)
+                        dest_offset = coalesced_offset(dest_offset,
+                                                       local->stripe_size,
+                                                       fctx->stripe_count);
+
+                STACK_WIND(fframe, stripe_zerofill_cbk, fctx->xl_array[idx],
+                           fctx->xl_array[idx]->fops->zerofill, fd,
+                           dest_offset, fill_size, xdata);
+                offset_offset += fill_size;
+                if (remaining_size == 0)
+                        break;
+        }
+
+        return 0;
+err:
+        if (fframe)
+                STRIPE_STACK_DESTROY(fframe);
+
+        STRIPE_STACK_UNWIND (zerofill, frame, -1, op_errno, NULL, NULL, NULL);
+        return 0;
+}
 
-        fd_ctx_del (fd, this, &tmp_fctx);
+int32_t
+stripe_release (xlator_t *this, fd_t *fd)
+{
+	return 0;
+}
+
+int
+stripe_forget (xlator_t *this, inode_t *inode)
+{
+        uint64_t          tmp_fctx = 0;
+        stripe_fd_ctx_t  *fctx = NULL;
+
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (inode, err);
+
+        (void) inode_ctx_del (inode, this, &tmp_fctx);
         if (!tmp_fctx) {
                 goto err;
         }
@@ -3572,18 +4296,17 @@ stripe_release (xlator_t *this, fd_t *fd)
                 GF_FREE (fctx->xl_array);
 
         GF_FREE (fctx);
-
 err:
-	return 0;
+        return 0;
 }
 
-
 int32_t
 notify (xlator_t *this, int32_t event, void *data, ...)
 {
         stripe_private_t *priv = NULL;
         int               down_client = 0;
         int               i = 0;
+        gf_boolean_t      heard_from_all_children = _gf_false;
 
         if (!this)
                 return 0;
@@ -3601,23 +4324,28 @@ notify (xlator_t *this, int32_t event, void *data, ...)
                         if (data == priv->xl_array[i])
                                 break;
                 }
-                priv->state[i] = 1;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!priv->state[i])
-                                down_client++;
+
+                if (priv->child_count == i) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "got GF_EVENT_CHILD_UP bad subvolume %s",
+                                data? ((xlator_t *)data)->name: NULL);
+                        break;
                 }
 
                 LOCK (&priv->lock);
                 {
-                        priv->nodes_down = down_client;
                         if (data == FIRST_CHILD (this))
                                 priv->first_child_down = 0;
-                        if (!priv->nodes_down)
-                                default_notify (this, event, data);
+                        priv->last_event[i] = event;
                 }
                 UNLOCK (&priv->lock);
         }
         break;
+        case GF_EVENT_CHILD_CONNECTING:
+        {
+                // 'CONNECTING' doesn't ensure its CHILD_UP, so do nothing
+                goto out;
+        }
         case GF_EVENT_CHILD_DOWN:
         {
                 /* get an index number to set */
@@ -3625,20 +4353,19 @@ notify (xlator_t *this, int32_t event, void *data, ...)
                         if (data == priv->xl_array[i])
                                 break;
                 }
-                priv->state[i] = 0;
-                for (i = 0; i < priv->child_count; i++) {
-                        if (!priv->state[i])
-                                down_client++;
+
+                if (priv->child_count == i) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "got GF_EVENT_CHILD_DOWN bad subvolume %s",
+                                data? ((xlator_t *)data)->name: NULL);
+                        break;
                 }
 
                 LOCK (&priv->lock);
                 {
-                        priv->nodes_down = down_client;
-
                         if (data == FIRST_CHILD (this))
                                 priv->first_child_down = 1;
-                        if (priv->nodes_down)
-                                default_notify (this, event, data);
+                        priv->last_event[i] = event;
                 }
                 UNLOCK (&priv->lock);
         }
@@ -3648,67 +4375,252 @@ notify (xlator_t *this, int32_t event, void *data, ...)
         {
                 /* */
                 default_notify (this, event, data);
+                goto out;
         }
         break;
         }
 
+        // Consider child as down if it's last_event is not CHILD_UP
+        for (i = 0, down_client = 0; i < priv->child_count; i++)
+                if (priv->last_event[i] != GF_EVENT_CHILD_UP)
+                        down_client++;
+
+        LOCK (&priv->lock);
+        {
+                priv->nodes_down = down_client;
+        }
+        UNLOCK (&priv->lock);
+
+        heard_from_all_children = _gf_true;
+        for (i = 0; i < priv->child_count; i++)
+                if (!priv->last_event[i])
+                        heard_from_all_children = _gf_false;
+
+        if (heard_from_all_children)
+                default_notify (this, event, data);
+out:
         return 0;
 }
 
 int
-set_stripe_block_size (xlator_t *this, stripe_private_t *priv, char *data)
+stripe_setxattr_cbk (call_frame_t *frame, void *cookie,
+                     xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
 {
-        int                    ret = -1;
-        char                  *tmp_str = NULL;
-        char                  *tmp_str1 = NULL;
-        char                  *dup_str = NULL;
-        char                  *stripe_str = NULL;
-        char                  *pattern = NULL;
-        char                  *num = NULL;
-        struct stripe_options *temp_stripeopt = NULL;
-        struct stripe_options *stripe_opt = NULL;
-
-        if (!this || !priv || !data)
-                goto out;
-
-        /* Get the pattern for striping.
-           "option block-size *avi:10MB" etc */
-        stripe_str = strtok_r (data, ",", &tmp_str);
-        while (stripe_str) {
-                dup_str = gf_strdup (stripe_str);
-                stripe_opt = CALLOC (1, sizeof (struct stripe_options));
-                if (!stripe_opt) {
-                        GF_FREE (dup_str);
-                        goto out;
-                }
+        int ret = -1;
+        int call_cnt = 0;
+        stripe_local_t *local = NULL;
+
+        if (!frame || !frame->local || !this) {
+                gf_log ("", GF_LOG_ERROR, "Possible NULL deref");
+                return ret;
+        }
 
-                pattern = strtok_r (dup_str, ":", &tmp_str1);
-                num = strtok_r (NULL, ":", &tmp_str1);
-                if (!num) {
-                        num = pattern;
-                        pattern = "*";
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                call_cnt = --local->wind_count;
+
+                /**
+                 * We overwrite ->op_* values here for subsequent faliure
+                 * conditions, hence we propogate the last errno down the
+                 * stack.
+                 */
+                if (op_ret < 0) {
+                        local->op_ret = op_ret;
+                        local->op_errno = op_errno;
+                        goto unlock;
                 }
-                if (gf_string2bytesize (num, &stripe_opt->block_size) != 0) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "invalid number format \"%s\"", num);
-                        goto out;
+        }
+
+ unlock:
+        UNLOCK (&frame->lock);
+
+        if (!call_cnt) {
+                STRIPE_STACK_UNWIND (setxattr, frame, local->op_ret,
+                                     local->op_errno, xdata);
+        }
+
+        return 0;
+}
+
+#ifdef HAVE_BD_XLATOR
+int
+stripe_is_bd (dict_t *this, char *key, data_t *value, void *data)
+{
+        gf_boolean_t *is_bd = data;
+
+        if (data == NULL)
+                return 0;
+
+        if (XATTR_IS_BD (key))
+            *is_bd = _gf_true;
+
+        return 0;
+}
+
+static inline gf_boolean_t
+stripe_setxattr_is_bd (dict_t *dict)
+{
+        gf_boolean_t is_bd = _gf_false;
+
+        if (dict == NULL)
+                goto out;
+
+        dict_foreach (dict, stripe_is_bd, &is_bd);
+out:
+        return is_bd;
+}
+#else
+#define stripe_setxattr_is_bd(dict) _gf_false
+#endif
+
+int
+stripe_setxattr (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, dict_t *dict, int flags, dict_t *xdata)
+{
+        int32_t           op_errno = EINVAL;
+        xlator_list_t    *trav     = NULL;
+        stripe_private_t *priv     = NULL;
+        stripe_local_t   *local    = NULL;
+        int               i        = 0;
+        gf_boolean_t      is_bd    = _gf_false;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+
+        GF_IF_INTERNAL_XATTR_GOTO ("trusted.*stripe*", dict,
+                                   op_errno, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        frame->local = local;
+        local->wind_count = priv->child_count;
+        local->op_ret = local->op_errno = 0;
+
+        is_bd = stripe_setxattr_is_bd (dict);
+
+        /**
+         * Set xattrs for directories on all subvolumes. Additionally
+         * this power is only given to a special client. Bd xlator
+         * also needs xattrs for regular files (ie LVs)
+         */
+        if (((frame->root->pid == GF_CLIENT_PID_GSYNCD) &&
+             IA_ISDIR (loc->inode->ia_type)) || is_bd) {
+                for (i = 0; i < priv->child_count; i++, trav = trav->next) {
+                        STACK_WIND (frame, stripe_setxattr_cbk,
+                                    trav->xlator, trav->xlator->fops->setxattr,
+                                    loc, dict, flags, xdata);
                 }
-                memcpy (stripe_opt->path_pattern, pattern, strlen (pattern));
+        } else {
+                local->wind_count = 1;
+                STACK_WIND (frame, stripe_setxattr_cbk,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->setxattr,
+                            loc, dict, flags, xdata);
+        }
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "block-size : pattern %s : size %"PRId64,
-                        stripe_opt->path_pattern, stripe_opt->block_size);
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (setxattr, frame, -1,  op_errno, NULL);
+        return 0;
+}
 
-                if (!priv->pattern) {
-                        priv->pattern = stripe_opt;
-                } else {
-                        temp_stripeopt = priv->pattern;
-                        while (temp_stripeopt->next)
-                                temp_stripeopt = temp_stripeopt->next;
-                        temp_stripeopt->next = stripe_opt;
+
+int
+stripe_fsetxattr_cbk (call_frame_t *frame, void *cookie,
+                      xlator_t *this, int op_ret, int op_errno, dict_t *xdata)
+{
+        STRIPE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+
+int
+stripe_is_special_key (dict_t *this,
+                       char *key,
+                       data_t *value,
+                       void *data)
+{
+        gf_boolean_t *is_special = NULL;
+
+        if (data == NULL) {
+                goto out;
+        }
+
+        is_special = data;
+
+        if (XATTR_IS_LOCKINFO (key) || XATTR_IS_BD (key))
+                *is_special = _gf_true;
+
+out:
+        return 0;
+}
+
+int32_t
+stripe_fsetxattr_everyone_cbk (call_frame_t *frame, void *cookie,
+                               xlator_t *this, int32_t op_ret, int32_t op_errno,
+                               dict_t *xdata)
+{
+        int             call_count = 0;
+        stripe_local_t *local      = NULL;
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                call_count = --local->wind_count;
+
+                if (op_ret < 0) {
+                        local->op_ret = op_ret;
+                        local->op_errno = op_errno;
                 }
-                stripe_str = strtok_r (NULL, ",", &tmp_str);
-                GF_FREE (dup_str);
+        }
+        UNLOCK (&frame->lock);
+
+        if (call_count == 0) {
+                STRIPE_STACK_UNWIND (fsetxattr, frame, local->op_ret,
+                                     local->op_errno, NULL);
+        }
+        return 0;
+}
+
+int
+stripe_fsetxattr_to_everyone (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                              dict_t *dict, int flags, dict_t *xdata)
+{
+        xlator_list_t    *trav  = NULL;
+        stripe_private_t *priv  = NULL;
+        int               ret   = -1;
+        stripe_local_t   *local = NULL;
+
+        priv = this->private;
+
+        local = mem_get0 (this->local_pool);
+        if (local == NULL) {
+                goto out;
+        }
+
+        frame->local = local;
+
+        local->wind_count = priv->child_count;
+
+        trav = this->children;
+
+        while (trav) {
+                STACK_WIND (frame, stripe_fsetxattr_everyone_cbk,
+                            trav->xlator, trav->xlator->fops->fsetxattr,
+                            fd, dict, flags, xdata);
+                trav = trav->next;
         }
 
         ret = 0;
@@ -3716,97 +4628,477 @@ out:
         return ret;
 }
 
-int32_t
-mem_acct_init (xlator_t *this)
+static inline gf_boolean_t
+stripe_fsetxattr_is_special (dict_t *dict)
 {
-        int     ret = -1;
+        gf_boolean_t is_spl = _gf_false;
 
-        if (!this)
+        if (dict == NULL) {
                 goto out;
+        }
 
-        ret = xlator_mem_acct_init (this, gf_stripe_mt_end + 1);
+        dict_foreach (dict, stripe_is_special_key, &is_spl);
+
+out:
+        return is_spl;
+}
+
+int
+stripe_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  dict_t *dict, int flags, dict_t *xdata)
+{
+        int32_t      op_ret = -1, ret = -1, op_errno = EINVAL;
+        gf_boolean_t is_spl = _gf_false;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        GF_IF_INTERNAL_XATTR_GOTO ("trusted.*stripe*", dict,
+                                   op_errno, err);
+
+        is_spl = stripe_fsetxattr_is_special (dict);
+        if (is_spl) {
+                ret = stripe_fsetxattr_to_everyone (frame, this, fd, dict,
+                                                    flags, xdata);
+                if (ret < 0) {
+                        op_errno = ENOMEM;
+                        goto err;
+                }
 
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                        "failed");
                 goto out;
         }
 
+        STACK_WIND (frame, stripe_fsetxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr,
+                    fd, dict, flags, xdata);
 out:
-        return ret;
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, NULL);
+        return 0;
 }
+
 int
-validate_options (xlator_t *this, dict_t *options, char **op_errstr)
+stripe_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        STRIPE_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
 
+int
+stripe_removexattr (call_frame_t *frame, xlator_t *this,
+                    loc_t *loc, const char *name, dict_t *xdata)
+{
+        int32_t         op_errno = EINVAL;
 
-        data_t           *data = NULL;
-        int               ret = 0;
+        VALIDATE_OR_GOTO (this, err);
+
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.*stripe*",
+                                 name, op_errno, err);
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (loc, err);
+
+        STACK_WIND (frame, stripe_removexattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr,
+                    loc, name, xdata);
+        return 0;
+err:
+        STRIPE_STACK_UNWIND (removexattr, frame, -1,  op_errno, NULL);
+        return 0;
+}
+
+
+int
+stripe_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        STRIPE_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int
+stripe_fremovexattr (call_frame_t *frame, xlator_t *this,
+                     fd_t *fd, const char *name, dict_t *xdata)
+{
+        int32_t         op_ret   = -1;
+        int32_t         op_errno = EINVAL;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        GF_IF_NATIVE_XATTR_GOTO ("trusted.*stripe*",
+                                 name, op_errno, err);
+
+        STACK_WIND (frame, stripe_fremovexattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fremovexattr,
+                    fd, name, xdata);
+        return 0;
+ err:
+        STRIPE_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+stripe_readdirp_lookup_cbk (call_frame_t *frame, void *cookie,
+                            xlator_t *this, int op_ret, int op_errno,
+                            inode_t *inode, struct iatt *stbuf,
+                            dict_t *xattr, struct iatt *parent)
+{
+        stripe_local_t          *local          = NULL;
+        call_frame_t            *main_frame     = NULL;
+        stripe_local_t          *main_local     = NULL;
+        gf_dirent_t             *entry          = NULL;
+        call_frame_t            *prev           = NULL;
+        int                      done           = 0;
+
+        local = frame->local;
+        prev = cookie;
+
+        entry = local->dirent;
+
+        main_frame = local->orig_frame;
+        main_local = main_frame->local;
+        LOCK (&frame->lock);
+        {
+
+                local->call_count--;
+                if (!local->call_count)
+                        done = 1;
+                if (op_ret == -1) {
+                        local->op_errno = op_errno;
+                        local->op_ret = op_ret;
+                        goto unlock;
+                }
+
+		if (stripe_ctx_handle(this, prev, local, xattr))
+			gf_log(this->name, GF_LOG_ERROR,
+				"Error getting fctx info from dict.");
+
+		correct_file_size(stbuf, local->fctx, prev);
+
+                stripe_iatt_merge (stbuf, &entry->d_stat);
+                local->stbuf_blocks += stbuf->ia_blocks;
+        }
+unlock:
+        UNLOCK(&frame->lock);
+
+        if (done) {
+                inode_ctx_put (entry->inode, this,
+                               (uint64_t) (long)local->fctx);
+
+                done = 0;
+                LOCK (&main_frame->lock);
+                {
+                        main_local->wind_count--;
+                        if (!main_local->wind_count)
+                                done = 1;
+                        if (local->op_ret == -1) {
+                                main_local->op_errno = local->op_errno;
+                                main_local->op_ret = local->op_ret;
+                        }
+                        entry->d_stat.ia_blocks = local->stbuf_blocks;
+                }
+                UNLOCK (&main_frame->lock);
+                if (done) {
+                        main_frame->local = NULL;
+                        STRIPE_STACK_UNWIND (readdir, main_frame,
+                                             main_local->op_ret,
+                                             main_local->op_errno,
+                                             &main_local->entries, NULL);
+                        gf_dirent_free (&main_local->entries);
+                        stripe_local_wipe (main_local);
+                        mem_put (main_local);
+                }
+                frame->local = NULL;
+                stripe_local_wipe (local);
+                mem_put (local);
+                STRIPE_STACK_DESTROY (frame);
+        }
+
+        return 0;
+}
+
+int32_t
+stripe_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno,
+                     gf_dirent_t *orig_entries, dict_t *xdata)
+{
+        stripe_local_t *local = NULL;
+        call_frame_t   *prev = NULL;
+        gf_dirent_t    *local_entry = NULL;
+        gf_dirent_t    *tmp_entry = NULL;
+        xlator_list_t  *trav = NULL;
+        loc_t          loc = {0, };
+        int32_t        count = 0;
         stripe_private_t *priv = NULL;
+        int32_t        subvols = 0;
+        dict_t         *xattrs = NULL;
+        call_frame_t   *local_frame = NULL;
+        stripe_local_t *local_ent = NULL;
 
-        data = dict_get (options, "block-size");
-        if (data) {
-                gf_log (this->name, GF_LOG_TRACE,"Reconfiguring Stripe"
-                                " Block-size");
-                priv = GF_CALLOC (1, sizeof (stripe_private_t),
-                                  gf_stripe_mt_stripe_private_t);
-                if (!priv) {
-                        gf_log ("",GF_LOG_ERROR, "Unable to allocate memory");
-                        ret = -1;
+        if (!this || !frame || !frame->local || !cookie) {
+                gf_log ("stripe", GF_LOG_DEBUG, "possible NULL deref");
+                goto out;
+        }
+        prev  = cookie;
+        local = frame->local;
+        trav = this->children;
+        priv = this->private;
+
+        subvols = priv->child_count;
+
+        LOCK (&frame->lock);
+        {
+                if (op_ret == -1) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "%s returned error %s",
+                                prev->this->name, strerror (op_errno));
+                        local->op_errno = op_errno;
+                        local->op_ret = op_ret;
+                        goto unlock;
+                } else {
+                        local->op_ret = op_ret;
+                        list_splice_init (&orig_entries->list,
+                                          &local->entries.list);
+                        local->wind_count = op_ret;
+                }
+
+        }
+unlock:
+        UNLOCK (&frame->lock);
+
+        if (op_ret == -1)
+                goto out;
+
+        xattrs = dict_new ();
+        if (xattrs)
+                (void) stripe_xattr_request_build (this, xattrs, 0, 0, 0, 0);
+        count = op_ret;
+        list_for_each_entry_safe (local_entry, tmp_entry,
+                                  (&local->entries.list), list) {
+
+                if (!local_entry)
+                        break;
+                if (!IA_ISREG (local_entry->d_stat.ia_type) || !local_entry->inode) {
+                        LOCK (&frame->lock);
+                        {
+                                local->wind_count--;
+                                count = local->wind_count;
+                        }
+                        UNLOCK (&frame->lock);
+                        continue;
+                }
+
+                local_frame = copy_frame (frame);
+
+                if (!local_frame) {
+                        op_errno = ENOMEM;
+                        op_ret = -1;
                         goto out;
                 }
 
-                ret = set_stripe_block_size (this, priv, data->data);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "Reconfigue: Block-Size reconfiguration failed");
-                        *op_errstr = gf_strdup ("Error, could not parse list");
-                        ret = -1;
+                local_ent = mem_get0 (this->local_pool);
+                if (!local_ent) {
+                        op_errno = ENOMEM;
+                        op_ret = -1;
                         goto out;
                 }
-                gf_log (this->name, GF_LOG_TRACE,
-                        "Reconfigue: Block-Size reconfigured Successfully");
+
+                loc.inode = inode_ref (local_entry->inode);
+
+                uuid_copy (loc.gfid, local_entry->d_stat.ia_gfid);
+
+                local_ent->orig_frame = frame;
+
+                local_ent->call_count = subvols;
+
+                local_ent->dirent = local_entry;
+
+                local_frame->local = local_ent;
+
+                trav = this->children;
+                while (trav) {
+                        STACK_WIND (local_frame, stripe_readdirp_lookup_cbk,
+                                    trav->xlator, trav->xlator->fops->lookup,
+                                    &loc, xattrs);
+                        trav = trav->next;
+                }
+                loc_wipe (&loc);
+        }
+out:
+        if (!count) {
+                /* all entries are directories */
+                frame->local = NULL;
+                STRIPE_STACK_UNWIND (readdir, frame, local->op_ret,
+                                     local->op_errno, &local->entries, NULL);
+                gf_dirent_free (&local->entries);
+                stripe_local_wipe (local);
+                mem_put (local);
+        }
+        if (xattrs)
+                dict_unref (xattrs);
+        return 0;
+
+}
+int32_t
+stripe_readdirp (call_frame_t *frame, xlator_t *this,
+                 fd_t *fd, size_t size, off_t off, dict_t *xdata)
+{
+        stripe_local_t  *local  = NULL;
+        stripe_private_t *priv = NULL;
+        xlator_list_t   *trav = NULL;
+        int             op_errno = -1;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (fd, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        if (priv->first_child_down) {
+                op_errno = ENOTCONN;
+                goto err;
+        }
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        frame->local = local;
+
+        local->fd = fd_ref (fd);
+
+        local->wind_count = 0;
+
+        local->count = 0;
+        local->op_ret = -1;
+        INIT_LIST_HEAD(&local->entries);
+
+        if (!trav)
+                goto err;
+
+        STACK_WIND (frame, stripe_readdirp_cbk, trav->xlator,
+                    trav->xlator->fops->readdirp, fd, size, off, xdata);
+        return 0;
+err:
+        op_errno = (op_errno == -1) ? errno : op_errno;
+        STRIPE_STACK_UNWIND (readdir, frame, -1, op_errno, NULL, NULL);
+
+        return 0;
+
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                goto out;
+
+        ret = xlator_mem_acct_init (this, gf_stripe_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                        "failed");
+                goto out;
         }
 
 out:
-                if (priv)
-                GF_FREE (priv);
         return ret;
+}
+
+static int
+clear_pattern_list (stripe_private_t *priv)
+{
+        struct stripe_options *prev = NULL;
+        struct stripe_options *trav = NULL;
+        int                    ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("stripe", priv, out);
+
+        trav = priv->pattern;
+        priv->pattern = NULL;
+        while (trav) {
+                prev = trav;
+                trav = trav->next;
+                GF_FREE (prev);
+        }
+
+        ret = 0;
+ out:
+        return ret;
+
 
 }
 
+
 int
 reconfigure (xlator_t *this, dict_t *options)
 {
 
-	stripe_private_t *priv = NULL;
-	data_t           *data = NULL;
-	int 		  ret = 0;
+        stripe_private_t *priv = NULL;
+        data_t           *data = NULL;
+        int               ret = -1;
+        volume_option_t  *opt = NULL;
 
+        GF_ASSERT (this);
+        GF_ASSERT (this->private);
 
-	priv = this->private;
+        priv = this->private;
 
-	data = dict_get (options, "block-size");
-        if (data) {
-		gf_log (this->name, GF_LOG_TRACE,"Reconfiguring Stripe"
-			" Block-size");
-                ret = set_stripe_block_size (this, priv, data->data);
-                if (ret) {
-			gf_log (this->name, GF_LOG_ERROR,
-                        "Reconfigue: Block-Size reconfiguration failed");
-                        ret = -1;
-			goto out;
-		}
-		gf_log (this->name, GF_LOG_TRACE,
-                        "Reconfigue: Block-Size reconfigured Successfully");
-	}
-        else {
-                priv->block_size = (128 * GF_UNIT_KB);
+
+        ret = 0;
+        LOCK (&priv->lock);
+        {
+                ret = clear_pattern_list (priv);
+                if (ret)
+                         goto unlock;
+
+                data = dict_get (options, "block-size");
+                if (data) {
+                        ret = set_stripe_block_size (this, priv, data->data);
+                        if (ret)
+                                goto unlock;
+                } else {
+                        opt = xlator_volume_option_get (this, "block-size");
+                        if (!opt) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "option 'block-size' not found");
+                                ret = -1;
+                                goto unlock;
+                        }
+
+                        if (gf_string2bytesize_uint64 (opt->default_value, &priv->block_size)){
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Unable to set default block-size ");
+                                ret = -1;
+                                goto unlock;
+                        }
+                }
+
+		GF_OPTION_RECONF("coalesce", priv->coalesce, options, bool,
+				unlock);
         }
-                        
+ unlock:
+        UNLOCK (&priv->lock);
+        if (ret)
+                goto out;
 
-out:
-	return ret;
+        ret = 0;
+ out:
+        return ret;
 
 }
 
@@ -3819,6 +5111,7 @@ int32_t
 init (xlator_t *this)
 {
         stripe_private_t *priv = NULL;
+        volume_option_t  *opt = NULL;
         xlator_list_t    *trav = NULL;
         data_t           *data = NULL;
         int32_t           count = 0;
@@ -3862,9 +5155,9 @@ init (xlator_t *this)
         if (!priv->xl_array)
                 goto out;
 
-        priv->state = GF_CALLOC (count, sizeof (int8_t),
-                                 gf_stripe_mt_int8_t);
-        if (!priv->state)
+        priv->last_event = GF_CALLOC (count, sizeof (int),
+                                      gf_stripe_mt_int32_t);
+        if (!priv->last_event)
                 goto out;
 
         priv->child_count = count;
@@ -3884,41 +5177,56 @@ init (xlator_t *this)
                 goto out;
         }
 
-        priv->block_size = (128 * GF_UNIT_KB);
-        /* option stripe-pattern *avi:1GB,*pdf:4096 */
-        data = dict_get (this->options, "block-size");
-        if (!data) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "No \"option block-size <x>\" given, defaulting "
-                        "to 128KB");
-        } else {
-                ret = set_stripe_block_size (this, priv, data->data);
-                if (ret)
-                        goto out;
-        }
-
-        priv->xattr_supported = 1;
-        data = dict_get (this->options, "use-xattr");
-        if (data) {
-                if (gf_string2boolean (data->data,
-                                       &priv->xattr_supported) == -1) {
+        ret = 0;
+        LOCK (&priv->lock);
+        {
+                opt = xlator_volume_option_get (this, "block-size");
+                if (!opt) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "option 'block-size' not found");
+                        ret = -1;
+                        goto unlock;
+                }
+                if (gf_string2bytesize_uint64 (opt->default_value, &priv->block_size)){
                         gf_log (this->name, GF_LOG_ERROR,
-                                "error setting hard check for extended "
-                                "attribute");
-                        //return -1;
+                                "Unable to set default block-size ");
+                        ret = -1;
+                        goto unlock;
+                }
+                /* option stripe-pattern *avi:1GB,*pdf:16K */
+                data = dict_get (this->options, "block-size");
+                if (data) {
+                        ret = set_stripe_block_size (this, priv, data->data);
+                        if (ret)
+                                goto unlock;
                 }
         }
+ unlock:
+        UNLOCK (&priv->lock);
+        if (ret)
+                goto out;
 
+        GF_OPTION_INIT ("use-xattr", priv->xattr_supported, bool, out);
         /* notify related */
         priv->nodes_down = priv->child_count;
+
+	GF_OPTION_INIT("coalesce", priv->coalesce, bool, out);
+
+        this->local_pool = mem_pool_new (stripe_local_t, 128);
+        if (!this->local_pool) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto out;
+        }
+
         this->private = priv;
 
         ret = 0;
 out:
         if (ret) {
                 if (priv) {
-                        if (priv->xl_array)
-                                GF_FREE (priv->xl_array);
+                        GF_FREE (priv->xl_array);
                         GF_FREE (priv);
                 }
         }
@@ -3942,15 +5250,15 @@ fini (xlator_t *this)
         priv = this->private;
         if (priv) {
                 this->private = NULL;
-                if (priv->xl_array)
-                        GF_FREE (priv->xl_array);
+                GF_FREE (priv->xl_array);
 
                 trav = priv->pattern;
                 while (trav) {
                         prev = trav;
                         trav = trav->next;
-                        FREE (prev);
+                        GF_FREE (prev);
                 }
+                GF_FREE (priv->last_event);
                 LOCK_DESTROY (&priv->lock);
                 GF_FREE (priv);
         }
@@ -3959,44 +5267,524 @@ out:
         return;
 }
 
+int32_t
+stripe_getxattr_unwind (call_frame_t *frame,
+                        int op_ret, int op_errno, dict_t *dict, dict_t *xdata)
+
+{
+        STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
+}
+
+int
+stripe_internal_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                              int op_ret, int op_errno, dict_t *xattr,
+                              dict_t *xdata)
+{
+
+        char        size_key[256]  = {0,};
+        char        index_key[256] = {0,};
+        char        count_key[256] = {0,};
+        char        coalesce_key[256] = {0,};
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (frame->local, out);
+
+        if (!xattr || (op_ret == -1))
+            goto out;
+
+        sprintf (size_key, "trusted.%s.stripe-size", this->name);
+        sprintf (count_key, "trusted.%s.stripe-count", this->name);
+        sprintf (index_key, "trusted.%s.stripe-index", this->name);
+	sprintf (coalesce_key, "trusted.%s.stripe-coalesce", this->name);
+
+        dict_del (xattr, size_key);
+        dict_del (xattr, count_key);
+        dict_del (xattr, index_key);
+        dict_del (xattr, coalesce_key);
+
+out:
+        STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, xattr, xdata);
+
+        return 0;
+
+}
+
+int
+stripe_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int op_ret, int op_errno, dict_t *xattr, dict_t *xdata)
+{
+        int                     call_cnt = 0;
+        stripe_local_t         *local = NULL;
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (frame->local, out);
+
+        local = frame->local;
+
+        LOCK (&frame->lock);
+        {
+                call_cnt = --local->wind_count;
+        }
+        UNLOCK (&frame->lock);
+
+        if (!xattr || (op_ret < 0))
+                goto out;
+
+        local->op_ret = 0;
+
+        if (!local->xattr) {
+                local->xattr = dict_ref (xattr);
+        } else {
+                stripe_aggregate_xattr (local->xattr, xattr);
+        }
+
+out:
+        if (!call_cnt) {
+                STRIPE_STACK_UNWIND (getxattr, frame, local->op_ret, op_errno,
+                                     local->xattr, xdata);
+        }
+
+        return 0;
+}
+
+int32_t
+stripe_vgetxattr_cbk (call_frame_t *frame, void *cookie,
+                      xlator_t *this, int32_t op_ret, int32_t op_errno,
+                      dict_t *dict, dict_t *xdata)
+{
+        stripe_local_t      *local         = NULL;
+        int32_t              callcnt       = 0;
+        int32_t              ret           = -1;
+        long                 cky           = 0;
+        void                *xattr_val     = NULL;
+        void                *xattr_serz    = NULL;
+        stripe_xattr_sort_t *xattr         = NULL;
+        dict_t              *stripe_xattr  = NULL;
+
+        if (!frame || !frame->local || !this) {
+                gf_log ("", GF_LOG_ERROR, "Possible NULL deref");
+                return ret;
+        }
+
+        local = frame->local;
+        cky = (long) cookie;
+
+        if (local->xsel[0] == '\0') {
+                gf_log (this->name, GF_LOG_ERROR, "Empty xattr in cbk");
+                return ret;
+        }
+
+        LOCK (&frame->lock);
+        {
+                callcnt = --local->wind_count;
+
+                if (!dict || (op_ret < 0))
+                        goto out;
+
+                if (!local->xattr_list)
+                        local->xattr_list = (stripe_xattr_sort_t *)
+                                GF_CALLOC (local->nallocs,
+                                           sizeof (stripe_xattr_sort_t),
+                                           gf_stripe_mt_xattr_sort_t);
+
+                if (local->xattr_list) {
+                        xattr = local->xattr_list + (int32_t) cky;
+
+                        ret = dict_get_ptr_and_len (dict, local->xsel,
+                                                    &xattr_val,
+                                                    &xattr->xattr_len);
+                        if (xattr->xattr_len == 0)
+                                goto out;
+
+                        xattr->pos = cky;
+                        xattr->xattr_value = gf_memdup (xattr_val,
+                                                        xattr->xattr_len);
+
+                        if (xattr->xattr_value != NULL)
+                                local->xattr_total_len += xattr->xattr_len + 1;
+                }
+        }
+ out:
+        UNLOCK (&frame->lock);
+
+        if (!callcnt) {
+                if (!local->xattr_total_len)
+                        goto unwind;
+
+                stripe_xattr = dict_new ();
+                if (!stripe_xattr)
+                        goto unwind;
+
+                /* select filler based on ->xsel */
+                if (XATTR_IS_PATHINFO (local->xsel))
+                        ret = stripe_fill_pathinfo_xattr (this, local,
+                                                          (char **)&xattr_serz);
+                else if (XATTR_IS_LOCKINFO (local->xsel)) {
+                        ret = stripe_fill_lockinfo_xattr (this, local,
+                                                          &xattr_serz);
+                } else {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Unknown xattr in xattr request");
+                        goto unwind;
+                }
+
+                if (!ret) {
+                        ret = dict_set_dynptr (stripe_xattr, local->xsel,
+                                               xattr_serz,
+                                               local->xattr_total_len);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Can't set %s key in dict",
+                                        local->xsel);
+                }
+
+        unwind:
+                STRIPE_STACK_UNWIND (getxattr, frame, op_ret, op_errno,
+                                     stripe_xattr, NULL);
+
+                ret = stripe_free_xattr_str (local);
+
+                GF_FREE (local->xattr_list);
+
+                if (stripe_xattr)
+                        dict_unref (stripe_xattr);
+        }
+
+        return ret;
+}
+
+int32_t
+stripe_getxattr (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, const char *name, dict_t *xdata)
+{
+        stripe_local_t    *local    = NULL;
+        xlator_list_t     *trav     = NULL;
+        stripe_private_t  *priv     = NULL;
+        int32_t            op_errno = EINVAL;
+        int                i        = 0;
+        xlator_t         **sub_volumes;
+        int                ret      = 0;
+
+        VALIDATE_OR_GOTO (frame, err);
+        VALIDATE_OR_GOTO (this, err);
+        VALIDATE_OR_GOTO (loc, err);
+        VALIDATE_OR_GOTO (loc->path, err);
+        VALIDATE_OR_GOTO (loc->inode, err);
+
+        priv = this->private;
+        trav = this->children;
+
+        /* Initialization */
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+        local->op_ret = -1;
+        frame->local = local;
+        loc_copy (&local->loc, loc);
+
+
+        if (name && (strcmp (GF_XATTR_MARKER_KEY, name) == 0)
+            && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+                local->marker.call_count = priv->child_count;
+
+                sub_volumes = alloca ( priv->child_count *
+                                       sizeof (xlator_t *));
+                for (i = 0, trav = this->children; trav ;
+                     trav = trav->next, i++) {
+
+                        *(sub_volumes + i)  = trav->xlator;
+
+                }
+
+                if (cluster_getmarkerattr (frame, this, loc, name,
+                                           local, stripe_getxattr_unwind,
+                                           sub_volumes, priv->child_count,
+                                           MARKER_UUID_TYPE, marker_uuid_default_gauge,
+                                           priv->vol_uuid)) {
+                        op_errno = EINVAL;
+                        goto err;
+                }
+
+                return 0;
+        }
+
+        if (name && strncmp (name, GF_XATTR_QUOTA_SIZE_KEY,
+                             strlen (GF_XATTR_QUOTA_SIZE_KEY)) == 0) {
+                local->wind_count = priv->child_count;
+
+                for (i = 0, trav=this->children; i < priv->child_count; i++,
+                             trav = trav->next) {
+                        STACK_WIND (frame, stripe_getxattr_cbk,
+                                    trav->xlator, trav->xlator->fops->getxattr,
+                                    loc, name, xdata);
+                }
+
+                return 0;
+        }
+
+        if (name && (XATTR_IS_PATHINFO (name))) {
+                if (IA_ISREG (loc->inode->ia_type)) {
+                        ret = inode_ctx_get (loc->inode, this,
+                                             (uint64_t *) &local->fctx);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "stripe size unavailable from fctx"
+                                        " relying on pathinfo could lead to"
+                                        " wrong results");
+                }
+
+                local->nallocs = local->wind_count = priv->child_count;
+                (void) strncpy (local->xsel, name, strlen (name));
+
+                /**
+                 * for xattrs that need info from all childs, fill ->xsel
+                 * as above and call the filler function in cbk based on
+                 * it
+                 */
+                for (i = 0, trav = this->children; i < priv->child_count; i++,
+                     trav = trav->next) {
+                        STACK_WIND_COOKIE (frame, stripe_vgetxattr_cbk,
+                                           (void *) (long) i, trav->xlator,
+                                           trav->xlator->fops->getxattr,
+                                           loc, name, xdata);
+                }
+
+                return 0;
+        }
+
+        if (name &&(*priv->vol_uuid)) {
+                if ((match_uuid_local (name, priv->vol_uuid) == 0)
+                    && (GF_CLIENT_PID_GSYNCD == frame->root->pid)) {
+
+                        if (!IA_FILE_OR_DIR (loc->inode->ia_type))
+                                local->marker.call_count = 1;
+                        else
+                                local->marker.call_count = priv->child_count;
+
+                        sub_volumes = alloca (local->marker.call_count *
+                                              sizeof (xlator_t *));
+
+                        for (i = 0, trav = this->children;
+                             i < local->marker.call_count;
+                             i++, trav = trav->next) {
+                                *(sub_volumes + i) = trav->xlator;
+
+                        }
+
+                        if (cluster_getmarkerattr (frame, this, loc, name,
+                                                   local,
+                                                   stripe_getxattr_unwind,
+                                                   sub_volumes,
+                                                   local->marker.call_count,
+                                                   MARKER_XTIME_TYPE,
+                                                   marker_xtime_default_gauge,
+                                                   priv->vol_uuid)) {
+                                op_errno = EINVAL;
+                                goto err;
+                        }
+
+                        return 0;
+                }
+        }
+
+
+        STACK_WIND (frame, stripe_internal_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+
+        return 0;
+
+err:
+        STRIPE_STACK_UNWIND (getxattr, frame, -1, op_errno, NULL, NULL);
+        return 0;
+}
+
+static inline gf_boolean_t
+stripe_is_special_xattr (const char *name)
+{
+        gf_boolean_t    is_spl = _gf_false;
+
+        if (!name) {
+                goto out;
+        }
+
+        if (!strncmp (name, GF_XATTR_LOCKINFO_KEY,
+                      strlen (GF_XATTR_LOCKINFO_KEY))
+            || XATTR_IS_PATHINFO (name))
+                is_spl = _gf_true;
+out:
+        return is_spl;
+}
+
+int32_t
+stripe_fgetxattr_from_everyone (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                                const char *name, dict_t *xdata)
+{
+        stripe_local_t   *local = NULL;
+        stripe_private_t *priv  = NULL;
+        int32_t           ret   = -1, op_errno = 0;
+        int               i     = 0;
+        xlator_list_t    *trav  = NULL;
+
+        priv = this->private;
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                op_errno = ENOMEM;
+                goto err;
+        }
+
+        local->op_ret = -1;
+        frame->local = local;
+
+        strncpy (local->xsel, name, strlen (name));
+        local->nallocs = local->wind_count = priv->child_count;
+
+        for (i = 0, trav = this->children; i < priv->child_count; i++,
+                     trav = trav->next) {
+                STACK_WIND_COOKIE (frame, stripe_vgetxattr_cbk,
+                                   (void *) (long) i, trav->xlator,
+                                   trav->xlator->fops->fgetxattr,
+                                   fd, name, xdata);
+        }
+
+        return 0;
+
+err:
+        STACK_UNWIND_STRICT (fgetxattr, frame, -1, op_errno, NULL, NULL);
+        return ret;
+}
+
+int32_t
+stripe_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                  const char *name, dict_t *xdata)
+{
+        if (stripe_is_special_xattr (name)) {
+                stripe_fgetxattr_from_everyone (frame, this, fd, name, xdata);
+                goto out;
+        }
+
+        STACK_WIND (frame, stripe_internal_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+
+out:
+        return 0;
+}
+
+
+
+int32_t
+stripe_priv_dump (xlator_t *this)
+{
+        char                    key[GF_DUMP_MAX_BUF_LEN];
+        int                     i = 0;
+        stripe_private_t       *priv = NULL;
+        int                     ret = -1;
+        struct stripe_options  *options = NULL;
+
+        GF_VALIDATE_OR_GOTO ("stripe", this, out);
+
+        priv = this->private;
+        if (!priv)
+                goto out;
+
+        ret = TRY_LOCK (&priv->lock);
+        if (ret != 0)
+                goto out;
+
+        gf_proc_dump_add_section("xlator.cluster.stripe.%s.priv", this->name);
+        gf_proc_dump_write("child_count","%d", priv->child_count);
+
+        for (i = 0; i < priv->child_count; i++) {
+                sprintf (key, "subvolumes[%d]", i);
+                gf_proc_dump_write (key, "%s.%s", priv->xl_array[i]->type,
+                                    priv->xl_array[i]->name);
+        }
+
+        options = priv->pattern;
+        while (options != NULL) {
+                gf_proc_dump_write ("path_pattern", "%s", priv->pattern->path_pattern);
+                gf_proc_dump_write ("options_block_size", "%ul", options->block_size);
+
+                options = options->next;
+        }
+
+        gf_proc_dump_write ("block_size", "%ul", priv->block_size);
+        gf_proc_dump_write ("nodes-down", "%d", priv->nodes_down);
+        gf_proc_dump_write ("first-child_down", "%d", priv->first_child_down);
+        gf_proc_dump_write ("xattr_supported", "%d", priv->xattr_supported);
+
+        UNLOCK (&priv->lock);
+
+out:
+        return ret;
+}
 
 struct xlator_fops fops = {
-        .stat        = stripe_stat,
-        .unlink      = stripe_unlink,
-        .rename      = stripe_rename,
-        .link        = stripe_link,
-        .truncate    = stripe_truncate,
-        .create      = stripe_create,
-        .open        = stripe_open,
-        .readv       = stripe_readv,
-        .writev      = stripe_writev,
-        .statfs      = stripe_statfs,
-        .flush       = stripe_flush,
-        .fsync       = stripe_fsync,
-        .ftruncate   = stripe_ftruncate,
-        .fstat       = stripe_fstat,
-        .mkdir       = stripe_mkdir,
-        .rmdir       = stripe_rmdir,
-        .lk          = stripe_lk,
-        .opendir     = stripe_opendir,
-        .fsyncdir    = stripe_fsyncdir,
-        .setattr     = stripe_setattr,
-        .fsetattr    = stripe_fsetattr,
-        .lookup      = stripe_lookup,
-        .mknod       = stripe_mknod,
+        .stat           = stripe_stat,
+        .unlink         = stripe_unlink,
+        .rename         = stripe_rename,
+        .link           = stripe_link,
+        .truncate       = stripe_truncate,
+        .create         = stripe_create,
+        .open           = stripe_open,
+        .readv          = stripe_readv,
+        .writev         = stripe_writev,
+        .statfs         = stripe_statfs,
+        .flush          = stripe_flush,
+        .fsync          = stripe_fsync,
+        .ftruncate      = stripe_ftruncate,
+        .fstat          = stripe_fstat,
+        .mkdir          = stripe_mkdir,
+        .rmdir          = stripe_rmdir,
+        .lk             = stripe_lk,
+        .opendir        = stripe_opendir,
+        .fsyncdir       = stripe_fsyncdir,
+        .setattr        = stripe_setattr,
+        .fsetattr       = stripe_fsetattr,
+        .lookup         = stripe_lookup,
+        .mknod          = stripe_mknod,
+        .setxattr       = stripe_setxattr,
+        .fsetxattr      = stripe_fsetxattr,
+        .getxattr       = stripe_getxattr,
+        .fgetxattr      = stripe_fgetxattr,
+        .removexattr    = stripe_removexattr,
+        .fremovexattr   = stripe_fremovexattr,
+        .readdirp       = stripe_readdirp,
+	.fallocate	= stripe_fallocate,
+	.discard	= stripe_discard,
+        .zerofill       = stripe_zerofill,
 };
 
 struct xlator_cbks cbks = {
         .release = stripe_release,
+        .forget  = stripe_forget,
 };
 
+struct xlator_dumpops dumpops = {
+        .priv = stripe_priv_dump,
+};
 
 struct volume_options options[] = {
         { .key  = {"block-size"},
-          .type = GF_OPTION_TYPE_ANY
+          .type = GF_OPTION_TYPE_SIZE_LIST,
+          .default_value = "128KB",
+          .min = STRIPE_MIN_BLOCK_SIZE,
+          .description = "Size of the stripe unit that would be read "
+                         "from or written to the striped servers."
         },
         { .key  = {"use-xattr"},
-          .type = GF_OPTION_TYPE_BOOL
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "true"
         },
+	{ .key = {"coalesce"},
+	  .type = GF_OPTION_TYPE_BOOL,
+	  .default_value = "true",
+	  .description = "Enable/Disable coalesce mode to flatten striped "
+			 "files as stored on the server (i.e., eliminate holes "
+			 "caused by the traditional format)."
+	},
         { .key  = {NULL} },
 };
diff --git a/xlators/cluster/stripe/src/stripe.h b/xlators/cluster/stripe/src/stripe.h
index 18fa158c9..5673d18f3 100644
--- a/xlators/cluster/stripe/src/stripe.h
+++ b/xlators/cluster/stripe/src/stripe.h
@@ -1,20 +1,11 @@
 /*
-  Copyright (c) 2009 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
 
@@ -33,9 +24,59 @@
 #include "compat.h"
 #include "compat-errno.h"
 #include "stripe-mem-types.h"
+#include "libxlator.h"
 #include <fnmatch.h>
 #include <signal.h>
 
+#define STRIPE_PATHINFO_HEADER "STRIPE:"
+#define STRIPE_MIN_BLOCK_SIZE  (16*GF_UNIT_KB)
+
+#define STRIPE_STACK_UNWIND(fop, frame, params ...) do {           \
+                stripe_local_t *__local = NULL;                    \
+                if (frame) {                                       \
+                        __local = frame->local;                    \
+                        frame->local = NULL;                       \
+                }                                                  \
+                STACK_UNWIND_STRICT (fop, frame, params);          \
+                if (__local) {                                     \
+                        stripe_local_wipe(__local);                \
+                        mem_put (__local);       \
+                }                                                  \
+        } while (0)
+
+#define STRIPE_STACK_DESTROY(frame) do {                        \
+                stripe_local_t *__local = NULL;                 \
+                __local = frame->local;                         \
+                frame->local = NULL;                            \
+                STACK_DESTROY (frame->root);                    \
+                if (__local) {                                  \
+                        stripe_local_wipe (__local);            \
+                        mem_put (__local);    \
+                }                                               \
+        } while (0)
+
+#define STRIPE_VALIDATE_FCTX(fctx, label) do {                  \
+        int     idx = 0;                                        \
+        if (!fctx) {                                            \
+                op_errno = EINVAL;                              \
+                goto label;                                     \
+        }                                                       \
+        for (idx = 0; idx < fctx->stripe_count; idx++) {        \
+                if (!fctx->xl_array[idx]) {                     \
+                        gf_log (this->name, GF_LOG_ERROR,       \
+                                "fctx->xl_array[%d] is NULL",   \
+                                idx);                           \
+                        op_errno = ESTALE;                      \
+                        goto label;                             \
+                }                                               \
+        }                                                       \
+       } while (0)
+
+typedef struct stripe_xattr_sort {
+        int   pos;
+        int   xattr_len;
+        char *xattr_value;
+} stripe_xattr_sort_t;
 
 /**
  * struct stripe_options : This keeps the pattern and the block-size
@@ -57,15 +98,17 @@ struct stripe_private {
         gf_lock_t               lock;
         uint8_t                 nodes_down;
         int8_t                  first_child_down;
+        int                    *last_event;
         int8_t                  child_count;
-        int8_t                 *state; /* Current state of child node */
         gf_boolean_t            xattr_supported;  /* default yes */
+	gf_boolean_t		coalesce;
+        char                    vol_uuid[UUID_SIZE + 1];
 };
 
 /**
- * Used to keep info about the replies received from fops->readv calls
+ * Used to keep info about the replies received from readv/writev calls
  */
-struct readv_replies {
+struct stripe_replies {
         struct iovec *vector;
         int32_t       count;    //count of vector
         int32_t       op_ret;   //op_ret of readv
@@ -77,6 +120,7 @@ struct readv_replies {
 typedef struct _stripe_fd_ctx {
         off_t      stripe_size;
         int        stripe_count;
+	int	   stripe_coalesce;
         int        static_array;
         xlator_t **xl_array;
 } stripe_fd_ctx_t;
@@ -112,9 +156,9 @@ struct stripe_local {
         blkcnt_t             preparent_blocks;
         blkcnt_t             postparent_blocks;
 
-        struct readv_replies *replies;
-        struct statvfs       statvfs_buf;
-        dir_entry_t         *entry;
+        struct stripe_replies *replies;
+        struct statvfs        statvfs_buf;
+        dir_entry_t          *entry;
 
         int8_t               revalidate;
         int8_t               failed;
@@ -136,8 +180,17 @@ struct stripe_local {
         loc_t                loc;
         loc_t                loc2;
 
+        mode_t               mode;
+        dev_t                rdev;
         /* For File I/O fops */
-        dict_t              *dict;
+        dict_t              *xdata;
+
+        stripe_xattr_sort_t *xattr_list;
+        int32_t              xattr_total_len;
+        int32_t              nallocs;
+        char xsel[256];
+
+        struct marker_str    marker;
 
         /* General usage */
         off_t                offset;
@@ -151,9 +204,85 @@ struct stripe_local {
         fd_t                *fd;
         void                *value;
         struct iobref       *iobref;
+        gf_dirent_t          entries;
+        gf_dirent_t         *dirent;
+        dict_t              *xattr;
+        uuid_t               ia_gfid;
+
+        int                  xflag;
+        mode_t               umask;
 };
 
 typedef struct stripe_local   stripe_local_t;
 typedef struct stripe_private stripe_private_t;
 
+/*
+ * Determine the stripe index of a particular frame based on the translator.
+ */
+static inline int32_t stripe_get_frame_index(stripe_fd_ctx_t *fctx,
+					     call_frame_t *prev)
+{
+	int32_t i, idx = -1;
+
+	for (i = 0; i < fctx->stripe_count; i++) {
+		if (fctx->xl_array[i] == prev->this) {
+			idx = i;
+			break;
+		}
+	}
+
+	return idx;
+}
+
+static inline void stripe_copy_xl_array(xlator_t **dst, xlator_t **src,
+					int count)
+{
+	int i;
+
+	for (i = 0; i < count; i++)
+		dst[i] = src[i];
+}
+
+void stripe_local_wipe (stripe_local_t *local);
+int32_t stripe_ctx_handle (xlator_t *this, call_frame_t *prev,
+                           stripe_local_t *local, dict_t *dict);
+void stripe_aggregate_xattr (dict_t *dst, dict_t *src);
+int32_t stripe_xattr_request_build (xlator_t *this, dict_t *dict,
+                                    uint64_t stripe_size, uint32_t stripe_count,
+                                    uint32_t stripe_index,
+				    uint32_t stripe_coalesce);
+int32_t stripe_get_matching_bs (const char *path, stripe_private_t *priv);
+int set_stripe_block_size (xlator_t *this, stripe_private_t *priv, char *data);
+int32_t stripe_iatt_merge (struct iatt *from, struct iatt *to);
+int32_t stripe_fill_pathinfo_xattr (xlator_t *this, stripe_local_t *local,
+                                    char **xattr_serz);
+int32_t stripe_free_xattr_str (stripe_local_t *local);
+int32_t stripe_xattr_aggregate (char *buffer, stripe_local_t *local,
+                                int32_t *total);
+off_t coalesced_offset(off_t offset, uint64_t stripe_size, int stripe_count);
+off_t uncoalesced_size(off_t size, uint64_t stripe_size, int stripe_count,
+			int stripe_index);
+int32_t
+stripe_fill_lockinfo_xattr (xlator_t *this, stripe_local_t *local,
+                            void **xattr_serz);
+
+/*
+ * Adjust the size attribute for files if coalesce is enabled.
+ */
+static inline void correct_file_size(struct iatt *buf, stripe_fd_ctx_t *fctx,
+	call_frame_t *prev)
+{
+	int index;
+
+	if (!IA_ISREG(buf->ia_type))
+		return;
+
+	if (!fctx || !fctx->stripe_coalesce)
+		return;
+
+	index = stripe_get_frame_index(fctx, prev);
+	buf->ia_size = uncoalesced_size(buf->ia_size, fctx->stripe_size,
+		fctx->stripe_count, index);
+}
+
 #endif /* _STRIPE_H_ */
diff --git a/xlators/cluster/unify/src/Makefile.am b/xlators/cluster/unify/src/Makefile.am
deleted file mode 100644
index 2a1fe8372..000000000
--- a/xlators/cluster/unify/src/Makefile.am
+++ /dev/null
@@ -1,16 +0,0 @@
-
-xlator_LTLIBRARIES = unify.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/legacy/cluster
-
-unify_la_LDFLAGS = -module -avoidversion
-
-unify_la_SOURCES = unify.c unify-self-heal.c
-unify_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = unify.h
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
-
-CLEANFILES = 
-
diff --git a/xlators/cluster/unify/src/unify-mem-types.h b/xlators/cluster/unify/src/unify-mem-types.h
deleted file mode 100644
index 9da061cfb..000000000
--- a/xlators/cluster/unify/src/unify-mem-types.h
+++ /dev/null
@@ -1,41 +0,0 @@
-
-/*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-   This file is part of GlusterFS.
-
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef __UNIFY_MEM_TYPES_H__
-#define __UNIFY_MEM_TYPES_H__
-
-#include "mem-types.h"
-
-enum gf_unify_mem_types_ {
-        gf_unify_mt_char = gf_common_mt_end + 1,
-        gf_unify_mt_int16_t,
-        gf_unify_mt_xlator_t,
-        gf_unify_mt_unify_private_t,
-        gf_unify_mt_xlator_list_t,
-        gf_unify_mt_dir_entry_t,
-        gf_unify_mt_off_t,
-        gf_unify_mt_int,
-        gf_unify_mt_unify_self_heal_struct,
-        gf_unify_mt_unify_local_t,
-        gf_unify_mt_end
-};
-#endif
-
diff --git a/xlators/cluster/unify/src/unify-self-heal.c b/xlators/cluster/unify/src/unify-self-heal.c
deleted file mode 100644
index b3013d363..000000000
--- a/xlators/cluster/unify/src/unify-self-heal.c
+++ /dev/null
@@ -1,1239 +0,0 @@
-/*
-  Copyright (c) 2007-2009 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-/**
- * unify-self-heal.c : 
- *   This file implements few functions which enables 'unify' translator 
- *  to be consistent in its behaviour when 
- *     > a node fails, 
- *     > a node gets added, 
- *     > a failed node comes back
- *     > a new namespace server is added (ie, an fresh namespace server).
- * 
- *  This functionality of 'unify' will enable glusterfs to support storage
- *  system failure, and maintain consistancy. This works both ways, ie, when
- *  an entry (either file or directory) is found on namespace server, and not
- *  on storage nodes, its created in storage nodes and vica-versa.
- * 
- *  The two fops, where it can be implemented are 'getdents ()' and 'lookup ()'
- *
- */
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "unify.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "common-utils.h"
-
-int32_t
-unify_sh_getdents_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       dir_entry_t *entry,
-		       int32_t count);
-
-int32_t
-unify_sh_ns_getdents_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  dir_entry_t *entry,
-			  int32_t count);
-
-int32_t
-unify_bgsh_getdents_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 dir_entry_t *entry,
-			 int32_t count);
-
-int32_t
-unify_bgsh_ns_getdents_cbk (call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    dir_entry_t *entry,
-			    int32_t count);
-
-/**
- * unify_local_wipe - free all the extra allocation of local->* here.
- */
-static void 
-unify_local_wipe (unify_local_t *local)
-{
-	/* Free the strdup'd variables in the local structure */
-	if (local->name) {
-		GF_FREE (local->name);
-	}
-
-	if (local->sh_struct) {
-		if (local->sh_struct->offset_list)
-			GF_FREE (local->sh_struct->offset_list);
-
-		if (local->sh_struct->entry_list)
-			GF_FREE (local->sh_struct->entry_list);
-
-		if (local->sh_struct->count_list)
-			GF_FREE (local->sh_struct->count_list);
-
-		GF_FREE (local->sh_struct);
-	}
-
-	loc_wipe (&local->loc1);
-	loc_wipe (&local->loc2);
-}
-
-int32_t 
-unify_sh_setdents_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	inode_t *inode = NULL;
-	dict_t *tmp_dict = NULL;
-	dir_entry_t *prev, *entry, *trav;
-
-	LOCK (&frame->lock);
-	{
-		/* if local->call_count == 0, that means, setdents on 
-		 * storagenodes is still pending.
-		 */
-		if (local->call_count)
-			callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-	if (callcnt == 0) {
-		if (local->sh_struct->entry_list[0]) {
-			prev = entry = local->sh_struct->entry_list[0];
-			if (!entry)
-				return 0;
-			trav = entry->next;
-			while (trav) {
-				prev->next = trav->next;
-				GF_FREE (trav->name);
-				if (IA_ISLNK (trav->buf.ia_type))
-					GF_FREE (trav->link);
-				GF_FREE (trav);
-				trav = prev->next;
-			}
-			GF_FREE (entry);
-		}
-
-		if (!local->flags) {
-			if (local->sh_struct->count_list[0] >= 
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-				/* count == size, that means, there are more entries
-				   to read from */
-				//local->call_count = 0;
-				local->sh_struct->offset_list[0] += 
-					UNIFY_SELF_HEAL_GETDENTS_COUNT;
-				STACK_WIND (frame,
-					    unify_sh_ns_getdents_cbk,
-					    NS(this),
-					    NS(this)->fops->getdents,
-					    local->fd,
-					    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-					    local->sh_struct->offset_list[0],
-					    GF_GET_DIR_ONLY);
-			}		
-		} else {
-			inode = local->loc1.inode;
-			fd_unref (local->fd);
-			tmp_dict = local->dict;
-
-			unify_local_wipe (local);
-			
-			STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-				      inode, &local->stbuf, local->dict,
-                                      &local->oldpostparent);
-			if (tmp_dict)
-				dict_unref (tmp_dict);
-		}
-	}
-  
-	return 0;
-}
-
-
-int32_t
-unify_sh_ns_getdents_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  dir_entry_t *entry,
-			  int32_t count)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = 0;
-	unsigned long final = 0;
-	dir_entry_t *tmp = GF_CALLOC (1, sizeof (dir_entry_t),
-                                      gf_unify_mt_dir_entry_t);
-	
-	local->sh_struct->entry_list[0] = tmp;
-	local->sh_struct->count_list[0] = count;
-	if (entry) {
-		tmp->next = entry->next;
-		entry->next = NULL;
-	}
-
-	if ((count < UNIFY_SELF_HEAL_GETDENTS_COUNT) || !entry) {
-		final = 1;
-	}
-
-	LOCK (&frame->lock);
-	{
-		/* local->call_count will be '0' till now. make it 1 so, it 
-		   can be UNWIND'ed for the last call. */
-		local->call_count = priv->child_count;
-		if (final)
-			local->flags = 1;
-	}
-	UNLOCK (&frame->lock);
-
-	for (index = 0; index < priv->child_count; index++) 
-	{
-		STACK_WIND_COOKIE (frame,
-				   unify_sh_setdents_cbk, 
-				   (void *)index,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->setdents,
-				   local->fd, GF_SET_DIR_ONLY,
-				   local->sh_struct->entry_list[0], count);
-	}
-
-	return 0;
-}
-
-int32_t 
-unify_sh_ns_setdents_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = (long)cookie;
-	dir_entry_t *prev, *entry, *trav;
-
-	LOCK (&frame->lock);
-	{
-		if (local->sh_struct->entry_list[index]) {
-			prev = entry = local->sh_struct->entry_list[index];
-			trav = entry->next;
-			while (trav) {
-				prev->next = trav->next;
-				GF_FREE (trav->name);
-				if (IA_ISLNK (trav->buf.ia_type))
-					GF_FREE (trav->link);
-				GF_FREE (trav);
-				trav = prev->next;
-			}
-			GF_FREE (entry);
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (local->sh_struct->count_list[index] < 
-	    UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-		LOCK (&frame->lock);
-		{
-			callcnt = --local->call_count;
-		}
-		UNLOCK (&frame->lock);
-	} else {
-		/* count == size, that means, there are more entries 
-		   to read from */
-		local->sh_struct->offset_list[index] += 
-			UNIFY_SELF_HEAL_GETDENTS_COUNT;
-		STACK_WIND_COOKIE (frame,
-				   unify_sh_getdents_cbk,
-				   cookie,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->getdents,
-				   local->fd,
-				   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-				   local->sh_struct->offset_list[index],
-				   GF_GET_ALL);
-    
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"readdir on (%s) with offset %"PRId64"", 
-			priv->xl_array[index]->name, 
-			local->sh_struct->offset_list[index]);
-	}
-
-	if (!callcnt) {
-		/* All storage nodes have done unified setdents on NS node.
-		 * Now, do getdents from NS and do setdents on storage nodes.
-		 */
-    
-		/* sh_struct->offset_list is no longer required for
-		   storage nodes now */
-		local->sh_struct->offset_list[0] = 0; /* reset */
-
-		STACK_WIND (frame,
-			    unify_sh_ns_getdents_cbk,
-			    NS(this),
-			    NS(this)->fops->getdents,
-			    local->fd,
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-			    0, /* In this call, do send '0' as offset */
-			    GF_GET_DIR_ONLY);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_sh_getdents_cbk -
- */
-int32_t
-unify_sh_getdents_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       dir_entry_t *entry,
-		       int32_t count)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = (long)cookie;
-	dir_entry_t *tmp = NULL; 
-
-	if (op_ret >= 0 && count > 0) {
-		/* There is some dentry found, just send the dentry to NS */
-		tmp = GF_CALLOC (1, sizeof (dir_entry_t),
-                                 gf_unify_mt_dir_entry_t);
-		local->sh_struct->entry_list[index] = tmp;
-		local->sh_struct->count_list[index] = count;
-		if (entry) {
-			tmp->next = entry->next;
-			entry->next = NULL;
-		}
-		STACK_WIND_COOKIE (frame,
-				   unify_sh_ns_setdents_cbk,
-				   cookie,
-				   NS(this),
-				   NS(this)->fops->setdents,
-				   local->fd,
-				   GF_SET_IF_NOT_PRESENT,
-				   local->sh_struct->entry_list[index],
-				   count);
-		return 0;
-	}
-  
-	if (count < UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-		LOCK (&frame->lock);
-		{
-			callcnt = --local->call_count;
-		}
-		UNLOCK (&frame->lock);
-	} else {
-		/* count == size, that means, there are more entries 
-		   to read from */
-		local->sh_struct->offset_list[index] += 
-			UNIFY_SELF_HEAL_GETDENTS_COUNT;
-		STACK_WIND_COOKIE (frame,
-				   unify_sh_getdents_cbk,
-				   cookie,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->getdents,
-				   local->fd,
-				   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-				   local->sh_struct->offset_list[index],
-				   GF_GET_ALL);
-    
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"readdir on (%s) with offset %"PRId64"", 
-			priv->xl_array[index]->name, 
-			local->sh_struct->offset_list[index]);
-	}
-
-	if (!callcnt) {
-		/* All storage nodes have done unified setdents on NS node.
-		 * Now, do getdents from NS and do setdents on storage nodes.
-		 */
-    
-		/* sh_struct->offset_list is no longer required for
-		   storage nodes now */
-		local->sh_struct->offset_list[0] = 0; /* reset */
-
-		STACK_WIND (frame,
-			    unify_sh_ns_getdents_cbk,
-			    NS(this),
-			    NS(this)->fops->getdents,
-			    local->fd,
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-			    0, /* In this call, do send '0' as offset */
-			    GF_GET_DIR_ONLY);
-	}
-
-	return 0;
-}
-
-/**
- * unify_sh_opendir_cbk -
- *
- * @cookie: 
- */
-int32_t 
-unify_sh_opendir_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      fd_t *fd)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-	inode_t *inode = NULL;
-	dict_t *tmp_dict = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-		} else {
-			gf_log (this->name, GF_LOG_WARNING, "failed");
-			local->failed = 1;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		local->call_count = priv->child_count + 1;
-    
-		if (!local->failed) {
-			/* send getdents() namespace after finishing
-			   storage nodes */
-			local->call_count--; 
-      
-			fd_bind (fd);
-
-			if (local->call_count) {
-				/* Used as the offset index. This list keeps
-				 * track of offset sent to each node during
-				 * STACK_WIND.
-				 */
-				local->sh_struct->offset_list = 
-					GF_CALLOC (priv->child_count, 
-					           sizeof (off_t),
-                                                   gf_unify_mt_off_t);
-				ERR_ABORT (local->sh_struct->offset_list);
-	
-				local->sh_struct->entry_list = 
-					GF_CALLOC (priv->child_count, 
-						   sizeof (dir_entry_t *),
-                                                   gf_unify_mt_dir_entry_t);
-				ERR_ABORT (local->sh_struct->entry_list);
-
-				local->sh_struct->count_list = 
-					GF_CALLOC (priv->child_count, 
-						   sizeof (int),
-                                                   gf_unify_mt_int);
-				ERR_ABORT (local->sh_struct->count_list);
-
-				/* Send getdents on all the fds */
-				for (index = 0; 
-				     index < priv->child_count; index++) {
-					STACK_WIND_COOKIE (frame,
-							   unify_sh_getdents_cbk,
-							   (void *)(long)index,
-							   priv->xl_array[index],
-							   priv->xl_array[index]->fops->getdents,
-							   local->fd,
-							   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-							   0, /* In this call, do send '0' as offset */
-							   GF_GET_ALL);
-				}
-
-				/* did stack wind, so no need to unwind here */
-				return 0;
-			} /* (local->call_count) */
-		} /* (!local->failed) */
-
-		/* Opendir failed on one node. */ 
-		inode = local->loc1.inode;
-		fd_unref (local->fd);
-		tmp_dict = local->dict;
-
-		unify_local_wipe (local);
-		/* Only 'self-heal' failed, lookup() was successful. */
-		local->op_ret = 0;
-
-		/* This is lookup_cbk ()'s UNWIND. */
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, inode,
-			      &local->stbuf, local->dict, &local->oldpostparent);
-		if (tmp_dict)
-			dict_unref (tmp_dict);
-	}
-
-	return 0;
-}
-
-/**
- * gf_sh_checksum_cbk - 
- * 
- * @frame: frame used in lookup. get a copy of it, and use that copy.
- * @this: pointer to unify xlator.
- * @inode: pointer to inode, for which the consistency check is required.
- *
- */
-int32_t 
-unify_sh_checksum_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       uint8_t *file_checksum,
-		       uint8_t *dir_checksum)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-	int32_t callcnt = 0;
-	inode_t *inode = NULL;
-	dict_t *tmp_dict = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret >= 0) {
-			if (NS(this) == (xlator_t *)cookie) {
-				memcpy (local->sh_struct->ns_file_checksum, 
-					file_checksum, NAME_MAX);
-				memcpy (local->sh_struct->ns_dir_checksum, 
-					dir_checksum, NAME_MAX);
-			} else {
-				if (local->entry_count == 0) {
-					/* Initialize the dir_checksum to be 
-					 * used for comparision with other
-					 * storage nodes. Should be done for
-					 * the first successful call *only*. 
-					 */
-                                        /* Using 'entry_count' as a flag */
-					local->entry_count = 1;
-					memcpy (local->sh_struct->dir_checksum,
-						dir_checksum, NAME_MAX);
-				}
-
-				/* Reply from the storage nodes */
-				for (index = 0; 
-				     index < NAME_MAX; index++) {
-					/* Files should be present in
-					   only one node */
-					local->sh_struct->file_checksum[index] ^= file_checksum[index];
-	  
-					/* directory structure should be
-					   same accross */
-					if (local->sh_struct->dir_checksum[index] != dir_checksum[index])
-						local->failed = 1;
-				}
-			}
-		} 
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		for (index = 0; index < NAME_MAX ; index++) {
-			if (local->sh_struct->file_checksum[index] != 
-			    local->sh_struct->ns_file_checksum[index]) {
-				local->failed = 1;
-				break;
-			}
-			if (local->sh_struct->dir_checksum[index] != 
-			    local->sh_struct->ns_dir_checksum[index]) {
-				local->failed = 1;
-				break;
-			}
-		}
-	
-		if (local->failed) {
-			/* Log it, it should be a rare event */
-			gf_log (this->name, GF_LOG_WARNING, 
-				"Self-heal triggered on directory %s", 
-				local->loc1.path);
-
-			/* Any self heal will be done at directory level */
-			local->call_count = 0;
-			local->op_ret = -1;
-			local->failed = 0;
-      
-			local->fd = fd_create (local->loc1.inode, 
-					       frame->root->pid);
-
-			local->call_count = priv->child_count + 1;
-	
-			for (index = 0; 
-			     index < (priv->child_count + 1); index++) {
-				STACK_WIND_COOKIE (frame,
-						   unify_sh_opendir_cbk,
-						   priv->xl_array[index]->name,
-						   priv->xl_array[index],
-						   priv->xl_array[index]->fops->opendir,
-						   &local->loc1,
-						   local->fd);
-			}
-			/* opendir can be done on the directory */
-			return 0;
-		}
-
-		/* no mismatch */
-		inode = local->loc1.inode;
-		tmp_dict = local->dict;
-
-		unify_local_wipe (local);
-
-		/* This is lookup_cbk ()'s UNWIND. */
-		STACK_UNWIND (frame,
-			      local->op_ret,
-			      local->op_errno,
-			      inode,
-			      &local->stbuf,
-			      local->dict, &local->oldpostparent);
-		if (tmp_dict)
-			dict_unref (tmp_dict);
-	}
-
-	return 0;
-}
-
-/* Foreground self-heal part over */
-
-/* Background self-heal part */
-
-int32_t 
-unify_bgsh_setdents_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	dir_entry_t *prev, *entry, *trav;
-
-	LOCK (&frame->lock);
-	{
-		/* if local->call_count == 0, that means, setdents 
-		   on storagenodes is still pending. */
-		if (local->call_count)
-			callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-
-	if (callcnt == 0) {
-		if (local->sh_struct->entry_list[0]) {
-			prev = entry = local->sh_struct->entry_list[0];
-			trav = entry->next;
-			while (trav) {
-				prev->next = trav->next;
-				GF_FREE (trav->name);
-				if (IA_ISLNK (trav->buf.ia_type))
-					GF_FREE (trav->link);
-				GF_FREE (trav);
-				trav = prev->next;
-			}
-			GF_FREE (entry);
-		}
-
-		if (!local->flags) {
-			if (local->sh_struct->count_list[0] >= 
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-				/* count == size, that means, there are more
-				   entries to read from */
-				//local->call_count = 0;
-				local->sh_struct->offset_list[0] += 
-					UNIFY_SELF_HEAL_GETDENTS_COUNT;
-				STACK_WIND (frame,
-					    unify_bgsh_ns_getdents_cbk,
-					    NS(this),
-					    NS(this)->fops->getdents,
-					    local->fd,
-					    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-					    local->sh_struct->offset_list[0],
-					    GF_GET_DIR_ONLY);
-			}		
-		} else {
-			fd_unref (local->fd);
-			unify_local_wipe (local);
-			STACK_DESTROY (frame->root);
-		}
-	}
-
-	return 0;
-}
-
-
-int32_t
-unify_bgsh_ns_getdents_cbk (call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno,
-			    dir_entry_t *entry,
-			    int32_t count)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = 0;
-	unsigned long final = 0;
-	dir_entry_t *tmp = GF_CALLOC (1, sizeof (dir_entry_t),
-                                      gf_unify_mt_dir_entry_t);
-
-	local->sh_struct->entry_list[0] = tmp;
-	local->sh_struct->count_list[0] = count;
-	if (entry) {
-		tmp->next = entry->next;
-		entry->next = NULL;
-	}
-
-	if ((count < UNIFY_SELF_HEAL_GETDENTS_COUNT) || !entry) {
-		final = 1;
-	}
-
-	LOCK (&frame->lock);
-	{
-		/* local->call_count will be '0' till now. make it 1 so, 
-		   it can be UNWIND'ed for the last call. */
-		local->call_count = priv->child_count;
-		if (final)
-			local->flags = 1;
-	}
-	UNLOCK (&frame->lock);
-
-	for (index = 0; index < priv->child_count; index++) 
-	{
-		STACK_WIND_COOKIE (frame,
-				   unify_bgsh_setdents_cbk, 
-				   (void *)index,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->setdents,
-				   local->fd, GF_SET_DIR_ONLY,
-				   local->sh_struct->entry_list[0], count);
-	}
-
-	return 0;
-}
-
-int32_t 
-unify_bgsh_ns_setdents_cbk (call_frame_t *frame,
-			    void *cookie,
-			    xlator_t *this,
-			    int32_t op_ret,
-			    int32_t op_errno)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = (long)cookie;
-	dir_entry_t *prev, *entry, *trav;
-
-	if (local->sh_struct->entry_list[index]) {
-		prev = entry = local->sh_struct->entry_list[index];
-		if (!entry)
-			return 0;
-		trav = entry->next;
-		while (trav) {
-			prev->next = trav->next;
-			GF_FREE (trav->name);
-			if (IA_ISLNK (trav->buf.ia_type))
-				GF_FREE (trav->link);
-			GF_FREE (trav);
-			trav = prev->next;
-		}
-		GF_FREE (entry);
-	}
-
-	if (local->sh_struct->count_list[index] < 
-	    UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-		LOCK (&frame->lock);
-		{
-			callcnt = --local->call_count;
-		}
-		UNLOCK (&frame->lock);
-	} else {
-		/* count == size, that means, there are more entries 
-		   to read from */
-		local->sh_struct->offset_list[index] += 
-			UNIFY_SELF_HEAL_GETDENTS_COUNT;
-		STACK_WIND_COOKIE (frame,
-				   unify_bgsh_getdents_cbk,
-				   cookie,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->getdents,
-				   local->fd,
-				   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-				   local->sh_struct->offset_list[index],
-				   GF_GET_ALL);
-    
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"readdir on (%s) with offset %"PRId64"", 
-			priv->xl_array[index]->name, 
-			local->sh_struct->offset_list[index]);
-	}
-
-	if (!callcnt) {
-		/* All storage nodes have done unified setdents on NS node.
-		 * Now, do getdents from NS and do setdents on storage nodes.
-		 */
-    
-		/* sh_struct->offset_list is no longer required for
-		   storage nodes now */
-		local->sh_struct->offset_list[0] = 0; /* reset */
-
-		STACK_WIND (frame,
-			    unify_bgsh_ns_getdents_cbk,
-			    NS(this),
-			    NS(this)->fops->getdents,
-			    local->fd,
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-			    0, /* In this call, do send '0' as offset */
-			    GF_GET_DIR_ONLY);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_bgsh_getdents_cbk -
- */
-int32_t
-unify_bgsh_getdents_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 dir_entry_t *entry,
-			 int32_t count)
-{
-	int32_t callcnt = -1;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	long index = (long)cookie;
-	dir_entry_t *tmp = NULL; 
-
-	if (op_ret >= 0 && count > 0) {
-		/* There is some dentry found, just send the dentry to NS */
-		tmp = GF_CALLOC (1, sizeof (dir_entry_t),
-                                 gf_unify_mt_dir_entry_t);
-		local->sh_struct->entry_list[index] = tmp;
-		local->sh_struct->count_list[index] = count;
-		if (entry) {
-			tmp->next = entry->next;
-			entry->next = NULL;
-		}
-		STACK_WIND_COOKIE (frame,
-				   unify_bgsh_ns_setdents_cbk,
-				   cookie,
-				   NS(this),
-				   NS(this)->fops->setdents,
-				   local->fd,
-				   GF_SET_IF_NOT_PRESENT,
-				   local->sh_struct->entry_list[index],
-				   count);
-		return 0;
-	}
-  
-	if (count < UNIFY_SELF_HEAL_GETDENTS_COUNT) {
-		LOCK (&frame->lock);
-		{
-			callcnt = --local->call_count;
-		}
-		UNLOCK (&frame->lock);
-	} else {
-		/* count == size, that means, there are more entries to read from */
-		local->sh_struct->offset_list[index] += 
-			UNIFY_SELF_HEAL_GETDENTS_COUNT;
-
-		STACK_WIND_COOKIE (frame,
-				   unify_bgsh_getdents_cbk,
-				   cookie,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->getdents,
-				   local->fd,
-				   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-				   local->sh_struct->offset_list[index],
-				   GF_GET_ALL);
-    
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"readdir on (%s) with offset %"PRId64"", 
-			priv->xl_array[index]->name, 
-			local->sh_struct->offset_list[index]);
-	}
-
-	if (!callcnt) {
-		/* All storage nodes have done unified setdents on NS node.
-		 * Now, do getdents from NS and do setdents on storage nodes.
-		 */
-    
-		/* sh_struct->offset_list is no longer required for 
-		   storage nodes now */
-		local->sh_struct->offset_list[0] = 0; /* reset */
-
-		STACK_WIND (frame,
-			    unify_bgsh_ns_getdents_cbk,
-			    NS(this),
-			    NS(this)->fops->getdents,
-			    local->fd,
-			    UNIFY_SELF_HEAL_GETDENTS_COUNT,
-			    0, /* In this call, do send '0' as offset */
-			    GF_GET_DIR_ONLY);
-	}
-
-	return 0;
-}
-
-/**
- * unify_bgsh_opendir_cbk -
- *
- * @cookie: 
- */
-int32_t 
-unify_bgsh_opendir_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-			fd_t *fd)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int32_t callcnt = 0;
-	int16_t index = 0;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-		} else {
-			local->failed = 1;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		local->call_count = priv->child_count + 1;
-    
-		if (!local->failed) {
-			/* send getdents() namespace after finishing 
-			   storage nodes */
-			local->call_count--; 
-			callcnt = local->call_count;
-      
-			fd_bind (fd);
-
-			if (local->call_count) {
-				/* Used as the offset index. This list keeps 
-				   track of offset sent to each node during 
-				   STACK_WIND. */
-				local->sh_struct->offset_list = 
-					GF_CALLOC (priv->child_count, 
-					           sizeof (off_t),
-                                                   gf_unify_mt_off_t);
-				ERR_ABORT (local->sh_struct->offset_list);
-	
-				local->sh_struct->entry_list = 
-					GF_CALLOC (priv->child_count, 
-						   sizeof (dir_entry_t *),
-                                                   gf_unify_mt_dir_entry_t);
-				ERR_ABORT (local->sh_struct->entry_list);
-
-				local->sh_struct->count_list = 
-					GF_CALLOC (priv->child_count, 
-						   sizeof (int),
-                                                   gf_unify_mt_int);
-				ERR_ABORT (local->sh_struct->count_list);
-
-				/* Send getdents on all the fds */
-				for (index = 0; 
-				     index < priv->child_count; index++) {
-					STACK_WIND_COOKIE (frame,
-							   unify_bgsh_getdents_cbk,
-							   (void *)(long)index,
-							   priv->xl_array[index],
-							   priv->xl_array[index]->fops->getdents,
-							   local->fd,
-							   UNIFY_SELF_HEAL_GETDENTS_COUNT,
-							   0, /* In this call, do send '0' as offset */
-							   GF_GET_ALL);
-				}
-				/* did a stack wind, so no need to unwind here */
-				return 0;
-			} /* (local->call_count) */
-		} /* (!local->failed) */
-
-		/* Opendir failed on one node. 	 */
-		fd_unref (local->fd);
-		
-		unify_local_wipe (local);
-		STACK_DESTROY (frame->root);
-	}
-
-	return 0;
-}
-
-/**
- * gf_bgsh_checksum_cbk - 
- * 
- * @frame: frame used in lookup. get a copy of it, and use that copy.
- * @this: pointer to unify xlator.
- * @inode: pointer to inode, for which the consistency check is required.
- *
- */
-int32_t 
-unify_bgsh_checksum_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 uint8_t *file_checksum,
-			 uint8_t *dir_checksum)
-{
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-	int32_t callcnt = 0;
-  
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret >= 0) {
-			if (NS(this) == (xlator_t *)cookie) {
-				memcpy (local->sh_struct->ns_file_checksum, 
-					file_checksum, NAME_MAX);
-				memcpy (local->sh_struct->ns_dir_checksum, 
-					dir_checksum, NAME_MAX);
-			} else {
-				if (local->entry_count == 0) {
-					/* Initialize the dir_checksum to be 
-					 * used for comparision with other 
-					 * storage nodes. Should be done for
-					 * the first successful call *only*. 
-					 */
-					/* Using 'entry_count' as a flag */
-					local->entry_count = 1; 
-					memcpy (local->sh_struct->dir_checksum,
-						dir_checksum, NAME_MAX);
-				}
-
-				/* Reply from the storage nodes */
-				for (index = 0; 
-				     index < NAME_MAX; index++) {
-					/* Files should be present in only 
-					   one node */
-					local->sh_struct->file_checksum[index] ^= file_checksum[index];
-	  
-					/* directory structure should be same 
-					   accross */
-					if (local->sh_struct->dir_checksum[index] != dir_checksum[index])
-						local->failed = 1;
-				}
-			}
-		} 
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		for (index = 0; index < NAME_MAX ; index++) {
-			if (local->sh_struct->file_checksum[index] != 
-			    local->sh_struct->ns_file_checksum[index]) {
-				local->failed = 1;
-				break;
-			}
-			if (local->sh_struct->dir_checksum[index] != 
-			    local->sh_struct->ns_dir_checksum[index]) {
-				local->failed = 1;
-				break;
-			}
-		}
-	
-		if (local->failed) {
-			/* Log it, it should be a rare event */
-			gf_log (this->name, GF_LOG_WARNING, 
-				"Self-heal triggered on directory %s", 
-				local->loc1.path);
-
-			/* Any self heal will be done at the directory level */
-			local->op_ret = -1;
-			local->failed = 0;
-      
-			local->fd = fd_create (local->loc1.inode, 
-					       frame->root->pid);
-			local->call_count = priv->child_count + 1;
-	
-			for (index = 0; 
-			     index < (priv->child_count + 1); index++) {
-				STACK_WIND_COOKIE (frame,
-						   unify_bgsh_opendir_cbk,
-						   priv->xl_array[index]->name,
-						   priv->xl_array[index],
-						   priv->xl_array[index]->fops->opendir,
-						   &local->loc1,
-						   local->fd);
-			}
-      
-			/* opendir can be done on the directory */
-			return 0;
-		}
-
-		/* no mismatch */
-		unify_local_wipe (local);
-		STACK_DESTROY (frame->root);
-	}
-
-	return 0;
-}
-
-/* Background self-heal part over */
-
-
-
-
-/**
- * zr_unify_self_heal - 
- * 
- * @frame: frame used in lookup. get a copy of it, and use that copy.
- * @this: pointer to unify xlator.
- * @inode: pointer to inode, for which the consistency check is required.
- *
- */
-int32_t 
-zr_unify_self_heal (call_frame_t *frame,
-		    xlator_t *this,
-		    unify_local_t *local)
-{
-	unify_private_t *priv = this->private;
-	call_frame_t *bg_frame = NULL;
-	unify_local_t *bg_local = NULL;
-	inode_t *tmp_inode = NULL;
-	dict_t *tmp_dict = NULL;
-	int16_t index = 0;
-  
-	if (local->inode_generation < priv->inode_generation) {
-		/* Any self heal will be done at the directory level */
-		/* Update the inode's generation to the current generation
-		   value. */
-		local->inode_generation = priv->inode_generation;
-		inode_ctx_put (local->loc1.inode, this, 
-			  (uint64_t)(long)local->inode_generation);
-
-		if (priv->self_heal == ZR_UNIFY_FG_SELF_HEAL) {
-			local->op_ret = 0;
-			local->failed = 0;
-			local->call_count = priv->child_count + 1;
-			local->sh_struct = 
-	         	     GF_CALLOC (1, sizeof (struct unify_self_heal_struct),
-                                        gf_unify_mt_unify_self_heal_struct);
-      
-			/* +1 is for NS */
-			for (index = 0; 
-			     index < (priv->child_count + 1); index++) {
-				STACK_WIND_COOKIE (frame,
-						   unify_sh_checksum_cbk,
-						   priv->xl_array[index],
-						   priv->xl_array[index],
-						   priv->xl_array[index]->fops->checksum,
-						   &local->loc1,
-						   0);
-			}
-
-			/* Self-heal in foreground, hence no need 
-			   to UNWIND here */
-			return 0;
-		}
-
-		/* Self Heal done in background */
-		bg_frame = copy_frame (frame);
-		INIT_LOCAL (bg_frame, bg_local);
-		loc_copy (&bg_local->loc1, &local->loc1);
-		bg_local->op_ret = 0;
-		bg_local->failed = 0;
-		bg_local->call_count = priv->child_count + 1;
-		bg_local->sh_struct = 
-			GF_CALLOC (1, sizeof (struct unify_self_heal_struct),
-                                   gf_unify_mt_unify_self_heal_struct);
-    
-		/* +1 is for NS */
-		for (index = 0; index < (priv->child_count + 1); index++) {
-			STACK_WIND_COOKIE (bg_frame,
-					   unify_bgsh_checksum_cbk,
-					   priv->xl_array[index],
-					   priv->xl_array[index],
-					   priv->xl_array[index]->fops->checksum,
-					   &bg_local->loc1,
-					   0);
-		}
-	}
-
-	/* generation number matches, self heal already done or
-	 * self heal done in background: just do STACK_UNWIND 
-	 */
-	tmp_inode = local->loc1.inode;
-	tmp_dict = local->dict;
-
-	unify_local_wipe (local);
-
-	/* This is lookup_cbk ()'s UNWIND. */
-	STACK_UNWIND (frame,
-		      local->op_ret,
-		      local->op_errno,
-		      tmp_inode,
-		      &local->stbuf,
-		      local->dict,
-                      &local->oldpostparent);
-
-	if (tmp_dict)
-		dict_unref (tmp_dict);
-
-	return 0;
-}
-
diff --git a/xlators/cluster/unify/src/unify.c b/xlators/cluster/unify/src/unify.c
deleted file mode 100644
index cec7f38c6..000000000
--- a/xlators/cluster/unify/src/unify.c
+++ /dev/null
@@ -1,4589 +0,0 @@
-/*
-  Copyright (c) 2006-2009 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-/**
- * xlators/cluster/unify:
- *     - This xlator is one of the main translator in GlusterFS, which
- *   actually does the clustering work of the file system. One need to 
- *   understand that, unify assumes file to be existing in only one of 
- *   the child node, and directories to be present on all the nodes. 
- *
- * NOTE:
- *   Now, unify has support for global namespace, which is used to keep a 
- * global view of fs's namespace tree. The stat for directories are taken
- * just from the namespace, where as for files, just 'ia_ino' is taken from
- * Namespace node, and other stat info is taken from the actual storage node.
- * Also Namespace node helps to keep consistant inode for files across 
- * glusterfs (re-)mounts.
- */
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "glusterfs.h"
-#include "unify.h"
-#include "dict.h"
-#include "xlator.h"
-#include "hashfn.h"
-#include "logging.h"
-#include "stack.h"
-#include "defaults.h"
-#include "common-utils.h"
-#include <signal.h>
-#include <libgen.h>
-#include "compat-errno.h"
-#include "compat.h"
-
-#define UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR(_loc) do { \
-  if (!(_loc && _loc->inode)) {                            \
-    STACK_UNWIND (frame, -1, EINVAL, NULL, NULL, NULL);    \
-    return 0;                                              \
-  }                                                        \
-} while(0)
-
-
-#define UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR(_fd) do { \
-  if (!(_fd && !fd_ctx_get (_fd, this, NULL))) {       \
-    STACK_UNWIND (frame, -1, EBADFD, NULL, NULL);      \
-    return 0;                                          \
-  }                                                    \
-} while(0)
-
-#define UNIFY_CHECK_FD_AND_UNWIND_ON_ERR(_fd) do { \
-  if (!_fd) {                                      \
-    STACK_UNWIND (frame, -1, EBADFD, NULL, NULL);  \
-    return 0;                                      \
-  }                                                \
-} while(0)
-
-/**
- * unify_local_wipe - free all the extra allocation of local->* here.
- */
-static void 
-unify_local_wipe (unify_local_t *local)
-{
-	/* Free the strdup'd variables in the local structure */
-	if (local->name) {
-		GF_FREE (local->name);
-	}
-	loc_wipe (&local->loc1);
-	loc_wipe (&local->loc2);
-}
-
-
-
-/*
- * unify_normalize_stats -
- */
-void
-unify_normalize_stats (struct statvfs *buf,
-		       unsigned long bsize,
-		       unsigned long frsize)
-{
-	double factor;
-
-	if (buf->f_bsize != bsize) {
-		factor = ((double) buf->f_bsize) / bsize;
-		buf->f_bsize  = bsize;
-		buf->f_bfree  = (fsblkcnt_t) (factor * buf->f_bfree);
-		buf->f_bavail = (fsblkcnt_t) (factor * buf->f_bavail);
-	}
-  
-	if (buf->f_frsize != frsize) {
-		factor = ((double) buf->f_frsize) / frsize;
-		buf->f_frsize = frsize;
-		buf->f_blocks = (fsblkcnt_t) (factor * buf->f_blocks);
-	}
-}
-
-
-xlator_t *
-unify_loc_subvol (loc_t *loc, xlator_t *this)
-{
-	unify_private_t *priv = NULL;
-	xlator_t        *subvol = NULL;
-	int16_t         *list = NULL;
-	long             index = 0;
-	xlator_t        *subvol_i = NULL;
-	int              ret = 0;
-	uint64_t         tmp_list = 0;
-
-	priv   = this->private;
-	subvol = NS (this);
-
-	if (!IA_ISDIR (loc->inode->ia_type)) {
-		ret = inode_ctx_get (loc->inode, this, &tmp_list);
-		list = (int16_t *)(long)tmp_list;
-		if (!list)
-			goto out;
-
-		for (index = 0; list[index] != -1; index++) {
-			subvol_i = priv->xl_array[list[index]];
-			if (subvol_i != NS (this)) {
-				subvol = subvol_i;
-				break;
-			}
-		}
-	}
-out:
-	return subvol;
-}
-
-
-
-/**
- * unify_statfs_cbk -
- */
-int32_t
-unify_statfs_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct statvfs *stbuf)
-{
-	int32_t callcnt = 0;
-	struct statvfs *dict_buf = NULL;
-	unsigned long bsize;
-	unsigned long frsize;
-	unify_local_t *local = (unify_local_t *)frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret >= 0) {
-			/* when a call is successfull, add it to local->dict */
-			dict_buf = &local->statvfs_buf;
-
-			if (dict_buf->f_bsize != 0) {
-				bsize  = max (dict_buf->f_bsize, 
-					      stbuf->f_bsize);
-
-				frsize = max (dict_buf->f_frsize, 
-					      stbuf->f_frsize);
-				unify_normalize_stats(dict_buf, bsize, frsize);
-				unify_normalize_stats(stbuf, bsize, frsize);
-			} else {
-				dict_buf->f_bsize   = stbuf->f_bsize;
-				dict_buf->f_frsize  = stbuf->f_frsize;
-			}
-      
-			dict_buf->f_blocks += stbuf->f_blocks;
-			dict_buf->f_bfree  += stbuf->f_bfree;
-			dict_buf->f_bavail += stbuf->f_bavail;
-			dict_buf->f_files  += stbuf->f_files;
-			dict_buf->f_ffree  += stbuf->f_ffree;
-			dict_buf->f_favail += stbuf->f_favail;
-			dict_buf->f_fsid    = stbuf->f_fsid;
-			dict_buf->f_flag    = stbuf->f_flag;
-			dict_buf->f_namemax = stbuf->f_namemax;
-			local->op_ret = op_ret;
-		} else {
-			/* fop on storage node has failed due to some error */
-			if (op_errno != ENOTCONN) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"child(%s): %s", 
-					prev_frame->this->name, 
-					strerror (op_errno));
-			}
-			local->op_errno = op_errno;
-		}
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->statvfs_buf);
-	}
-
-	return 0;
-}
-
-/**
- * unify_statfs -
- */
-int32_t
-unify_statfs (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc)
-{
-	unify_local_t *local = NULL;
-	xlator_list_t *trav = this->children;
-
-	INIT_LOCAL (frame, local);
-	local->call_count = ((unify_private_t *)this->private)->child_count;
-
-	while(trav) {
-		STACK_WIND (frame,
-			    unify_statfs_cbk,
-			    trav->xlator,
-			    trav->xlator->fops->statfs,
-			    loc);
-		trav = trav->next;
-	}
-
-	return 0;
-}
-
-/**
- * unify_buf_cbk - 
- */
-int32_t
-unify_buf_cbk (call_frame_t *frame,
-	       void *cookie,
-	       xlator_t *this,
-	       int32_t op_ret,
-	       int32_t op_errno,
-	       struct iatt *buf)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s(): child(%s): path(%s): %s", 
-				gf_fop_list[frame->root->op],
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-
-			local->op_errno = op_errno;
-			if ((op_errno == ENOENT) && priv->optimist) 
-				local->op_ret = 0;
-		}
-
-		if (op_ret >= 0) {
-			local->op_ret = 0;
-
-			if (NS (this) == prev_frame->this) {
-				local->ia_ino = buf->ia_ino;
-				/* If the entry is directory, get the stat
-				   from NS node */
-				if (IA_ISDIR (buf->ia_type) || 
-				    !local->stbuf.ia_blksize) {
-					local->stbuf = *buf;
-				}
-			}
-
-			if ((!IA_ISDIR (buf->ia_type)) && 
-			    (NS (this) != prev_frame->this)) {
-				/* If file, take the stat info from Storage 
-				   node. */
-				local->stbuf = *buf;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-    
-	if (!callcnt) {
-		/* If the inode number is not filled, operation should
-		   fail */
-		if (!local->ia_ino)
-			local->op_ret = -1;
-
-		local->stbuf.ia_ino = local->ia_ino;
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->stbuf);
-	}
-
-	return 0;
-}
-
-#define check_if_dht_linkfile(s) \
-        ((st_mode_from_ia (s->ia_prot, s->ia_type) & ~S_IFMT) == S_ISVTX)
-
-/**
- * unify_lookup_cbk - 
- */
-int32_t 
-unify_lookup_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  inode_t *inode,
-		  struct iatt *buf,
-		  dict_t *dict,
-                  struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	inode_t *tmp_inode = NULL;
-	dict_t *local_dict = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
- 
-		if (op_ret == -1) {
-                        if (local->revalidate &&
-                            (op_errno == ESTALE)) {
-                                /* ESTALE takes priority */
-                                local->op_errno = op_errno;
-                                local->failed = 1;
-                        }
-
-			if ((op_errno != ENOTCONN) 
-                            && (op_errno != ENOENT)
-                            && (local->op_errno != ESTALE)) {
-                                /* if local->op_errno is already ESTALE, then
-                                 * ESTALE has to propogated to the parent first.
-                                 * do not enter here.
-                                 */
-				gf_log (this->name, GF_LOG_ERROR,
-					"child(%s): path(%s): %s", 
-					priv->xl_array[(long)cookie]->name, 
-					local->loc1.path, strerror (op_errno));
-                                local->op_errno = op_errno;
-				local->failed = 1;
-
-			} else if (local->revalidate && 
-                                   (local->op_errno != ESTALE) &&
-				   !(priv->optimist && (op_errno == ENOENT))) {
-
-				gf_log (this->name, 
-					(op_errno == ENOTCONN) ? 
-					GF_LOG_DEBUG:GF_LOG_ERROR,
-					"child(%s): path(%s): %s", 
-					priv->xl_array[(long)cookie]->name, 
-					local->loc1.path, strerror (op_errno));
-				local->op_errno = op_errno;
-				local->failed = 1;
-			}
-		}
-
-		if (op_ret == 0) {
-			local->op_ret = 0; 
-			
-			if (check_if_dht_linkfile(buf)) {
-				gf_log (this->name, GF_LOG_CRITICAL,
-					"file %s may be DHT link file on %s, "
-					"make sure the backend is not shared "
-					"between unify and DHT", 
-					local->loc1.path, 
-					priv->xl_array[(long)cookie]->name);
-			}
-
-			if (local->stbuf.ia_type && local->stbuf.ia_blksize) {
-				/* make sure we already have a stbuf
-				   stored in local->stbuf */
-				if (IA_ISDIR (local->stbuf.ia_type) && 
-				    !IA_ISDIR (buf->ia_type)) {
-					gf_log (this->name, GF_LOG_CRITICAL, 
-						"[CRITICAL] '%s' is directory "
-						"on namespace, non-directory "
-						"on node '%s', returning EIO",
-						local->loc1.path, 
-						priv->xl_array[(long)cookie]->name);
-					local->return_eio = 1;
-				}
-				if (!IA_ISDIR (local->stbuf.ia_type) && 
-				    IA_ISDIR (buf->ia_type)) {
-					gf_log (this->name, GF_LOG_CRITICAL, 
-						"[CRITICAL] '%s' is directory "
-						"on node '%s', non-directory "
-						"on namespace, returning EIO",
-						local->loc1.path, 
-						priv->xl_array[(long)cookie]->name);
-					local->return_eio = 1;
-				}
-			}
-	
-			if (!local->revalidate && !IA_ISDIR (buf->ia_type)) {
-				/* This is the first time lookup on file*/
-				if (!local->list) {
-					/* list is not allocated, allocate 
-					   the max possible range */
-					local->list = GF_CALLOC (1, 2 * (priv->child_count + 2),
-                                                                 gf_unify_mt_int16_t);
-					if (!local->list) {
-						gf_log (this->name, 
-							GF_LOG_CRITICAL, 
-							"Not enough memory");
-						STACK_UNWIND (frame, -1, 
-							      ENOMEM, inode, 
-							      NULL, NULL, NULL);
-						return 0;
-					}
-				}
-				/* update the index of the list */
-				local->list [local->index++] = 
-					(int16_t)(long)cookie;
-			}
-                        
-                        if (!local->revalidate && IA_ISDIR (buf->ia_type)) {
-                                /* fresh lookup of a directory */
-                                inode_ctx_put (local->loc1.inode, this, 
-                                               priv->inode_generation);
-                        }
-
-			if ((!local->dict) && dict &&
-			    (priv->xl_array[(long)cookie] != NS(this)))	{
-				local->dict = dict_ref (dict);
-			}
-
-			/* index of NS node is == total child count */
-			if (priv->child_count == (int16_t)(long)cookie) {
-				/* Take the inode number from namespace */
-				local->ia_ino = buf->ia_ino;
-				if (IA_ISDIR (buf->ia_type) || 
-				    !(local->stbuf.ia_blksize)) {
-					local->stbuf = *buf;
-                                        local->oldpostparent = *postparent;
-				}
-			} else if (!IA_ISDIR (buf->ia_type)) {
-				/* If file, then get the stat from 
-				   storage node */
-				local->stbuf = *buf;
-			}
-
-			if (local->ia_nlink < buf->ia_nlink) {
-				local->ia_nlink = buf->ia_nlink;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		local_dict = local->dict;
-		if (local->return_eio) {
-			gf_log (this->name, GF_LOG_CRITICAL, 
-				"[CRITICAL] Unable to fix the path (%s) with "
-				"self-heal, try manual verification. "
-				"returning EIO.", local->loc1.path);
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, -1, EIO, inode, NULL, NULL);
-			if (local_dict)	{
-				dict_unref (local_dict);
-			}
-			return 0;
-		}
-
-		if (!local->stbuf.ia_blksize) {
-			/* Inode not present */
-			local->op_ret = -1;
-		} else {
-			if (!local->revalidate && 
-			    !IA_ISDIR (local->stbuf.ia_type)) { 
-				/* If its a file, big array is useless, 
-				   allocate the smaller one */
-				int16_t *list = NULL;
-				list = GF_CALLOC (1, 2 * (local->index + 1),
-                                                  gf_unify_mt_int16_t);
-				ERR_ABORT (list);
-				memcpy (list, local->list, 2 * local->index);
-				/* Make the end of the list as -1 */
-				GF_FREE (local->list);
-				local->list = list;
-				local->list [local->index] = -1;
-				/* Update the inode's ctx with proper array */
-				/* TODO: log on failure */
-				inode_ctx_put (local->loc1.inode, this, 
-					       (uint64_t)(long)local->list);
-			}
-
-			if (IA_ISDIR(local->loc1.inode->ia_type)) {
-				/* lookup is done for directory */
-				if (local->failed && priv->self_heal) {
-					/* Triggering self-heal */
-                                        /* means, self-heal required for this 
-					   inode */
-					local->inode_generation = 0; 
-					priv->inode_generation++;
-				}
-			} else {
-				local->stbuf.ia_ino = local->ia_ino;
-			}
-	  
-			local->stbuf.ia_nlink = local->ia_nlink;
-		}
-		if (local->op_ret == -1) {
-			if (!local->revalidate && local->list)
-				GF_FREE (local->list);
-		}
-
-		if ((local->op_ret >= 0) && local->failed && 
-		    local->revalidate) {
-			/* Done revalidate, but it failed */
-			if ((op_errno != ENOTCONN) 
-                            && (local->op_errno != ESTALE)) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Revalidate failed for path(%s): %s", 
-					local->loc1.path, strerror (op_errno));
-			}
-			local->op_ret = -1;
-		}
-
-		if ((priv->self_heal && !priv->optimist) && 
-		    (!local->revalidate && (local->op_ret == 0) && 
-		     IA_ISDIR(local->stbuf.ia_type))) {
-			/* Let the self heal be done here */
-			zr_unify_self_heal (frame, this, local);
-			local_dict = NULL;
-		} else {
-                        if (local->failed) {
-                                /* NOTE: directory lookup is sent to all 
-                                 * subvolumes and success from a subvolume
-                                 * might set local->op_ret to 0 (zero) */
-                                local->op_ret = -1;
-                        }
-
-			/* either no self heal, or op_ret == -1 (failure) */
-			tmp_inode = local->loc1.inode;
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-				      tmp_inode, &local->stbuf, local->dict,
-                                      &local->oldpostparent);
-		}
-		if (local_dict) {
-			dict_unref (local_dict);
-		}
-	}
-  
-	return 0;
-}
-
-/**
- * unify_lookup - 
- */
-int32_t 
-unify_lookup (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      dict_t *xattr_req)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int16_t *list = NULL;
-	long index = 0;
-
-	if (!(loc && loc->inode)) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"%s: Argument not right", loc?loc->path:"(null)");
-		STACK_UNWIND (frame, -1, EINVAL, NULL, NULL, NULL, NULL);
-		return 0;
-	}
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, loc->inode, NULL, NULL, NULL);
-		return 0;
-	}
-        
-        if (inode_ctx_get (loc->inode, this, NULL)
-            && IA_ISDIR (loc->inode->ia_type)) {
-                local->revalidate = 1;
-        }
-
-	if (!inode_ctx_get (loc->inode, this, NULL) && 
-	    loc->inode->ia_type && 
-	    !IA_ISDIR (loc->inode->ia_type)) {
-		uint64_t tmp_list = 0;
-		/* check if revalidate or fresh lookup */
-		inode_ctx_get (loc->inode, this, &tmp_list);
-		local->list = (int16_t *)(long)tmp_list;
-	}
-
-	if (local->list) {
-		list = local->list;
-		for (index = 0; list[index] != -1; index++);
-		if (index != 2) {
-			if (index < 2) {
-				gf_log (this->name, GF_LOG_ERROR,
-					"returning ESTALE for %s: file "
-					"count is %ld", loc->path, index);
-				/* Print where all the file is present */
-				for (index = 0; 
-				     local->list[index] != -1; index++) {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"%s: found on %s", loc->path, 
-						priv->xl_array[list[index]]->name);
-				}
-				unify_local_wipe (local);
-				STACK_UNWIND (frame, -1, ESTALE, 
-					      NULL, NULL, NULL, NULL);
-				return 0;  
-			} else {
-				/* There are more than 2 presences */
-				/* Just log and continue */
-				gf_log (this->name, GF_LOG_ERROR,
-					"%s: file count is %ld", 
-					loc->path, index);
-				/* Print where all the file is present */
-				for (index = 0; 
-				     local->list[index] != -1; index++) {
-					gf_log (this->name, GF_LOG_ERROR, 
-						"%s: found on %s", loc->path, 
-						priv->xl_array[list[index]]->name);
-				}
-			}
-		}
-      
-		/* is revalidate */
-		local->revalidate = 1;
-      
-		for (index = 0; list[index] != -1; index++)
-			local->call_count++;
-      
-		for (index = 0; list[index] != -1; index++) {
-			char need_break = (list[index+1] == -1);
-			STACK_WIND_COOKIE (frame,
-					   unify_lookup_cbk,
-					   (void *)(long)list[index], //cookie
-					   priv->xl_array [list[index]],
-					   priv->xl_array [list[index]]->fops->lookup,
-					   loc,
-					   xattr_req);
-			if (need_break)
-				break;
-		}
-	} else {
-		if (loc->inode->ia_type) {
-			if (inode_ctx_get (loc->inode, this, NULL)) {
-				inode_ctx_get (loc->inode, this, 
-					       &local->inode_generation);
-			}
-		}
-		/* This is first call, there is no list */
-		/* call count should be all child + 1 namespace */
-		local->call_count = priv->child_count + 1;
-      
-		for (index = 0; index <= priv->child_count; index++) {
-			STACK_WIND_COOKIE (frame,
-					   unify_lookup_cbk,
-					   (void *)index, //cookie
-					   priv->xl_array[index],
-					   priv->xl_array[index]->fops->lookup,
-					   loc,
-					   xattr_req);
-		}
-	}
-
-	return 0;
-}
-
-/**
- * unify_stat - if directory, get the stat directly from NameSpace child.
- *     if file, check for a hint and send it only there (also to NS).
- *     if its a fresh stat, then do it on all the nodes.
- *
- * NOTE: for all the call, sending cookie as xlator pointer, which will be 
- *       used in cbk.
- */
-int32_t
-unify_stat (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-	int16_t *list = NULL;
-	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, NULL);
-		return 0;
-	}
-	local->ia_ino = loc->inode->ino;
-	if (IA_ISDIR (loc->inode->ia_type)) {
-		/* Directory */
-		local->call_count = 1;
-		STACK_WIND (frame, unify_buf_cbk, NS(this),
-			    NS(this)->fops->stat, loc);
-	} else {
-		/* File */
-		inode_ctx_get (loc->inode, this, &tmp_list);
-    		list = (int16_t *)(long)tmp_list;
-
-		for (index = 0; list[index] != -1; index++)
-			local->call_count++;
-    
-		for (index = 0; list[index] != -1; index++) {
-			char need_break = (list[index+1] == -1);
-			STACK_WIND (frame,
-				    unify_buf_cbk,
-				    priv->xl_array[list[index]],
-				    priv->xl_array[list[index]]->fops->stat,
-				    loc);
-			if (need_break)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-/**
- * unify_access_cbk -
- */
-int32_t
-unify_access_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-/**
- * unify_access - Send request to only namespace, which has all the 
- *      attributes set for the file.
- */
-int32_t
-unify_access (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      int32_t mask)
-{
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	STACK_WIND (frame,
-		    unify_access_cbk,
-		    NS(this),
-		    NS(this)->fops->access,
-		    loc,
-		    mask);
-
-	return 0;
-}
-
-int32_t
-unify_mkdir_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 inode_t *inode,
-                 struct iatt *buf,
-                 struct iatt *preparent,
-                 struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	inode_t *tmp_inode = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-  
-		if ((op_ret == -1) && !(priv->optimist && 
-					(op_errno == ENOENT || 
-					 op_errno == EEXIST))) {
-			/* TODO: Decrement the inode_generation of 
-			 * this->inode's parent inode, hence the missing 
-			 * directory is created properly by self-heal. 
-			 * Currently, there is no way to get the parent 
-			 * inode directly.
-			 */
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				priv->xl_array[(long)cookie]->name, 
-				local->loc1.path, strerror (op_errno));
-			if (op_errno != EEXIST)
-				local->failed = 1;
-			local->op_errno = op_errno;
-		}
-  
-		if (op_ret >= 0)
-			local->op_ret = 0;
-
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if (!local->failed) {
-			inode_ctx_put (local->loc1.inode, this, 
-				       priv->inode_generation);
-		}
-		
-		tmp_inode = local->loc1.inode;
-		unify_local_wipe (local);
-
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      tmp_inode, &local->stbuf,
-                              &local->oldpreparent, &local->oldpostparent);
-	}
-
-	return 0;
-}
-
-/**
- * unify_ns_mkdir_cbk -
- */
-int32_t
-unify_ns_mkdir_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-                    struct iatt *buf,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	long index = 0;
-
-	if (op_ret == -1) {
-		/* No need to send mkdir request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, GF_LOG_ERROR,
-			"namespace: path(%s): %s", 
-			local->name, strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, inode, NULL,
-                              NULL, NULL);
-		return 0;
-	}
-  
-	/* Create one inode for this entry */
-	local->op_ret = 0;
-	local->stbuf = *buf;
-
-        local->oldpreparent = *preparent;
-        local->oldpostparent = *postparent;
-
-	local->call_count = priv->child_count;
-
-	/* Send mkdir request to all the nodes now */
-	for (index = 0; index < priv->child_count; index++) {
-		STACK_WIND_COOKIE (frame,
-				   unify_mkdir_cbk,
-				   (void *)index, //cookie
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->mkdir,
-				   &local->loc1,
-				   local->mode);
-	}
-  
-	return 0;
-}
-
-
-/**
- * unify_mkdir -
- */
-int32_t
-unify_mkdir (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     mode_t mode)
-{
-	unify_local_t *local = NULL;
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->mode = mode;
-
-	loc_copy (&local->loc1, loc);
-
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_mkdir_cbk,
-		    NS(this),
-		    NS(this)->fops->mkdir,
-		    loc,
-		    mode);
-	return 0;
-}
-
-/**
- * unify_rmdir_cbk -
- */
-int32_t
-unify_rmdir_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-                 struct iatt *preparent,
-                 struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret == 0 || (priv->optimist && (op_errno == ENOENT)))
-			local->op_ret = 0;
-		if (op_ret == -1)
-			local->op_errno = op_errno;
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-                              &local->oldpreparent, &local->oldpostparent);
-	}
-
-	return 0;
-}
-
-/**
- * unify_ns_rmdir_cbk -
- */
-int32_t
-unify_ns_rmdir_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
-{
-	int16_t index = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-  
-	if (op_ret == -1) {
-		/* No need to send rmdir request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, 
-			((op_errno != ENOTEMPTY) ? 
-			 GF_LOG_ERROR : GF_LOG_DEBUG),
-			"namespace: path(%s): %s", 
-			local->loc1.path, strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, NULL, NULL);
-		return 0;
-	}
-
-	local->call_count = priv->child_count;
-
-        local->oldpreparent = *preparent;
-        local->oldpostparent = *postparent;
-
-	for (index = 0; index < priv->child_count; index++) {
-		STACK_WIND (frame,
-			    unify_rmdir_cbk,
-			    priv->xl_array[index],
-			    priv->xl_array[index]->fops->rmdir,
-			    &local->loc1);
-	}
-
-	return 0;
-}
-
-/**
- * unify_rmdir -
- */
-int32_t
-unify_rmdir (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc)
-{
-	unify_local_t *local = NULL;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_rmdir_cbk,
-		    NS(this),
-		    NS(this)->fops->rmdir,
-		    loc);
-
-	return 0;
-}
-
-/**
- * unify_open_cbk -
- */
-int32_t
-unify_open_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		fd_t *fd)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-			if (NS(this) != (xlator_t *)cookie) {
-				/* Store child node's ptr, used in 
-				   all the f*** / FileIO calls */
-				fd_ctx_set (fd, this, (uint64_t)(long)cookie);
-			}
-		}
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			local->failed = 1;
-		}
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if ((local->failed == 1) && (local->op_ret >= 0)) {
-			local->call_count = 1;
-			/* return -1 to user */
-			local->op_ret = -1;
-			//local->op_errno = EIO; 
-      
-			if (!fd_ctx_get (local->fd, this, NULL)) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Open success on child node, "
-					"failed on namespace");
-			} else {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Open success on namespace, "
-					"failed on child node");
-			}
-		}
-
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, 
-			      local->op_errno, local->fd);
-	}
-
-	return 0;
-}
-
-#ifdef GF_DARWIN_HOST_OS
-/**
- * unify_create_lookup_cbk - 
- */
-int32_t 
-unify_open_lookup_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       inode_t *inode,
-		       struct iatt *buf,
-		       dict_t *dict,
-		       struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	int16_t index = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if ((op_ret == -1) && (op_errno != ENOENT)) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				priv->xl_array[(long)cookie]->name, 
-				local->loc1.path, strerror (op_errno));
-			local->op_errno = op_errno;
-		}
-    
-		if (op_ret >= 0) {
-			local->op_ret = op_ret; 
-			local->index++;
-			if (NS(this) == priv->xl_array[(long)cookie]) {
-				local->list[0] = (int16_t)(long)cookie;
-			} else {
-				local->list[1] = (int16_t)(long)cookie;
-			}
-			if (IA_ISDIR (buf->ia_type))
-				local->failed = 1;
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		int16_t file_list[3] = {0,};
-		local->op_ret = -1;
-
-		file_list[0] = local->list[0];
-		file_list[1] = local->list[1];
-		file_list[2] = -1;
-
-		if (local->index != 2) {
-			/* Lookup failed, can't do open */
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s: present on %d nodes", 
-				local->name, local->index);
-
-			if (local->index < 2) {
-				unify_local_wipe (local);
-				gf_log (this->name, GF_LOG_ERROR,
-					"returning as file found on less "
-					"than 2 nodes");
-				STACK_UNWIND (frame, local->op_ret, 
-					      local->op_errno, local->fd);
-				return 0;
-			}
-		}
-
-		if (local->failed) {
-			/* Open on directory, return EISDIR */
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, -1, EISDIR, local->fd);
-			return 0;
-		}
-
-		/* Everything is perfect :) */    
-		local->call_count = 2;
-    
-		for (index = 0; file_list[index] != -1; index++) {
-			char need_break = (file_list[index+1] == -1);
-			STACK_WIND_COOKIE (frame,
-					   unify_open_cbk,
-					   priv->xl_array[file_list[index]],
-					   priv->xl_array[file_list[index]],
-					   priv->xl_array[file_list[index]]->fops->open,
-					   &local->loc1,
-					   local->flags,
-					   local->fd, local->wbflags);
-			if (need_break)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-int32_t
-unify_open_readlink_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 const char *path,
-			 struct iatt *sbuf)
-{
-	int16_t index = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		STACK_UNWIND (frame, -1, ENOENT);
-		return 0;
-	}
-
-	if (path[0] == '/') {
-		local->name = gf_strdup (path);
-		ERR_ABORT (local->name);
-	} else {
-		char *tmp_str = gf_strdup (local->loc1.path);
-		char *tmp_base = dirname (tmp_str);
-		local->name = GF_CALLOC (1, ZR_PATH_MAX, gf_unify_mt_char);
-		strcpy (local->name, tmp_base);
-		strncat (local->name, "/", 1);
-		strcat (local->name, path);
-		GF_FREE (tmp_str);
-	}
-  
-	local->list = GF_CALLOC (1, sizeof (int16_t) * 3,
-                                 gf_unify_mt_int16_t);
-	ERR_ABORT (local->list);
-	local->call_count = priv->child_count + 1;
-	local->op_ret = -1;
-	for (index = 0; index <= priv->child_count; index++) {
-		/* Send the lookup to all the nodes including namespace */
-		STACK_WIND_COOKIE (frame,
-				   unify_open_lookup_cbk,
-				   (void *)(long)index,
-				   priv->xl_array[index],
-				   priv->xl_array[index]->fops->lookup,
-				   &local->loc1,
-				   NULL);
-	}
-
-	return 0;
-}
-#endif /* GF_DARWIN_HOST_OS */
-
-/**
- * unify_open - 
- */
-int32_t
-unify_open (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *loc,
-	    int32_t flags,
-	    fd_t *fd,
-            int32_t wbflags)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = NULL;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	int16_t file_list[3] = {0,};
-	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Init */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	local->fd    = fd;
-	local->flags = flags;
-        local->wbflags = wbflags;
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	local->list = list;
-	file_list[0] = priv->child_count; /* Thats namespace */
-	file_list[2] = -1;
-	for (index = 0; list[index] != -1; index++) {
-		local->call_count++;
-		if (list[index] != priv->child_count)
-			file_list[1] = list[index];
-	}
-
-	if (local->call_count != 2) {
-		/* If the lookup was done for file */
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s: entry_count is %d",
-			loc->path, local->call_count);
-		for (index = 0; local->list[index] != -1; index++)
-			gf_log (this->name, GF_LOG_ERROR, "%s: found on %s",
-				loc->path, priv->xl_array[list[index]]->name);
-
-		if (local->call_count < 2) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"returning EIO as file found on onlyone node");
-			STACK_UNWIND (frame, -1, EIO, fd);
-			return 0;
-		}
-	}
-
-#ifdef GF_DARWIN_HOST_OS
-	/* Handle symlink here */
-	if (IA_ISLNK (loc->inode->ia_type)) {
-		/* Callcount doesn't matter here */
-		STACK_WIND (frame,
-			    unify_open_readlink_cbk,
-			    NS(this),
-			    NS(this)->fops->readlink,
-			    loc, ZR_PATH_MAX);
-		return 0;
-	}
-#endif /* GF_DARWIN_HOST_OS */
-
-	local->call_count = 2;
-	for (index = 0; file_list[index] != -1; index++) {
-		char need_break = (file_list[index+1] == -1);
-		STACK_WIND_COOKIE (frame,
-				   unify_open_cbk,
-				   priv->xl_array[file_list[index]], //cookie
-				   priv->xl_array[file_list[index]],
-				   priv->xl_array[file_list[index]]->fops->open,
-				   loc,
-				   flags,
-				   fd, wbflags);
-		if (need_break)
-			break;
-	}
-
-	return 0;
-}
-
-
-int32_t 
-unify_create_unlink_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-                         struct iatt *preparent,
-                         struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-	inode_t *inode = local->loc1.inode;
-
-	unify_local_wipe (local);
-
-	STACK_UNWIND (frame, local->op_ret, local->op_errno, local->fd, 
-		      inode, &local->stbuf,
-                      &local->oldpreparent, &local->oldpostparent);
-  
-	return 0;
-}
-
-/**
- * unify_create_open_cbk -
- */
-int32_t
-unify_create_open_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno,
-		       fd_t *fd)
-{
-	int ret = 0;
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	inode_t *inode = NULL;
-	xlator_t *child = NULL;
-	uint64_t tmp_value = 0;
-
-	LOCK (&frame->lock);
-	{
-		if (op_ret >= 0) {
-			local->op_ret = op_ret;
-			if (NS(this) != (xlator_t *)cookie) {
-				/* Store child node's ptr, used in all 
-				   the f*** / FileIO calls */
-				/* TODO: log on failure */
-				ret = fd_ctx_get (fd, this, &tmp_value);
-				cookie = (void *)(long)tmp_value;
-			} else {
-				/* NOTE: open successful on namespace.
-				 *       fd's ctx can be used to identify open 
-				 *       failure on storage subvolume. cool 
-				 *       ide ;) */
-				local->failed = 0;
-			}
-		} else {
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				((xlator_t *)cookie)->name,
-				local->loc1.path, strerror (op_errno));
-			local->op_errno = op_errno;
-			local->failed = 1;
-		}
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if (local->failed == 1 && (local->op_ret >= 0)) {
-			local->call_count = 1;
-			/* return -1 to user */
-			local->op_ret = -1;
-			local->op_errno = EIO;
-			local->fd = fd;
-			local->call_count = 1;
-
-			if (!fd_ctx_get (local->fd, this, &tmp_value)) {
-				child = (xlator_t *)(long)tmp_value;
-
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Create success on child node, "
-					"failed on namespace");
-
-				STACK_WIND (frame,
-					    unify_create_unlink_cbk,
-					    child,
-					    child->fops->unlink,
-					    &local->loc1);
-			} else {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"Create success on namespace, "
-					"failed on child node");
-
-				STACK_WIND (frame,
-					    unify_create_unlink_cbk,
-					    NS(this),
-					    NS(this)->fops->unlink,
-					    &local->loc1);
-			}
-			return 0;
-		}
-		inode = local->loc1.inode;
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, fd,
-			      inode, &local->stbuf,
-                              &local->oldpreparent, &local->oldpostparent);
-	}
-	return 0;
-}
-
-/**
- * unify_create_lookup_cbk - 
- */
-int32_t 
-unify_create_lookup_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-			 inode_t *inode,
-			 struct iatt *buf,
-			 dict_t *dict,
-                         struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	int16_t index = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				priv->xl_array[(long)cookie]->name, 
-				local->loc1.path, strerror (op_errno));
-			local->op_errno = op_errno;
-			local->failed = 1;
-		}
-
-		if (op_ret >= 0) {
-			local->op_ret = op_ret; 
-			local->list[local->index++] = (int16_t)(long)cookie;
-			if (NS(this) == priv->xl_array[(long)cookie]) {
-				local->ia_ino = buf->ia_ino;
-			} else {
-				local->stbuf = *buf;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		int16_t *list = local->list;
-		int16_t file_list[3] = {0,};
-		local->op_ret = -1;
-
-		local->list [local->index] = -1;
-		file_list[0] = list[0];
-		file_list[1] = list[1];
-		file_list[2] = -1;
-
-		local->stbuf.ia_ino = local->ia_ino;
-		/* TODO: log on failure */
-		inode_ctx_put (local->loc1.inode, this, 
-			       (uint64_t)(long)local->list);
-
-		if (local->index != 2) {
-			/* Lookup failed, can't do open */
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s: present on %d nodes", 
-				local->loc1.path, local->index);
-			file_list[0] = priv->child_count;
-			for (index = 0; list[index] != -1; index++) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"%s: found on %s", local->loc1.path, 
-					priv->xl_array[list[index]]->name);
-				if (list[index] != priv->child_count)
-					file_list[1] = list[index];
-			}
-
-			if (local->index < 2) {
-				unify_local_wipe (local);
-				gf_log (this->name, GF_LOG_ERROR,
-					"returning EIO as file found on "
-					"only one node");
-				STACK_UNWIND (frame, -1, EIO, 
-					      local->fd, inode, NULL,
-                                              NULL, NULL);
-				return 0;
-			}
-		}
-		/* Everything is perfect :) */    
-		local->call_count = 2;
-    
-		for (index = 0; file_list[index] != -1; index++) {
-			char need_break = (file_list[index+1] == -1);
-			STACK_WIND_COOKIE (frame,
-					   unify_create_open_cbk,
-					   priv->xl_array[file_list[index]],
-					   priv->xl_array[file_list[index]],
-					   priv->xl_array[file_list[index]]->fops->open,
-					   &local->loc1,
-					   local->flags,
-					   local->fd, 0);
-			if (need_break)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_create_cbk -
- */
-int32_t
-unify_create_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  fd_t *fd,
-		  inode_t *inode,
-		  struct iatt *buf,
-                  struct iatt *preparent,
-                  struct iatt *postparent)
-{
-	int ret = 0;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-	inode_t *tmp_inode = NULL;
-
-	if (op_ret == -1) {
-		/* send unlink () on Namespace */
-		local->op_errno = op_errno;
-		local->op_ret = -1;
-		local->call_count = 1;
-		gf_log (this->name, GF_LOG_ERROR,
-			"create failed on %s (file %s, error %s), "
-			"sending unlink to namespace", 
-			prev_frame->this->name, 
-			local->loc1.path, strerror (op_errno));
-
-		STACK_WIND (frame,
-			    unify_create_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-
-		return 0;
-	}
-
-	if (op_ret >= 0) {
-		local->op_ret = op_ret;
-		local->stbuf = *buf;
-		/* Just inode number should be from NS node */
-		local->stbuf.ia_ino = local->ia_ino;
-
-		/* TODO: log on failure */
-		ret = fd_ctx_set (fd, this, (uint64_t)(long)prev_frame->this);
-	}
-  
-	tmp_inode = local->loc1.inode;
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, local->op_ret, local->op_errno, local->fd, 
-		      tmp_inode, &local->stbuf,
-                      &local->oldpreparent, &local->oldpostparent);
-
-	return 0;
-}
-
-/**
- * unify_ns_create_cbk -
- * 
- */
-int32_t
-unify_ns_create_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     fd_t *fd,
-		     inode_t *inode,
-		     struct iatt *buf,
-                     struct iatt *preparent,
-                     struct iatt *postparent)
-{
-	struct sched_ops *sched_ops = NULL;
-	xlator_t *sched_xl = NULL;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t *list = NULL;
-	int16_t index = 0;
-
-	if (op_ret == -1) {
-		/* No need to send create request to other servers, as 
-		   namespace action failed. Handle exclusive create here. */
-		if ((op_errno != EEXIST) || 
-		    ((op_errno == EEXIST) && 
-		     ((local->flags & O_EXCL) == O_EXCL))) {
-			/* If its just a create call without O_EXCL, 
-			   don't do this */
-			gf_log (this->name, GF_LOG_ERROR,
-				"namespace: path(%s): %s", 
-				local->loc1.path, strerror (op_errno));
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, op_ret, op_errno, fd, inode, buf,
-                                      preparent, postparent);
-			return 0;
-		}
-	}
-  
-	if (op_ret >= 0) {
-		/* Get the inode number from the NS node */
-		local->ia_ino = buf->ia_ino;
-  
-                local->oldpreparent = *preparent;
-                local->oldpostparent = *postparent;
-
-		local->op_ret = -1;
-
-		/* Start the mapping list */
-		list = GF_CALLOC (1, sizeof (int16_t) * 3,
-                                  gf_unify_mt_int16_t);
-		ERR_ABORT (list);
-		inode_ctx_put (inode, this, (uint64_t)(long)list);
-		list[0] = priv->child_count;
-		list[2] = -1;
-
-		/* This means, file doesn't exist anywhere in the Filesystem */
-		sched_ops = priv->sched_ops;
-
-		/* Send create request to the scheduled node now */
-		sched_xl = sched_ops->schedule (this, local->loc1.path);
-		if (sched_xl == NULL)
-		{
-			/* send unlink () on Namespace */
-			local->op_errno = ENOTCONN;
-			local->op_ret = -1;
-			local->call_count = 1;
-			gf_log (this->name, GF_LOG_ERROR,
-				"no node online to schedule create:(file %s) "
-				"sending unlink to namespace", 
-				(local->loc1.path)?local->loc1.path:"");
-
-			STACK_WIND (frame,
-				    unify_create_unlink_cbk,
-				    NS(this),
-				    NS(this)->fops->unlink,
-				    &local->loc1);
-	
-			return 0;
-		}
-
-		for (index = 0; index < priv->child_count; index++)
-			if (sched_xl == priv->xl_array[index])
-				break;
-		list[1] = index;
-
-		STACK_WIND (frame, unify_create_cbk,
-			    sched_xl, sched_xl->fops->create,
-			    &local->loc1, local->flags, local->mode, fd);
-	} else {
-		/* File already exists, and there is no O_EXCL flag */
-
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"File(%s) already exists on namespace, sending "
-			"open instead", local->loc1.path);
-
-		local->list = GF_CALLOC (1, sizeof (int16_t) * 3,
-                                         gf_unify_mt_int16_t);
-		ERR_ABORT (local->list);
-		local->call_count = priv->child_count + 1;
-		local->op_ret = -1;
-		for (index = 0; index <= priv->child_count; index++) {
-			/* Send lookup() to all nodes including namespace */
-			STACK_WIND_COOKIE (frame,
-					   unify_create_lookup_cbk,
-					   (void *)(long)index,
-					   priv->xl_array[index],
-					   priv->xl_array[index]->fops->lookup,
-					   &local->loc1,
-					   NULL);
-		}
-	}
-	return 0;
-}
-
-/**
- * unify_create - create a file in global namespace first, so other 
- *    clients can see them. Create the file in storage nodes in background.
- */
-int32_t
-unify_create (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc,
-	      int32_t flags,
-	      mode_t mode,
-	      fd_t *fd)
-{
-	unify_local_t *local = NULL;
-  
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->mode = mode;
-	local->flags = flags;
-	local->fd = fd;
-
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, fd, loc->inode, NULL,
-                              NULL, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_create_cbk,
-		    NS(this),
-		    NS(this)->fops->create,
-		    loc,
-		    flags | O_EXCL,
-		    mode,
-		    fd);
-  
-	return 0;
-}
-
-
-/**
- * unify_opendir_cbk - 
- */
-int32_t
-unify_opendir_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   fd_t *fd)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fd);
-
-	return 0;
-}
-
-/** 
- * unify_opendir -
- */
-int32_t
-unify_opendir (call_frame_t *frame,
-	       xlator_t *this,
-	       loc_t *loc,
-	       fd_t *fd)
-{
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	STACK_WIND (frame, unify_opendir_cbk,
-		    NS(this), NS(this)->fops->opendir, loc, fd);
-
-	return 0;
-}
-
-
-int32_t
-unify_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *statpre,
-                   struct iatt *statpost)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"%s(): child(%s): path(%s): %s", 
-				gf_fop_list[frame->root->op],
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-
-			local->op_errno = op_errno;
-			if ((op_errno == ENOENT) && priv->optimist) 
-				local->op_ret = 0;
-		}
-
-		if (op_ret >= 0) {
-			local->op_ret = 0;
-
-			if (NS (this) == prev_frame->this) {
-				local->ia_ino = statpost->ia_ino;
-				/* If the entry is directory, get the stat
-				   from NS node */
-				if (IA_ISDIR (statpost->ia_type) || 
-				    !local->stpost.ia_blksize) {
-					local->stpre = *statpre;
-                                        local->stpost = *statpost;
-				}
-			}
-
-			if ((!IA_ISDIR (statpost->ia_type)) && 
-			    (NS (this) != prev_frame->this)) {
-				/* If file, take the stat info from Storage 
-				   node. */
-				local->stpre = *statpre;
-                                local->stpost = *statpost;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-    
-	if (!callcnt) {
-		/* If the inode number is not filled, operation should
-		   fail */
-		if (!local->ia_ino)
-			local->op_ret = -1;
-
-                local->stpre.ia_ino = local->ia_ino;
-		local->stpost.ia_ino = local->ia_ino;
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->stpre, &local->stpost);
-	}
-
-	return 0;
-}
-
-
-int32_t
-unify_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               struct iatt *stbuf, int32_t valid)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int32_t index = 0;
-	int32_t callcnt = 0;
-  	uint64_t tmp_list = 0;
-
-        if (!(loc && loc->inode)) {                     
-                STACK_UNWIND (frame, -1, EINVAL, NULL, NULL);
-                return 0;
-        }                                                    
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-
-	if (IA_ISDIR (loc->inode->ia_type)) {
-		local->call_count = 1;
-      
-                STACK_WIND (frame,
-                            unify_setattr_cbk,
-                            NS (this),
-                            NS (this)->fops->setattr,
-                            loc, stbuf, valid);
-        } else {
-		inode_ctx_get (loc->inode, this, &tmp_list);
-		local->list = (int16_t *)(long)tmp_list;
-
-		for (index = 0; local->list[index] != -1; index++) {
-			local->call_count++;
-			callcnt++;
-		}
-      
-		for (index = 0; local->list[index] != -1; index++) {
-			STACK_WIND (frame,
-				    unify_setattr_cbk,
-				    priv->xl_array[local->list[index]],
-				    priv->xl_array[local->list[index]]->fops->setattr,
-				    loc, stbuf, valid);
-
-			if (!--callcnt)
-				break;
-		}
-	}
-
-	return 0;
-}
-
-
-int32_t
-unify_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iatt *stbuf, int32_t valid)
-{
-	unify_local_t *local = NULL;
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR(fd);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-
-	if (!fd_ctx_get (fd, this, &tmp_child)) {
-		/* If its set, then its file */
-		child = (xlator_t *)(long)tmp_child;		     
-
-		local->call_count = 2;
-
-		STACK_WIND (frame, unify_setattr_cbk, child,
-			    child->fops->fsetattr, fd, stbuf, valid);
-
-		STACK_WIND (frame, unify_setattr_cbk, NS(this),
-			    NS(this)->fops->fsetattr, fd, stbuf, valid);
-	} else {
-		local->call_count = 1;
-    
-		STACK_WIND (frame, unify_setattr_cbk,
-			    NS(this), NS(this)->fops->fsetattr,
-			    fd, stbuf, valid);
-	}
-  
-	return 0;
-}
-
-
-/**
- * unify_truncate_cbk - 
- */
-int32_t
-unify_truncate_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    struct iatt *prebuf,
-                    struct iatt *postbuf)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"child(%s): path(%s): %s", 
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-			local->op_errno = op_errno;
-			if (!((op_errno == ENOENT) && priv->optimist))
-				local->op_ret = -1;
-		}
-
-		if (op_ret >= 0) {
-			if (NS (this) == prev_frame->this) {
-				local->ia_ino = postbuf->ia_ino;
-				/* If the entry is directory, get the 
-				   stat from NS node */
-				if (IA_ISDIR (postbuf->ia_type) ||
-				    !local->stbuf.ia_blksize) {
-					local->stbuf = *prebuf;
-                                        local->poststbuf = *postbuf;
-				}
-			}
-
-			if ((!IA_ISDIR (postbuf->ia_type)) &&
-			    (NS (this) != prev_frame->this)) {
-				/* If file, take the stat info from 
-				   Storage node. */
-				local->stbuf = *prebuf;
-                                local->poststbuf = *postbuf;
-			}
-		}
-	}
-	UNLOCK (&frame->lock);
-    
-	if (!callcnt) {
-		if (local->ia_ino) {
-			local->stbuf.ia_ino = local->ia_ino;
-                        local->poststbuf.ia_ino = local->ia_ino;
-                } else {
-			local->op_ret = -1;
-                }
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->stbuf, &local->poststbuf);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_truncate - 
- */
-int32_t
-unify_truncate (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		off_t offset)
-{
-	unify_local_t *local = NULL;
-	unify_private_t *priv = this->private;
-	int32_t index = 0;
-	int32_t callcnt = 0;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	local->ia_ino = loc->inode->ino;
-
-	if (IA_ISDIR (loc->inode->ia_type)) {
-		local->call_count = 1;
-      
-		STACK_WIND (frame,
-			    unify_truncate_cbk,
-			    NS(this),
-			    NS(this)->fops->truncate,
-			    loc,
-                            0);
-	} else {
-		local->op_ret = 0;
-		inode_ctx_get (loc->inode, this, &tmp_list);
-		local->list = (int16_t *)(long)tmp_list;
-
-		for (index = 0; local->list[index] != -1; index++) {
-			local->call_count++;
-			callcnt++;
-		}
-      
-		/* Don't send offset to NS truncate */
-		STACK_WIND (frame, unify_truncate_cbk, NS(this),
-			    NS(this)->fops->truncate, loc, 0);
-		callcnt--;
-
-		for (index = 0; local->list[index] != -1; index++) {
-			if (NS(this) != priv->xl_array[local->list[index]]) {
-				STACK_WIND (frame,
-					    unify_truncate_cbk,
-					    priv->xl_array[local->list[index]],
-					    priv->xl_array[local->list[index]]->fops->truncate,
-					    loc,
-					    offset);
-				if (!--callcnt)
-					break;
-			}
-		}
-	}
-
-	return 0;
-}
-
-/**
- * unify_readlink_cbk - 
- */
-int32_t
-unify_readlink_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    const char *path,
-                    struct iatt *sbuf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, path, sbuf);
-	return 0;
-}
-
-/**
- * unify_readlink - Read the link only from the storage node.
- */
-int32_t
-unify_readlink (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		size_t size)
-{
-	unify_private_t *priv = this->private;
-	int32_t entry_count = 0;
-	int16_t *list = NULL;
-	int16_t index = 0;
-  	uint64_t tmp_list = 0;
-  
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-	
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++)
-		entry_count++;
-
-	if (entry_count >= 2) {
-		for (index = 0; list[index] != -1; index++) {
-			if (priv->xl_array[list[index]] != NS(this)) {
-				STACK_WIND (frame,
-					    unify_readlink_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->readlink,
-					    loc,
-					    size);
-				break;
-			}
-		}
-	} else {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"returning ENOENT, no softlink files found "
-			"on storage node");
-		STACK_UNWIND (frame, -1, ENOENT, NULL);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_unlink_cbk - 
- */
-int32_t
-unify_unlink_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-                  struct iatt *preparent,
-                  struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret == 0  || ((op_errno == ENOENT) && priv->optimist))
-			local->op_ret = 0;
-		if (op_ret == -1)
-			local->op_errno = op_errno;
-
-                if (((call_frame_t *)cookie)->this == NS(this)) {
-                        local->oldpreparent = *preparent;
-                        local->oldpostparent = *postparent;
-                }
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno,
-                              &local->oldpreparent, &local->oldpostparent);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_unlink - 
- */
-int32_t
-unify_unlink (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *loc)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = NULL;
-	int16_t *list = NULL;
-	int16_t index = 0;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++)
-		local->call_count++;
-
-	if (local->call_count) {
-		for (index = 0; list[index] != -1; index++) {
-			char need_break = (list[index+1] == -1);
-			STACK_WIND (frame,
-				    unify_unlink_cbk,
-				    priv->xl_array[list[index]],
-				    priv->xl_array[list[index]]->fops->unlink,
-				    loc);
-			if (need_break)
-				break;
-		}
-	} else {
-		gf_log (this->name, GF_LOG_ERROR,
-			"%s: returning ENOENT", loc->path);
-		STACK_UNWIND (frame, -1, ENOENT, NULL, NULL);
-	}
-
-	return 0;
-}
-
-
-/**
- * unify_readv_cbk - 
- */
-int32_t
-unify_readv_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 struct iovec *vector,
-		 int32_t count,
-		 struct iatt *stbuf,
-                 struct iobref *iobref)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, vector, count, stbuf, iobref);
-	return 0;
-}
-
-/**
- * unify_readv - 
- */
-int32_t
-unify_readv (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     size_t size,
-	     off_t offset)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame,
-		    unify_readv_cbk,
-		    child,
-		    child->fops->readv,
-		    fd,
-		    size,
-		    offset);
-
-
-	return 0;
-}
-
-/**
- * unify_writev_cbk - 
- */
-int32_t
-unify_writev_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-                  struct iatt *prebuf,
-		  struct iatt *postbuf)
-{
-	unify_local_t *local = NULL;
-
-        local = frame->local;
-
-        local->stbuf = *prebuf;
-        local->stbuf.ia_ino = local->ia_ino;
-
-        local->poststbuf = *postbuf;
-        local->poststbuf.ia_ino = local->ia_ino;
-
-	STACK_UNWIND (frame, op_ret, op_errno,
-                      &local->stbuf, &local->poststbuf);
-	return 0;
-}
-
-/**
- * unify_writev - 
- */
-int32_t
-unify_writev (call_frame_t *frame,
-	      xlator_t *this,
-	      fd_t *fd,
-	      struct iovec *vector,
-	      int32_t count,
-	      off_t off,
-              struct iobref *iobref)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-	unify_local_t *local = NULL;
-
-	INIT_LOCAL (frame, local);
-        local->ia_ino = fd->inode->ino;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame,
-		    unify_writev_cbk,
-		    child,
-		    child->fops->writev,
-		    fd,
-		    vector,
-		    count,
-		    off,
-                    iobref);
-
-	return 0;
-}
-
-/**
- * unify_ftruncate -
- */
-int32_t
-unify_ftruncate (call_frame_t *frame,
-		 xlator_t *this,
-		 fd_t *fd,
-		 off_t offset)
-{
-	xlator_t *child = NULL;
-	unify_local_t *local = NULL;
-	uint64_t tmp_child = 0;
-
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR(fd);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->op_ret = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	local->call_count = 2;
-  
-	STACK_WIND (frame, unify_truncate_cbk,
-		    child, child->fops->ftruncate,
-		    fd, offset);
-  
-	STACK_WIND (frame, unify_truncate_cbk,
-		    NS(this), NS(this)->fops->ftruncate,
-		    fd, 0);
-  
-	return 0;
-}
-
-
-/**
- * unify_flush_cbk - 
- */
-int32_t
-unify_flush_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_flush -
- */
-int32_t
-unify_flush (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_flush_cbk, child, 
-		    child->fops->flush, fd);
-
-	return 0;
-}
-
-
-/**
- * unify_fsync_cbk - 
- */
-int32_t
-unify_fsync_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-                 struct iatt *prebuf,
-                 struct iatt *postbuf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
-}
-
-/**
- * unify_fsync - 
- */
-int32_t
-unify_fsync (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd,
-	     int32_t flags)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_fsync_cbk, child,
-		    child->fops->fsync, fd, flags);
-
-	return 0;
-}
-
-/**
- * unify_fstat - Send fstat FOP to Namespace only if its directory, and to 
- *     both namespace and the storage node if its a file.
- */
-int32_t
-unify_fstat (call_frame_t *frame,
-	     xlator_t *this,
-	     fd_t *fd)
-{
-	unify_local_t *local = NULL;
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR(fd);
-
-	INIT_LOCAL (frame, local);
-	local->ia_ino = fd->inode->ino;
-
-	if (!fd_ctx_get (fd, this, &tmp_child)) {
-		/* If its set, then its file */
-		child = (xlator_t *)(long)tmp_child;		     
-		local->call_count = 2;
-
-		STACK_WIND (frame, unify_buf_cbk, child,
-			    child->fops->fstat, fd);
-
-		STACK_WIND (frame, unify_buf_cbk, NS(this),
-			    NS(this)->fops->fstat, fd);
-
-	} else {
-		/* this is an directory */
-		local->call_count = 1;
-		STACK_WIND (frame, unify_buf_cbk, NS(this),
-			    NS(this)->fops->fstat, fd);
-	}
-
-	return 0;
-}
-
-/**
- * unify_getdents_cbk - 
- */
-int32_t
-unify_getdents_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    dir_entry_t *entry,
-		    int32_t count)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, entry, count);
-	return 0;
-}
-
-/**
- * unify_getdents - send the FOP request to all the nodes.
- */
-int32_t
-unify_getdents (call_frame_t *frame,
-		xlator_t *this,
-		fd_t *fd,
-		size_t size,
-		off_t offset,
-		int32_t flag)
-{
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR (fd);
-
-	STACK_WIND (frame, unify_getdents_cbk, NS(this),
-		    NS(this)->fops->getdents, fd, size, offset, flag);
-
-	return 0;
-}
-
-
-/**
- * unify_readdir_cbk - 
- */
-int32_t
-unify_readdir_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   gf_dirent_t *buf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-
-	return 0;
-}
-
-/**
- * unify_readdir - send the FOP request to all the nodes.
- */
-int32_t
-unify_readdir (call_frame_t *frame,
-	       xlator_t *this,
-	       fd_t *fd,
-	       size_t size,
-	       off_t offset)
-{
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR (fd);
-
-	STACK_WIND (frame, unify_readdir_cbk, NS(this),
-		    NS(this)->fops->readdir, fd, size, offset);
-
-	return 0;
-}
-
-
-int32_t
-unify_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, gf_dirent_t *buf)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, buf);
-
-	return 0;
-}
-
-
-int32_t
-unify_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t offset)
-{
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR (fd);
-
-	STACK_WIND (frame, unify_readdirp_cbk, NS(this),
-		    NS(this)->fops->readdirp, fd, size, offset);
-
-	return 0;
-}
-
-
-/**
- * unify_fsyncdir_cbk - 
- */
-int32_t
-unify_fsyncdir_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-
-	return 0;
-}
-
-/**
- * unify_fsyncdir -
- */
-int32_t
-unify_fsyncdir (call_frame_t *frame,
-		xlator_t *this,
-		fd_t *fd,
-		int32_t flags)
-{
-	UNIFY_CHECK_FD_AND_UNWIND_ON_ERR (fd);
-
-	STACK_WIND (frame, unify_fsyncdir_cbk,
-		    NS(this), NS(this)->fops->fsyncdir, fd, flags);
-
-	return 0;
-}
-
-/**
- * unify_lk_cbk - UNWIND frame with the proper return arguments.
- */
-int32_t
-unify_lk_cbk (call_frame_t *frame,
-	      void *cookie,
-	      xlator_t *this,
-	      int32_t op_ret,
-	      int32_t op_errno,
-	      struct gf_flock *lock)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, lock);
-	return 0;
-}
-
-/**
- * unify_lk - Send it to all the storage nodes, (should be 1) which has file.
- */
-int32_t
-unify_lk (call_frame_t *frame,
-	  xlator_t *this,
-	  fd_t *fd,
-	  int32_t cmd,
-	  struct gf_flock *lock)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_lk_cbk, child,
-		    child->fops->lk, fd, cmd, lock);
-
-	return 0;
-}
-
-
-int32_t
-unify_setxattr_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno);
-
-static int32_t
-unify_setxattr_file_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno)
-{
-	unify_private_t *private = this->private;
-	unify_local_t *local = frame->local;
-	xlator_t *sched_xl = NULL;
-	struct sched_ops *sched_ops = NULL;
-
-	if (op_ret == -1) {
-		if (!ENOTSUP)
-			gf_log (this->name, GF_LOG_ERROR,
-				"setxattr with XATTR_CREATE on ns: "
-				"path(%s) key(%s): %s",
-				local->loc1.path, local->name, 
-				strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno);
-		return 0;
-	} 
-
-	LOCK (&frame->lock);
-	{
-		local->failed = 0;
-		local->op_ret = 0;
-		local->op_errno = 0;
-		local->call_count = 1;
-	}
-	UNLOCK (&frame->lock);
-
-	/* schedule XATTR_CREATE on one of the child node */
-	sched_ops = private->sched_ops;
-    
-	/* Send create request to the scheduled node now */
-	sched_xl = sched_ops->schedule (this, local->name); 
-	if (!sched_xl) {
-		STACK_UNWIND (frame, -1, ENOTCONN);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_setxattr_cbk,
-		    sched_xl,
-		    sched_xl->fops->setxattr,
-		    &local->loc1,
-		    local->dict,
-		    local->flags);
-	return 0;
-}
-
-/**
- * unify_setxattr_cbk - When all the child nodes return, UNWIND frame.
- */
-int32_t
-unify_setxattr_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-	dict_t *dict = NULL;
-
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			gf_log (this->name, (((op_errno == ENOENT) || 
-					      (op_errno == ENOTSUP))? 
-					     GF_LOG_DEBUG : GF_LOG_ERROR), 
-				"child(%s): path(%s): %s", 
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-			if (local->failed == -1) {
-				local->failed = 1;
-			}
-			local->op_errno = op_errno;
-		} else {
-			local->failed = 0;
-			local->op_ret = op_ret;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		if (local->failed && local->name && 
-		    ZR_FILE_CONTENT_REQUEST(local->name)) {      
-			dict = get_new_dict ();
-			dict_set (dict, local->dict->members_list->key, 
-				  data_from_dynptr(NULL, 0));
-			dict_ref (dict);
-
-			local->call_count = 1;
-
-			STACK_WIND (frame,
-				    unify_setxattr_file_cbk,
-				    NS(this),
-				    NS(this)->fops->setxattr,
-				    &local->loc1,
-				    dict,
-				    XATTR_CREATE);
-
-			dict_unref (dict);
-			return 0;
-		}
-    
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno);
-	}
-
-	return 0;
-}
-
-/**
- * unify_sexattr - This function should be sent to all the storage nodes, 
- *       which contains the file, (excluding namespace).
- */
-int32_t
-unify_setxattr (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		dict_t *dict,
-		int32_t flags)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = NULL;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	int32_t call_count = 0;
-  	uint64_t tmp_list = 0;
-	data_pair_t *trav = dict->members_list;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->failed = -1;
-	loc_copy (&local->loc1, loc);
-
-	if (IA_ISDIR (loc->inode->ia_type)) {
-
-		if (trav && trav->key && ZR_FILE_CONTENT_REQUEST(trav->key)) {
-			/* direct the storage xlators to change file 
-			   content only if file exists */
-			local->flags = flags;
-			local->dict = dict;
-			local->name = gf_strdup (trav->key);
-			flags |= XATTR_REPLACE;
-		}
-
-		local->call_count = priv->child_count;
-		for (index = 0; index < priv->child_count; index++) {
-			STACK_WIND (frame,
-				    unify_setxattr_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->setxattr,
-				    loc, dict, flags);
-		}
-		return 0;
-	}
-
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++) {
-		if (NS(this) != priv->xl_array[list[index]]) {
-			local->call_count++;
-			call_count++;
-		}
-	}
-  
-	if (local->call_count) {
-		for (index = 0; list[index] != -1; index++) {
-			if (priv->xl_array[list[index]] != NS(this)) {
-				STACK_WIND (frame,
-					    unify_setxattr_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->setxattr,
-					    loc,
-					    dict,
-					    flags);
-				if (!--call_count)
-					break;
-			}
-		}
-		return 0;
-	}
-
-	/* No entry in storage nodes */
-	gf_log (this->name, GF_LOG_DEBUG, 
-		"returning ENOENT, file not found on storage node.");
-	STACK_UNWIND (frame, -1, ENOENT);
-
-	return 0;
-}
-
-
-/**
- * unify_getxattr_cbk - This function is called from only one child, so, no
- *     need of any lock or anything else, just send it to above layer 
- */
-int32_t
-unify_getxattr_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    dict_t *value)
-{
-	int32_t callcnt = 0;
-	dict_t *local_value = NULL;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-  
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-    
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			gf_log (this->name, 
-				(((op_errno == ENOENT) || 
-				  (op_errno == ENODATA) || 
-				  (op_errno == ENOTSUP)) ? 
-				 GF_LOG_DEBUG : GF_LOG_ERROR), 
-				"child(%s): path(%s): %s", 
-				prev_frame->this->name, 
-				(local->loc1.path)?local->loc1.path:"", 
-				strerror (op_errno));
-		} else {
-			if (!local->dict)
-				local->dict = dict_ref (value);
-			local->op_ret = op_ret;
-		}
-	}
-	UNLOCK (&frame->lock);
-  
-	if (!callcnt) {
-		local_value = local->dict;
-		local->dict = NULL;
-      
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      local_value);
-      
-		if (local_value)
-			dict_unref (local_value);
-	} 
-
-	return 0;
-}
-
-
-/** 
- * unify_getxattr - This FOP is sent to only the storage node.
- */
-int32_t
-unify_getxattr (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		const char *name)
-{
-	unify_private_t *priv = this->private;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	int16_t count = 0;
-	unify_local_t *local = NULL;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-	INIT_LOCAL (frame, local);
-
-	if (IA_ISDIR (loc->inode->ia_type)) {
-		local->call_count = priv->child_count;
-		for (index = 0; index < priv->child_count; index++)
-			STACK_WIND (frame,
-				    unify_getxattr_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->getxattr,
-				    loc,
-				    name);
-		return 0;
-	}
-
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++) {
-		if (NS(this) != priv->xl_array[list[index]]) {
-			local->call_count++;
-			count++;
-		}
-	}
-
-	if (count) {
-		for (index = 0; list[index] != -1; index++) {
-			if (priv->xl_array[list[index]] != NS(this)) {
-				STACK_WIND (frame,
-					    unify_getxattr_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->getxattr,
-					    loc,
-					    name);
-				if (!--count)
-					break;
-			}
-		}
-	} else {
-		dict_t *tmp_dict = get_new_dict ();
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"%s: returning ENODATA, no file found on storage node",
-			loc->path);
-		STACK_UNWIND (frame, -1, ENODATA, tmp_dict);
-		dict_destroy (tmp_dict);
-	}
-
-	return 0;
-}
-
-/**
- * unify_removexattr_cbk - Wait till all the child node returns the call
- *      and then UNWIND to above layer.
- */
-int32_t
-unify_removexattr_cbk (call_frame_t *frame,
-		       void *cookie,
-		       xlator_t *this,
-		       int32_t op_ret,
-		       int32_t op_errno)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-
-	LOCK (&frame->lock);
-	{ 
-		callcnt = --local->call_count;
-		if (op_ret == -1) {
-			local->op_errno = op_errno;
-			if (op_errno != ENOTSUP)
-				gf_log (this->name, GF_LOG_ERROR, 
-					"child(%s): path(%s): %s", 
-					prev_frame->this->name, 
-					local->loc1.path, strerror (op_errno));
-		} else {
-			local->op_ret = op_ret;
-		}
-	}
-	UNLOCK (&frame->lock);  
-
-	if (!callcnt) {
-		STACK_UNWIND (frame, local->op_ret, local->op_errno);
-	}
-
-	return 0;
-}
-
-/**
- * unify_removexattr - Send it to all the child nodes which has the files.
- */
-int32_t
-unify_removexattr (call_frame_t *frame,
-		   xlator_t *this,
-		   loc_t *loc,
-		   const char *name)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = NULL;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	int32_t call_count = 0;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (loc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-
-	if (IA_ISDIR (loc->inode->ia_type)) {
-		local->call_count = priv->child_count;
-		for (index = 0; index < priv->child_count; index++)
-			STACK_WIND (frame, 		    
-				    unify_removexattr_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->removexattr,
-				    loc,
-				    name);
-
-		return 0;
-	}
-
-	inode_ctx_get (loc->inode, this, &tmp_list);
-	list = (int16_t *)(long)tmp_list;
-
-	for (index = 0; list[index] != -1; index++) {
-		if (NS(this) != priv->xl_array[list[index]]) {
-			local->call_count++;
-			call_count++;
-		}
-	}
-
-	if (local->call_count) {
-		for (index = 0; list[index] != -1; index++) {
-			if (priv->xl_array[list[index]] != NS(this)) {
-				STACK_WIND (frame,
-					    unify_removexattr_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->removexattr,
-					    loc,
-					    name);
-				if (!--call_count)
-					break;
-			}
-		}
-		return 0;
-	} 
-
-	gf_log (this->name, GF_LOG_DEBUG, 
-		"%s: returning ENOENT, not found on storage node.", loc->path);
-	STACK_UNWIND (frame, -1, ENOENT);
-
-	return 0;
-}
-
-
-int32_t 
-unify_mknod_unlink_cbk (call_frame_t *frame,
-			void *cookie,
-			xlator_t *this,
-			int32_t op_ret,
-			int32_t op_errno,
-                        struct iatt *preparent,
-                        struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1)
-		gf_log (this->name, GF_LOG_ERROR, 
-			"%s: %s", local->loc1.path, strerror (op_errno));
-  
-	unify_local_wipe (local);
-	/* No log required here as this -1 is for mknod call */
-	STACK_UNWIND (frame, -1, local->op_errno, NULL, NULL);
-	return 0;
-}
-
-/**
- * unify_mknod_cbk - 
- */
-int32_t
-unify_mknod_cbk (call_frame_t *frame,
-		 void *cookie,
-		 xlator_t *this,
-		 int32_t op_ret,
-		 int32_t op_errno,
-		 inode_t *inode,
-                 struct iatt *buf,
-                 struct iatt *preparent,
-                 struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"mknod failed on storage node, sending unlink to "
-			"namespace");
-		local->op_errno = op_errno;
-		STACK_WIND (frame,
-			    unify_mknod_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-		return 0;
-	}
-  
-	local->stbuf = *buf;
-	local->stbuf.ia_ino = local->ia_ino;
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, op_ret, op_errno, inode, &local->stbuf,
-                      &local->oldpreparent, &local->oldpostparent);
-	return 0;
-}
-
-/**
- * unify_ns_mknod_cbk - 
- */
-int32_t
-unify_ns_mknod_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-                    struct iatt *buf,
-                    struct iatt *preparent,
-                    struct iatt *postparent)
-{
-	struct sched_ops *sched_ops = NULL;
-	xlator_t *sched_xl = NULL;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t *list = NULL;
-	int16_t index = 0;
-	call_frame_t *prev_frame = cookie;
-
-	if (op_ret == -1) {
-		/* No need to send mknod request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, GF_LOG_ERROR, 
-			"child(%s): path(%s): %s", 
-			prev_frame->this->name, local->loc1.path, 
-			strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, inode, buf,
-                              preparent, postparent);
-		return 0;
-	}
-  
-	/* Create one inode for this entry */
-	local->op_ret = 0;
-	local->stbuf = *buf;
-	local->ia_ino = buf->ia_ino;
-
-        local->oldpreparent = *preparent;
-        local->oldpostparent = *postparent;
-
-	list = GF_CALLOC (1, sizeof (int16_t) * 3, gf_unify_mt_int16_t);
-	ERR_ABORT (list);
-	list[0] = priv->child_count;
-	list[2] = -1;
-	inode_ctx_put (inode, this, (uint64_t)(long)list);
-
-	sched_ops = priv->sched_ops;
-
-	/* Send mknod request to scheduled node now */
-	sched_xl = sched_ops->schedule (this, local->loc1.path); 
-	if (!sched_xl) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"mknod failed on storage node, no node online "
-			"at the moment, sending unlink to NS");
-		local->op_errno = ENOTCONN;
-		STACK_WIND (frame,
-			    unify_mknod_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-      
-		return 0;
-	}
-
-	for (index = 0; index < priv->child_count; index++)
-		if (sched_xl == priv->xl_array[index])
-			break;
-	list[1] = index;
-  
-	STACK_WIND (frame,  unify_mknod_cbk,
-		    sched_xl,  sched_xl->fops->mknod,
-		    &local->loc1, local->mode, local->dev);
-
-	return 0;
-}
-
-/**
- * unify_mknod - Create a device on namespace first, and later create on 
- *       the storage node.
- */
-int32_t
-unify_mknod (call_frame_t *frame,
-	     xlator_t *this,
-	     loc_t *loc,
-	     mode_t mode,
-	     dev_t rdev)
-{
-	unify_local_t *local = NULL;
-  
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	local->mode = mode;
-	local->dev = rdev;
-	loc_copy (&local->loc1, loc);
-	if (local->loc1.path == NULL) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, loc->inode, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_mknod_cbk,
-		    NS(this),
-		    NS(this)->fops->mknod,
-		    loc,
-		    mode,
-		    rdev);
-
-	return 0;
-}
-
-int32_t 
-unify_symlink_unlink_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-                          struct iatt *preparent,
-                          struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-	if (op_ret == -1)
-		gf_log (this->name, GF_LOG_ERROR, 
-			"%s: %s", local->loc1.path, strerror (op_errno));
-
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, -1, local->op_errno, NULL, NULL);
-	return 0;
-}
-
-/**
- * unify_symlink_cbk - 
- */
-int32_t
-unify_symlink_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   inode_t *inode,
-                   struct iatt *buf,
-                   struct iatt *preparent,
-                   struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		/* Symlink on storage node failed, hence send unlink 
-		   to the NS node */
-		local->op_errno = op_errno;
-		gf_log (this->name, GF_LOG_ERROR, 
-			"symlink on storage node failed, sending unlink "
-			"to namespace");
-
-		STACK_WIND (frame,
-			    unify_symlink_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-    
-		return 0;
-	}
-  
-	local->stbuf = *buf;
-	local->stbuf.ia_ino = local->ia_ino;
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, op_ret, op_errno, inode, &local->stbuf,
-                      &local->oldpreparent, &local->oldpostparent);
-
-	return 0;
-}
-
-/**
- * unify_ns_symlink_cbk - 
- */
-int32_t
-unify_ns_symlink_cbk (call_frame_t *frame,
-		      void *cookie,
-		      xlator_t *this,
-		      int32_t op_ret,
-		      int32_t op_errno,
-		      inode_t *inode,
-                      struct iatt *buf,
-                      struct iatt *preparent,
-                      struct iatt *postparent)
-{
-
-	struct sched_ops *sched_ops = NULL;
-	xlator_t *sched_xl = NULL;
-	int16_t *list = NULL;
-	unify_local_t *local = frame->local;
-	unify_private_t *priv = this->private;
-	int16_t index = 0;
-
-	if (op_ret == -1) {
-		/* No need to send symlink request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, GF_LOG_ERROR, 
-			"namespace: path(%s): %s", 
-			local->loc1.path, strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, NULL, buf,
-                              preparent, postparent);
-		return 0;
-	}
-  
-	/* Create one inode for this entry */
-	local->op_ret = 0;
-	local->ia_ino = buf->ia_ino;
-  
-        local->oldpreparent = *preparent;
-        local->oldpostparent = *postparent;
-
-	/* Start the mapping list */
-
-	list = GF_CALLOC (1, sizeof (int16_t) * 3, gf_unify_mt_int16_t);
-	ERR_ABORT (list);
-	list[0] = priv->child_count; //namespace's index
-	list[2] = -1;
-	inode_ctx_put (inode, this, (uint64_t)(long)list);
-
-	sched_ops = priv->sched_ops;
-
-	/* Send symlink request to all the nodes now */
-	sched_xl = sched_ops->schedule (this, local->loc1.path); 
-	if (!sched_xl) {
-		/* Symlink on storage node failed, hence send unlink 
-		   to the NS node */
-		local->op_errno = ENOTCONN;
-		gf_log (this->name, GF_LOG_ERROR, 
-			"symlink on storage node failed, no node online, "
-			"sending unlink to namespace");
-      
-		STACK_WIND (frame,
-			    unify_symlink_unlink_cbk,
-			    NS(this),
-			    NS(this)->fops->unlink,
-			    &local->loc1);
-      
-		return 0;
-	}
-
-	for (index = 0; index < priv->child_count; index++)
-		if (sched_xl == priv->xl_array[index])
-			break;
-	list[1] = index;
-	
-	STACK_WIND (frame,
-		    unify_symlink_cbk,
-		    sched_xl,
-		    sched_xl->fops->symlink,
-		    local->name,
-		    &local->loc1);
-
-	return 0;
-}
-
-/**
- * unify_symlink - 
- */
-int32_t
-unify_symlink (call_frame_t *frame,
-	       xlator_t *this,
-	       const char *linkpath,
-	       loc_t *loc)
-{
-	unify_local_t *local = NULL;
-  
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, loc);
-	local->name = gf_strdup (linkpath);
-
-	if ((local->name == NULL) || 
-	    (local->loc1.path == NULL)) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, loc->inode, NULL);
-		return 0;
-	}
-
-	STACK_WIND (frame,
-		    unify_ns_symlink_cbk,
-		    NS(this),
-		    NS(this)->fops->symlink,
-		    linkpath,
-		    loc);
-
-	return 0;
-}
-
-
-int32_t 
-unify_rename_unlink_cbk (call_frame_t *frame,
-			 void *cookie,
-			 xlator_t *this,
-			 int32_t op_ret,
-			 int32_t op_errno,
-                         struct iatt *preparent,
-                         struct iatt *postparent)
-{
-	int32_t callcnt = 0;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-  
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"child(%s): path(%s -> %s): %s", 
-			prev_frame->this->name, 
-			local->loc1.path, local->loc2.path, 
-			strerror (op_errno));
-      
-	}
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		local->stbuf.ia_ino = local->ia_ino;
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, local->op_errno, 
-			      &local->stbuf);
-	}
-	return 0;
-}
-
-int32_t 
-unify_ns_rename_undo_cbk (call_frame_t *frame,
-			  void *cookie,
-			  xlator_t *this,
-			  int32_t op_ret,
-			  int32_t op_errno,
-			  struct iatt *buf,
-                          struct iatt *preoldparent,
-                          struct iatt *postoldparent,
-                          struct iatt *prenewparent,
-                          struct iatt *postnewparent)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"namespace: path(%s -> %s): %s", 
-			local->loc1.path, local->loc2.path, 
-			strerror (op_errno));
-	}
-
-	local->stbuf.ia_ino = local->ia_ino;
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, local->op_ret, local->op_errno, &local->stbuf);
-	return 0;
-}
-
-int32_t 
-unify_rename_cbk (call_frame_t *frame,
-		  void *cookie,
-		  xlator_t *this,
-		  int32_t op_ret,
-		  int32_t op_errno,
-		  struct iatt *buf,
-                  struct iatt *preoldparent,
-                  struct iatt *postoldparent,
-                  struct iatt *prenewparent,
-                  struct iatt *postnewparent)
-{
-	int32_t index = 0;
-	int32_t callcnt = 0;
-	int16_t *list = NULL;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	call_frame_t *prev_frame = cookie;
-  
-	LOCK (&frame->lock);
-	{
-		callcnt = --local->call_count;
-		if (op_ret >= 0) {
-			if (!IA_ISDIR (buf->ia_type))
-				local->stbuf = *buf;
-			local->op_ret = op_ret;
-		} else {
-			gf_log (this->name, GF_LOG_ERROR, 
-				"child(%s): path(%s -> %s): %s", 
-				prev_frame->this->name, 
-				local->loc1.path, local->loc2.path, 
-				strerror (op_errno));
-			local->op_errno = op_errno;
-		}
-	}
-	UNLOCK (&frame->lock);
-
-	if (!callcnt) {
-		local->stbuf.ia_ino = local->ia_ino;
-		if (IA_ISDIR (local->loc1.inode->ia_type)) {
-			unify_local_wipe (local);
-			STACK_UNWIND (frame, local->op_ret, local->op_errno,
-                                      &local->stbuf, &local->oldpreparent,
-                                      &local->oldpostparent, &local->newpreparent,
-                                      &local->newpostparent);
-			return 0;
-		}
-
-		if (local->op_ret == -1) {
-			/* TODO: check this logic */
-
-			/* Rename failed in storage node, successful on NS, 
-			 * hence, rename back the entries in NS */
-			/* NOTE: this will be done only if the destination 
-			 * doesn't exists, if  the destination exists, the 
-			 * job of correcting NS is left to self-heal
-			 */
-			if (!local->index) {
-				loc_t tmp_oldloc = {
-                                        /* its actual 'newloc->path' */
-					.path = local->loc2.path, 
-					.inode = local->loc1.inode,
-					.parent = local->loc2.parent
-				};
-	
-				loc_t tmp_newloc = {
-					/* Actual 'oldloc->path' */
-					.path = local->loc1.path,
-					.parent = local->loc1.parent
-				};
-
-				gf_log (this->name, GF_LOG_ERROR, 
-					"rename succussful on namespace, on "
-					"stroage node failed, reverting back");
-
-				STACK_WIND (frame,
-					    unify_ns_rename_undo_cbk,
-					    NS(this),
-					    NS(this)->fops->rename,
-					    &tmp_oldloc,
-					    &tmp_newloc);
-				return 0;
-			}
-		} else {
-			/* Rename successful on storage nodes */
-
-			int32_t idx = 0;
-			int16_t *tmp_list = NULL;
-			uint64_t tmp_list_int64 = 0;
-			if (local->loc2.inode) {
-				inode_ctx_get (local->loc2.inode, 
-					       this, &tmp_list_int64);
-				list = (int16_t *)(long)tmp_list_int64;
-
-			}
-
-			if (list) {				
-				for (index = 0; list[index] != -1; index++);
-				tmp_list = GF_CALLOC (1, index * 2, 
-                                                      gf_unify_mt_int16_t);
-				memcpy (tmp_list, list, index * 2);
-
-				for (index = 0; list[index] != -1; index++) {
-					/* TODO: Check this logic. */
-					/* If the destination file exists in 
-					 * the same storage node where we sent
-					 * 'rename' call, no need to send 
-					 * unlink 
-					 */
-					for (idx = 0; 
-					     local->list[idx] != -1; idx++) {
-						if (tmp_list[index] == local->list[idx]) {
-							tmp_list[index] = priv->child_count;
-							continue;
-						}
-					}
-	  
-					if (NS(this) != priv->xl_array[tmp_list[index]]) {
-						local->call_count++;
-						callcnt++;
-					}
-				}
-
-				if (local->call_count) {
-					if (callcnt > 1)
-						gf_log (this->name, 
-							GF_LOG_ERROR, 
-							"%s->%s: more (%d) "
-							"subvolumes have the "
-							"newloc entry", 
-							local->loc1.path, 
-							local->loc2.path, 
-							callcnt);
-
-					for (index=0; 
-					     tmp_list[index] != -1; index++) {
-						if (NS(this) != priv->xl_array[tmp_list[index]]) {		    
-							STACK_WIND (frame,
-								    unify_rename_unlink_cbk,
-								    priv->xl_array[tmp_list[index]],
-								    priv->xl_array[tmp_list[index]]->fops->unlink,
-								    &local->loc2);
-							if (!--callcnt)
-								break;
-						}
-					}
-
-					GF_FREE (tmp_list);
-					return 0;
-				}
-				if (tmp_list)
-					GF_FREE (tmp_list);
-			}
-		}
-    
-		/* Need not send 'unlink' to storage node */
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, local->op_ret, 
-			      local->op_errno, &local->stbuf,
-                              &local->oldpreparent, &local->oldpostparent,
-                              &local->newpreparent, &local->newpostparent);
-	}
-
-	return 0;
-}
-
-int32_t 
-unify_ns_rename_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     struct iatt *buf,
-                     struct iatt *preoldparent,
-                     struct iatt *postoldparent,
-                     struct iatt *prenewparent,
-                     struct iatt *postnewparent)
-{
-	int32_t index = 0;
-	int32_t callcnt = 0;
-	int16_t *list = NULL;
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-
-	if (op_ret == -1) {
-		/* Free local->new_inode */
-		gf_log (this->name, GF_LOG_ERROR, 
-			"namespace: path(%s -> %s): %s", 
-			local->loc1.path, local->loc2.path, 
-			strerror (op_errno));
-
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, buf,
-                              preoldparent, postoldparent,
-                              prenewparent, postnewparent);
-		return 0;
-	}
-
-	local->stbuf = *buf;
-	local->ia_ino = buf->ia_ino;
-
-        local->oldpreparent = *preoldparent;
-        local->oldpostparent = *postoldparent;
-        local->newpreparent = *prenewparent;
-        local->newpostparent = *postnewparent;
-
-	/* Everything is fine. */
-	if (IA_ISDIR (buf->ia_type)) {
-		local->call_count = priv->child_count;
-		for (index=0; index < priv->child_count; index++) {
-			STACK_WIND (frame,
-				    unify_rename_cbk,
-				    priv->xl_array[index],
-				    priv->xl_array[index]->fops->rename,
-				    &local->loc1,
-				    &local->loc2);
-		}
-
-		return 0;
-	}
-
-	local->call_count = 0;  
-	/* send rename */
-	list = local->list;
-	for (index=0; list[index] != -1; index++) {
-		if (NS(this) != priv->xl_array[list[index]]) {
-			local->call_count++;
-			callcnt++;
-		}
-	}
-
-	if (local->call_count) {
-		for (index=0; list[index] != -1; index++) {
-			if (NS(this) != priv->xl_array[list[index]]) {
-				STACK_WIND (frame,
-					    unify_rename_cbk,
-					    priv->xl_array[list[index]],
-					    priv->xl_array[list[index]]->fops->rename,
-					    &local->loc1,
-					    &local->loc2);
-				if (!--callcnt)
-					break;
-			}
-		}
-	} else {
-		/* file doesn't seem to be present in storage nodes */
-		gf_log (this->name, GF_LOG_CRITICAL,
-			"CRITICAL: source file not in storage node, "
-			"rename successful on namespace :O");
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, -1, EIO, NULL,
-                              NULL, NULL, /* preoldparent, postoldparent */
-                              NULL, NULL); /* prenewparent, postnewparent */
-	}
-	return 0;
-}
-
-
-/**
- * unify_rename - One of the tricky function. The deadliest of all :O
- */
-int32_t
-unify_rename (call_frame_t *frame,
-	      xlator_t *this,
-	      loc_t *oldloc,
-	      loc_t *newloc)
-{
-	unify_local_t *local = NULL;
-  	uint64_t tmp_list = 0;
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-	loc_copy (&local->loc1, oldloc);
-	loc_copy (&local->loc2, newloc);
-
-	if ((local->loc1.path == NULL) || 
-	    (local->loc2.path == NULL)) {
-		gf_log (this->name, GF_LOG_CRITICAL, "Not enough memory :O");
-		STACK_UNWIND (frame, -1, ENOMEM, NULL,
-                              NULL, NULL, /* preoldparent, postoldparent */
-                              NULL, NULL); /* prenewparent, postnewparent */
-		return 0;
-	}
-  
-	inode_ctx_get (oldloc->inode, this, &tmp_list);
-	local->list = (int16_t *)(long)tmp_list;
-
-	STACK_WIND (frame,
-		    unify_ns_rename_cbk,
-		    NS(this),
-		    NS(this)->fops->rename,
-		    oldloc,
-		    newloc);
-	return 0;
-}
-
-/**
- * unify_link_cbk -
- */
-int32_t
-unify_link_cbk (call_frame_t *frame,
-		void *cookie,
-		xlator_t *this,
-		int32_t op_ret,
-		int32_t op_errno,
-		inode_t *inode,
-                struct iatt *buf,
-                struct iatt *preparent,
-                struct iatt *postparent)
-{
-	unify_local_t *local = frame->local;
-
-	if (op_ret >= 0) 
-		local->stbuf = *buf;
-	local->stbuf.ia_ino = local->ia_ino;
-
-	unify_local_wipe (local);
-	STACK_UNWIND (frame, op_ret, op_errno, inode, &local->stbuf,
-                      &local->oldpreparent, &local->oldpostparent);
-
-	return 0;
-}
-
-/**
- * unify_ns_link_cbk - 
- */
-int32_t
-unify_ns_link_cbk (call_frame_t *frame,
-		   void *cookie,
-		   xlator_t *this,
-		   int32_t op_ret,
-		   int32_t op_errno,
-		   inode_t *inode,
-                   struct iatt *buf,
-                   struct iatt *preparent,
-                   struct iatt *postparent)
-{
-	unify_private_t *priv = this->private;
-	unify_local_t *local = frame->local;
-	int16_t *list = local->list;
-	int16_t index = 0;
-
-	if (op_ret == -1) {
-		/* No need to send link request to other servers, 
-		 * as namespace action failed 
-		 */
-		gf_log (this->name, GF_LOG_ERROR, 
-			"namespace: path(%s -> %s): %s", 
-			local->loc1.path, local->loc2.path, 
-			strerror (op_errno));
-		unify_local_wipe (local);
-		STACK_UNWIND (frame, op_ret, op_errno, inode, buf,
-                              preparent, postparent);
-		return 0;
-	}
-
-	/* Update inode for this entry */
-	local->op_ret = 0;
-	local->ia_ino = buf->ia_ino;
-
-        local->oldpreparent = *preparent;
-        local->oldpostparent = *postparent;
-
-	/* Send link request to the node now */
-	for (index = 0; list[index] != -1; index++) {
-		char need_break = (list[index+1] == -1);
-		if (priv->xl_array[list[index]] != NS (this)) {
-			STACK_WIND (frame,
-				    unify_link_cbk,
-				    priv->xl_array[list[index]],
-				    priv->xl_array[list[index]]->fops->link,
-				    &local->loc1,
-				    &local->loc2);
-                        break;
-		}
-		if (need_break)
-			break;
-	}
-
-	return 0;
-}
-
-/**
- * unify_link - 
- */
-int32_t
-unify_link (call_frame_t *frame,
-	    xlator_t *this,
-	    loc_t *oldloc,
-	    loc_t *newloc)
-{
-	unify_local_t *local = NULL;
-  	uint64_t tmp_list = 0;
-
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (oldloc);
-	UNIFY_CHECK_INODE_CTX_AND_UNWIND_ON_ERR (newloc);
-
-	/* Initialization */
-	INIT_LOCAL (frame, local);
-
-	loc_copy (&local->loc1, oldloc);
-	loc_copy (&local->loc2, newloc);
-
-	inode_ctx_get (oldloc->inode, this, &tmp_list);
-	local->list = (int16_t *)(long)tmp_list;
-
-	STACK_WIND (frame,
-		    unify_ns_link_cbk,
-		    NS(this),
-		    NS(this)->fops->link,
-		    oldloc,
-		    newloc);
-
-	return 0;
-}
-
-
-/**
- * unify_checksum_cbk - 
- */
-int32_t
-unify_checksum_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    uint8_t *fchecksum,
-		    uint8_t *dchecksum)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, fchecksum, dchecksum);
-
-	return 0;
-}
-
-/**
- * unify_checksum - 
- */
-int32_t
-unify_checksum (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		int32_t flag)
-{
-	STACK_WIND (frame,
-		    unify_checksum_cbk,
-		    NS(this),
-		    NS(this)->fops->checksum,
-		    loc,
-		    flag);
-
-	return 0;
-}
-
-
-/**
- * unify_finodelk_cbk - 
- */
-int
-unify_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_finodelk
- */
-int
-unify_finodelk (call_frame_t *frame, xlator_t *this,
-		const char *volume, fd_t *fd, int cmd, struct gf_flock *flock)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_finodelk_cbk,
-		    child, child->fops->finodelk,
-		    volume, fd, cmd, flock);
-
-	return 0;
-}
-
-
-
-/**
- * unify_fentrylk_cbk - 
- */
-int
-unify_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_fentrylk
- */
-int
-unify_fentrylk (call_frame_t *frame, xlator_t *this,
-		const char *volume, fd_t *fd, const char *basename,
-		entrylk_cmd cmd, entrylk_type type)
-		
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_fentrylk_cbk,
-		    child, child->fops->fentrylk,
-		    volume, fd, basename, cmd, type);
-
-	return 0;
-}
-
-
-
-/**
- * unify_fxattrop_cbk - 
- */
-int
-unify_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, dict_t *xattr)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, xattr);
-	return 0;
-}
-
-/**
- * unify_fxattrop
- */
-int
-unify_fxattrop (call_frame_t *frame, xlator_t *this,
-		fd_t *fd, gf_xattrop_flags_t optype, dict_t *xattr)
-{
-	UNIFY_CHECK_FD_CTX_AND_UNWIND_ON_ERR (fd);
-	xlator_t *child = NULL;
-	uint64_t tmp_child = 0;
-
-	fd_ctx_get (fd, this, &tmp_child);
-	child = (xlator_t *)(long)tmp_child;		     
-
-	STACK_WIND (frame, unify_fxattrop_cbk,
-		    child, child->fops->fxattrop,
-		    fd, optype, xattr);
-
-	return 0;
-}
-
-
-/**
- * unify_inodelk_cbk - 
- */
-int
-unify_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-
-/**
- * unify_inodelk
- */
-int
-unify_inodelk (call_frame_t *frame, xlator_t *this,
-	       const char *volume, loc_t *loc, int cmd, struct gf_flock *flock)
-{
-	xlator_t *child = NULL;
-
-	child = unify_loc_subvol (loc, this);
-
-	STACK_WIND (frame, unify_inodelk_cbk,
-		    child, child->fops->inodelk,
-		    volume, loc, cmd, flock);
-
-	return 0;
-}
-
-
-
-/**
- * unify_entrylk_cbk - 
- */
-int
-unify_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno)
-{
-	STACK_UNWIND (frame, op_ret, op_errno);
-	return 0;
-}
-
-/**
- * unify_entrylk
- */
-int
-unify_entrylk (call_frame_t *frame, xlator_t *this,
-	       const char *volume, loc_t *loc, const char *basename,
-	       entrylk_cmd cmd, entrylk_type type)
-		
-{
-	xlator_t *child = NULL;
-
-	child = unify_loc_subvol (loc, this);
-
-	STACK_WIND (frame, unify_entrylk_cbk,
-		    child, child->fops->entrylk,
-		    volume, loc, basename, cmd, type);
-
-	return 0;
-}
-
-
-
-/**
- * unify_xattrop_cbk - 
- */
-int
-unify_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		   int32_t op_ret, int32_t op_errno, dict_t *xattr)
-{
-	STACK_UNWIND (frame, op_ret, op_errno, xattr);
-	return 0;
-}
-
-/**
- * unify_xattrop
- */
-int
-unify_xattrop (call_frame_t *frame, xlator_t *this,
-		loc_t *loc, gf_xattrop_flags_t optype, dict_t *xattr)
-{
-	xlator_t *child = NULL;
-
-	child = unify_loc_subvol (loc, this);
-
-	STACK_WIND (frame, unify_xattrop_cbk,
-		    child, child->fops->xattrop,
-		    loc, optype, xattr);
-
-	return 0;
-}
-
-int
-unify_forget (xlator_t *this,
-              inode_t *inode)
-{
-        int16_t *list = NULL;
-        uint64_t tmp_list = 0;
-
-        if (inode->ia_type && (!IA_ISDIR(inode->ia_type))) {
-                inode_ctx_get (inode, this, &tmp_list);
-                if (tmp_list) {
-                        list = (int16_t *)(long)tmp_list;
-                        GF_FREE (list);
-                }
-        }
-
-        return 0;
-}
-
-/**
- * notify
- */
-int32_t
-notify (xlator_t *this,
-        int32_t event,
-        void *data,
-        ...)
-{
-	unify_private_t *priv = this->private;
-	struct sched_ops *sched = NULL;
-
-	if (!priv) {
-		return 0;
-	}
-
-	sched = priv->sched_ops;    
-	if (!sched) {
-		gf_log (this->name, GF_LOG_CRITICAL, "No scheduler :O");
-		raise (SIGTERM);
-		return 0;
-	}
-	if (priv->namespace == data) {
-		if (event == GF_EVENT_CHILD_UP) {
-			sched->notify (this, event, data);
-		}
-		return 0;
-	}
-
-	switch (event)
-	{
-	case GF_EVENT_CHILD_UP:
-	{
-		/* Call scheduler's update () to enable it for scheduling */
-		sched->notify (this, event, data);
-	
-		LOCK (&priv->lock);
-		{
-			/* Increment the inode's generation, which is 
-			   used for self_heal */
-			++priv->inode_generation;
-			++priv->num_child_up;
-		}
-		UNLOCK (&priv->lock);
-
-		if (!priv->is_up) {
-			default_notify (this, event, data);
-			priv->is_up = 1;
-		}
-	}
-	break;
-	case GF_EVENT_CHILD_DOWN:
-	{
-		/* Call scheduler's update () to disable the child node 
-		 * for scheduling
-		 */
-		sched->notify (this, event, data);
-		LOCK (&priv->lock);
-		{
-			--priv->num_child_up;
-		}
-		UNLOCK (&priv->lock);
-
-		if (priv->num_child_up == 0) {
-			/* Send CHILD_DOWN to upper layer */
-			default_notify (this, event, data);
-			priv->is_up = 0;
-		}
-	}
-	break;
-
-	default:
-	{
-		default_notify (this, event, data);
-	}
-	break;
-	}
-
-	return 0;
-}
-
-int32_t
-mem_acct_init (xlator_t *this)
-{
-        int     ret = -1;
-
-        if (!this)
-                return ret;
-
-        ret = xlator_mem_acct_init (this, gf_unify_mt_end + 1);
-        
-        if (ret != 0) {
-                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
-                                "failed");
-                return ret;
-        }
-
-        return ret;
-}
-
-/** 
- * init - This function is called first in the xlator, while initializing.
- *   All the config file options are checked and appropriate flags are set.
- *
- * @this - 
- */
-int32_t 
-init (xlator_t *this)
-{
-	int32_t          ret       = 0;
-	int32_t          count     = 0;
-	data_t          *scheduler = NULL;
-	data_t          *data      = NULL;
-	xlator_t        *ns_xl     = NULL;
-	xlator_list_t   *trav      = NULL;
-	xlator_list_t   *xlparent  = NULL;
-	xlator_list_t   *parent    = NULL;
-	unify_private_t *_private  = NULL; 
-
-
-	/* Check for number of child nodes, if there is no child nodes, exit */
-	if (!this->children) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"No child nodes specified. check \"subvolumes \" "
-			"option in volfile");
-		return -1;
-	}
-
-  	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-  
-	/* Check for 'scheduler' in volume */
-	scheduler = dict_get (this->options, "scheduler");
-	if (!scheduler) {
-		gf_log (this->name, GF_LOG_ERROR, 
-			"\"option scheduler <x>\" is missing in volfile");
-		return -1;
-	}
-
-	/* Setting "option namespace <node>" */
-	data = dict_get (this->options, "namespace");
-	if(!data) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"namespace option not specified, Exiting");
-		return -1;
-	}
-	/* Search namespace in the child node, if found, exit */
-	trav = this->children;
-	while (trav) {
-		if (strcmp (trav->xlator->name, data->data) == 0)
-			break;
-		trav = trav->next;
-	}
-	if (trav) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"namespace node used as a subvolume, Exiting");
-		return -1;
-	}
-	
-	/* Search for the namespace node, if found, continue */
-	ns_xl = this->next;
-	while (ns_xl) {
-		if (strcmp (ns_xl->name, data->data) == 0)
-			break;
-		ns_xl = ns_xl->next;
-	}
-	if (!ns_xl) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"namespace node not found in volfile, Exiting");
-		return -1;
-	}
-	
-	gf_log (this->name, GF_LOG_DEBUG, 
-		"namespace node specified as %s", data->data);
-	
-	_private = GF_CALLOC (1, sizeof (*_private), 
-                              gf_unify_mt_unify_private_t);
-	ERR_ABORT (_private);
-	_private->sched_ops = get_scheduler (this, scheduler->data);
-	if (!_private->sched_ops) {
-		gf_log (this->name, GF_LOG_CRITICAL, 
-			"Error while loading scheduler. Exiting");
-		GF_FREE (_private);
-		return -1;
-	}
-	
-	if (ns_xl->parents) {
-		gf_log (this->name, GF_LOG_CRITICAL,
-			"Namespace node should not be a child of any other node. Exiting");
-		GF_FREE (_private);
-		return -1;
-	}
-
-	_private->namespace = ns_xl;
-	
-	/* update _private structure */
-	{
-		count = 0;
-		trav = this->children;
-		/* Get the number of child count */
-		while (trav) {
-			count++;
-			trav = trav->next;
-		}
-		
-		gf_log (this->name, GF_LOG_DEBUG, 
-			"Child node count is %d", count);    
-
-		_private->child_count = count;
-		if (count == 1) {
-			/* TODO: Should I error out here? */
-			gf_log (this->name, GF_LOG_CRITICAL, 
-				"WARNING: You have defined only one "
-				"\"subvolumes\" for unify volume. It may not "
-				"be the desired config, review your volume "
-				"volfile. If this is how you are testing it,"
-				" you may hit some performance penalty");
-		}
-		
-		_private->xl_array = GF_CALLOC (1, 
-					        sizeof (xlator_t) * (count + 1),
-                                                gf_unify_mt_xlator_t);
-		ERR_ABORT (_private->xl_array);
-		
-		count = 0;
-		trav = this->children;
-		while (trav) {
-			_private->xl_array[count++] = trav->xlator;
-			trav = trav->next;
-		}
-		_private->xl_array[count] = _private->namespace;
-		
-		/* self-heal part, start with generation '1' */
-		_private->inode_generation = 1; 
-                /* Because, Foreground part is tested well */
-		_private->self_heal = ZR_UNIFY_FG_SELF_HEAL; 
-		data = dict_get (this->options, "self-heal");
-		if (data) {
-			if (strcasecmp (data->data, "off") == 0) 
-				_private->self_heal = ZR_UNIFY_SELF_HEAL_OFF;
-
-			if (strcasecmp (data->data, "foreground") == 0)
-				_private->self_heal = ZR_UNIFY_FG_SELF_HEAL;
-
-			if (strcasecmp (data->data, "background") == 0)
-				_private->self_heal = ZR_UNIFY_BG_SELF_HEAL;
-		}
-    
-		/* optimist - ask bulde for more about it */
-		data = dict_get (this->options, "optimist");
-		if (data) {
-			if (gf_string2boolean (data->data, 
-					       &_private->optimist) == -1) {
-				gf_log (this->name, GF_LOG_ERROR, 
-					"optimist excepts only boolean "
-					"options");
-			}
-		}
-
-		LOCK_INIT (&_private->lock);
-	}
-
-	/* Now that everything is fine. */
-	this->private = (void *)_private;
-	{
-                ret = _private->sched_ops->mem_acct_init (this);
-
-                if (ret == -1) {
-                        return -1;
-                }
-
-		/* Initialize scheduler, if everything else is successful */
-		ret = _private->sched_ops->init (this); 
-		if (ret == -1) {
-			gf_log (this->name, GF_LOG_CRITICAL,
-				"Initializing scheduler failed, Exiting");
-			GF_FREE (_private);
-			return -1;
-		}
-
-
-		ret = 0;
-
-		/* This section is required because some fops may look 
-		 * for 'xl->parent' variable 
-		 */
-		xlparent = GF_CALLOC (1, sizeof (*xlparent),
-                                      gf_unify_mt_xlator_list_t);
-		xlparent->xlator = this;
-		if (!ns_xl->parents) {
-			ns_xl->parents = xlparent;
-		} else {
-			parent = ns_xl->parents;
-			while (parent->next)
-				parent = parent->next;
-			parent->next = xlparent;
-		}
-	}
-
-	/* Tell namespace node that init is done */
-	xlator_notify (ns_xl, GF_EVENT_PARENT_UP, this);
-
-	return 0;
-}
-
-/** 
- * fini  - Free all the allocated memory 
- */
-void
-fini (xlator_t *this)
-{
-	unify_private_t *priv = this->private;
-	priv->sched_ops->fini (this);
-	this->private = NULL;
-	LOCK_DESTROY (&priv->lock);
-	GF_FREE (priv->xl_array);
-	GF_FREE (priv);
-	return;
-}
-
-
-struct xlator_fops fops = {
-	.stat        = unify_stat,
-	.readlink    = unify_readlink,
-	.mknod       = unify_mknod,
-	.mkdir       = unify_mkdir,
-	.unlink      = unify_unlink,
-	.rmdir       = unify_rmdir,
-	.symlink     = unify_symlink,
-	.rename      = unify_rename,
-	.link        = unify_link,
-	.truncate    = unify_truncate,
-	.create      = unify_create,
-	.open        = unify_open,
-	.readv       = unify_readv,
-	.writev      = unify_writev,
-	.statfs      = unify_statfs,
-	.flush       = unify_flush,
-	.fsync       = unify_fsync,
-	.setxattr    = unify_setxattr,
-	.getxattr    = unify_getxattr,
-	.removexattr = unify_removexattr,
-	.opendir     = unify_opendir,
-	.readdir     = unify_readdir,
-	.readdirp    = unify_readdirp,
-	.fsyncdir    = unify_fsyncdir,
-	.access      = unify_access,
-	.ftruncate   = unify_ftruncate,
-	.fstat       = unify_fstat,
-	.lk          = unify_lk,
-	.lookup      = unify_lookup,
-	.getdents    = unify_getdents,
-	.checksum    = unify_checksum,
-	.inodelk     = unify_inodelk,
-	.finodelk    = unify_finodelk,
-	.entrylk     = unify_entrylk,
-	.fentrylk    = unify_fentrylk,
-	.xattrop     = unify_xattrop,
-	.fxattrop    = unify_fxattrop,
-        .setattr     = unify_setattr, 
-        .fsetattr    = unify_fsetattr,
-};
-
-
-struct xlator_cbks cbks = {
-        .forget  = unify_forget,
-};
-
-struct volume_options options[] = {
-	{ .key   = { "namespace" },  
-	  .type  = GF_OPTION_TYPE_XLATOR 
-	},
-	{ .key   = { "scheduler" },  
-	  .value = { "alu", "rr", "random", "nufa", "switch" },
-	  .type  = GF_OPTION_TYPE_STR
-	},
-	{ .key   = {"self-heal"},  
-	  .value = { "foreground", "background", "off" },
-	  .type  = GF_OPTION_TYPE_STR
-	},
-	/* TODO: remove it some time later */
-	{ .key   = {"optimist"},  
-	  .type  = GF_OPTION_TYPE_BOOL 
-	},
-
-	{ .key   = {NULL} },
-};
diff --git a/xlators/cluster/unify/src/unify.h b/xlators/cluster/unify/src/unify.h
deleted file mode 100644
index 18e3db11f..000000000
--- a/xlators/cluster/unify/src/unify.h
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
-  Copyright (c) 2006-2009 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#ifndef _UNIFY_H
-#define _UNIFY_H
-
-#include "scheduler.h"
-#include "list.h"
-#include "unify-mem-types.h"
-
-#define MAX_DIR_ENTRY_STRING     (32 * 1024)
-
-#define ZR_UNIFY_SELF_HEAL_OFF 0
-#define ZR_UNIFY_FG_SELF_HEAL  1
-#define ZR_UNIFY_BG_SELF_HEAL  2
-
-/* Sometimes one should use completely random numbers.. its good :p */
-#define UNIFY_SELF_HEAL_GETDENTS_COUNT 512
-
-#define NS(xl)          (((unify_private_t *)xl->private)->namespace)
-
-/* This is used to allocate memory for local structure */
-#define INIT_LOCAL(fr, loc)                   \
-do {                                          \
-  loc = GF_CALLOC (1, sizeof (unify_local_t), gf_unify_mt_unify_local_t);   \
-  ERR_ABORT (loc);			      \
-  if (!loc) {                                 \
-    STACK_UNWIND (fr, -1, ENOMEM);            \
-    return 0;                                 \
-  }                                           \
-  fr->local = loc;                            \
-  loc->op_ret = -1;                           \
-  loc->op_errno = ENOENT;                     \
-} while (0)
-
-
-
-struct unify_private {
-	/* Update this structure depending on requirement */
-	void *scheduler;               /* THIS SHOULD BE THE FIRST VARIABLE, 
-					  if xlator is using scheduler */
-	struct sched_ops *sched_ops;   /* Scheduler options  */
-	xlator_t *namespace;           /* ptr to namespace xlator */
-	xlator_t **xl_array;
-	gf_boolean_t optimist;
-	int16_t child_count;
-	int16_t num_child_up;
-	uint8_t self_heal;
-	uint8_t is_up;
-	uint64_t inode_generation;
-	gf_lock_t lock;
-};
-typedef struct unify_private unify_private_t;
-
-struct unify_self_heal_struct {
-	uint8_t dir_checksum[NAME_MAX];
-	uint8_t ns_dir_checksum[NAME_MAX];
-	uint8_t file_checksum[NAME_MAX];
-	uint8_t ns_file_checksum[NAME_MAX];
-	off_t *offset_list;
-	int   *count_list;
-	dir_entry_t **entry_list;
-};
-
-
-struct _unify_local_t {
-	int32_t call_count;
-	int32_t op_ret;
-	int32_t op_errno;
-	mode_t mode;
-	off_t offset;
-	dev_t dev;
-	uid_t uid;
-	gid_t gid;
-	int32_t flags;
-	int32_t entry_count;
-	int32_t count;    // dir_entry_t count;
-	fd_t *fd;
-	struct iatt stbuf;
-        struct iatt stpre;
-        struct iatt stpost;
-	struct statvfs statvfs_buf;
-	struct timespec tv[2];
-	char *name;
-	int32_t revalidate;
-
-	ino_t ia_ino;
-	nlink_t ia_nlink;
-  
-	dict_t *dict;
-
-	int16_t *list;
-	int16_t *new_list; /* Used only in case of rename */
-	int16_t index;
-
-	int32_t failed;
-	int32_t return_eio;  /* Used in case of different st-mode 
-				present for a given path */
-
-	uint64_t inode_generation; /* used to store the per directory 
-				    * inode_generation. Got from inode's ctx 
-				    * of directory inodes
-				    */
-
-	struct unify_self_heal_struct *sh_struct;
-	loc_t loc1, loc2;
-
-        struct iatt poststbuf;
-        /* When not used for rename, old*
-         * are used as the attrs for the current
-         * parent directory.
-         */
-        struct iatt oldpreparent;
-        struct iatt oldpostparent;
-        struct iatt newpreparent;
-        struct iatt newpostparent;
-        int32_t wbflags;
-};
-typedef struct _unify_local_t unify_local_t;
-
-int32_t zr_unify_self_heal (call_frame_t *frame,
-			    xlator_t *this,
-			    unify_local_t *local);
-
-#endif /* _UNIFY_H */
diff --git a/xlators/debug/error-gen/src/Makefile.am b/xlators/debug/error-gen/src/Makefile.am
index f353b61e6..5075c59a8 100644
--- a/xlators/debug/error-gen/src/Makefile.am
+++ b/xlators/debug/error-gen/src/Makefile.am
@@ -2,15 +2,16 @@
 xlator_LTLIBRARIES = error-gen.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
 
-error_gen_la_LDFLAGS = -module -avoidversion
+error_gen_la_LDFLAGS = -module -avoid-version
 
 error_gen_la_SOURCES = error-gen.c
 error_gen_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = error-gen.h
+noinst_HEADERS = error-gen.h error-gen-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/debug/error-gen/src/error-gen-mem-types.h b/xlators/debug/error-gen/src/error-gen-mem-types.h
new file mode 100644
index 000000000..f02280535
--- /dev/null
+++ b/xlators/debug/error-gen/src/error-gen-mem-types.h
@@ -0,0 +1,20 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __ERROR_GEN_MEM_TYPES_H__
+#define __ERROR_GEN_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_error_gen_mem_types_ {
+        gf_error_gen_mt_eg_t = gf_common_mt_end + 1,
+        gf_error_gen_mt_end
+};
+#endif
diff --git a/xlators/debug/error-gen/src/error-gen.c b/xlators/debug/error-gen/src/error-gen.c
index 8945de43e..ec0874b35 100644
--- a/xlators/debug/error-gen/src/error-gen.c
+++ b/xlators/debug/error-gen/src/error-gen.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -24,6 +14,7 @@
 
 #include "xlator.h"
 #include "error-gen.h"
+#include "statedump.h"
 
 sys_error_t error_no_list[] = {
         [GF_FOP_LOOKUP]            = { .error_no_count = 4,
@@ -91,9 +82,10 @@ sys_error_t error_no_list[] = {
         [GF_FOP_READ]              = { .error_no_count = 5,
                                     .error_no = {EINVAL,EBADF,EFAULT,EISDIR,
                                                  ENAMETOOLONG}},
-        [GF_FOP_WRITE]             = { .error_no_count = 5,
+        [GF_FOP_WRITE]             = { .error_no_count = 7,
                                     .error_no = {EINVAL,EBADF,EFAULT,EISDIR,
-                                                 ENAMETOOLONG}},
+                                                 ENAMETOOLONG,ENOSPC,
+						 GF_ERROR_SHORT_WRITE}},
         [GF_FOP_STATFS]            = {.error_no_count = 10,
                                    .error_no = {EACCES,EBADF,EFAULT,EINTR,
                                                 EIO,ENAMETOOLONG,ENOENT,
@@ -113,6 +105,15 @@ sys_error_t error_no_list[] = {
         [GF_FOP_REMOVEXATTR]       = { .error_no_count = 4,
                                     .error_no = {EACCES,EBADF,ENAMETOOLONG,
                                                  EINTR}},
+        [GF_FOP_FSETXATTR]          = { .error_no_count = 4,
+                                        .error_no = {EACCES,EBADF,EINTR,
+                                                     ENAMETOOLONG}},
+        [GF_FOP_FGETXATTR]          = { .error_no_count = 4,
+                                        .error_no = {EACCES,EBADF,ENAMETOOLONG,
+                                                     EINTR}},
+        [GF_FOP_FREMOVEXATTR]       = { .error_no_count = 4,
+                                        .error_no = {EACCES,EBADF,ENAMETOOLONG,
+                                                     EINTR}},
         [GF_FOP_OPENDIR]           = { .error_no_count = 8,
                                     .error_no = {EACCES,EEXIST,EFAULT,
                                                  EISDIR,EMFILE,
@@ -237,6 +238,8 @@ conv_errno_to_int (char **error_no)
                 return EINTR;
         else if (!strcmp ((*error_no), "EFBIG"))
                 return EFBIG;
+	else if (!strcmp((*error_no), "GF_ERROR_SHORT_WRITE"))
+		return GF_ERROR_SHORT_WRITE;
         else
                 return EAGAIN;
 }
@@ -286,6 +289,12 @@ get_fop_int (char **op_no_str)
                 return GF_FOP_GETXATTR;
         else if (!strcmp ((*op_no_str), "removexattr"))
                 return GF_FOP_REMOVEXATTR;
+        else if (!strcmp ((*op_no_str), "fsetxattr"))
+                return GF_FOP_FSETXATTR;
+        else if (!strcmp ((*op_no_str), "fgetxattr"))
+                return GF_FOP_FGETXATTR;
+        else if (!strcmp ((*op_no_str), "fremovexattr"))
+                return GF_FOP_FREMOVEXATTR;
         else if (!strcmp ((*op_no_str), "opendir"))
                 return GF_FOP_OPENDIR;
         else if (!strcmp ((*op_no_str), "readdir"))
@@ -362,7 +371,8 @@ error_gen (xlator_t *this, int op_no)
                                 rand_no = 0;
                         ret = error_no_list[op_no].error_no[rand_no];
                 }
-                egp->failure_iter_no = 3 + (rand () % GF_UNIVERSAL_ANSWER);
+                if (egp->random_failure == _gf_true)
+                        egp->failure_iter_no = 3 + (rand () % GF_UNIVERSAL_ANSWER);
         }
         return ret;
 }
@@ -371,17 +381,17 @@ error_gen (xlator_t *this, int op_no)
 int
 error_gen_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		      int32_t op_ret, int32_t op_errno, inode_t *inode,
-		      struct iatt *buf, dict_t *dict, struct iatt *postparent)
+		      struct iatt *buf, dict_t *xdata, struct iatt *postparent)
 {
 	STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode,
-                             buf, dict, postparent);
-	return 0;
+                             buf, xdata, postparent);
+        return 0;
 }
 
 
 int
 error_gen_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  dict_t *xattr_req)
+                  dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -396,36 +406,28 @@ error_gen_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (lookup, frame, -1, op_errno, NULL, NULL, NULL,
-                              NULL);
-		return 0;
+                                     NULL);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_lookup_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->lookup,
-		    loc, xattr_req);
-	return 0;
-}
-
-
-int
-error_gen_forget (xlator_t *this, inode_t *inode)
-{
-	return 0;
+		    loc, xdata);
+        return 0;
 }
 
 
 int
 error_gen_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, struct iatt *buf)
+		    int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf);
-
-	return 0;
+	STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
 }
 
 int
-error_gen_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+error_gen_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -439,32 +441,31 @@ error_gen_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (stat, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_stat_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->stat,
-		    loc);
-	return 0;
+		    loc, xdata);
+        return 0;
 }
 
 
 int
 error_gen_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno,
-                       struct iatt *preop, struct iatt *postop)
+                       struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop);
-
-	return 0;
+	STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop, xdata);
+        return 0;
 }
 
 
 int
 error_gen_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   struct iatt *stbuf, int32_t valid)
+                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -478,21 +479,21 @@ error_gen_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (setattr, frame, -1, op_errno, NULL, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (setattr, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_setattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->setattr,
-		    loc, stbuf, valid);
-	return 0;
+		    loc, stbuf, valid, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    struct iatt *stbuf, int32_t valid)
+                    struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
 	int             op_errno = 0;
         eg_t            *egp = NULL;
@@ -506,32 +507,32 @@ error_gen_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsetattr, frame, -1, op_errno, NULL, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (fsetattr, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_setattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fsetattr,
-		    fd, stbuf, valid);
-	return 0;
+		    fd, stbuf, valid, xdata);
+        return 0;
 }
 
 
 int
 error_gen_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 			int32_t op_ret, int32_t op_errno,
-			struct iatt *prebuf, struct iatt *postbuf)
+			struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno,
-                             prebuf, postbuf);
-	return 0;
+                             prebuf, postbuf, xdata);
+        return 0;
 }
 
 
 int
 error_gen_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    off_t offset)
+                    off_t offset, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -546,32 +547,32 @@ error_gen_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (truncate, frame, -1, op_errno,
-                                     NULL, NULL);
-		return 0;
+                                     NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_truncate_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->truncate,
-		    loc, offset);
-	return 0;
+		    loc, offset, xdata);
+        return 0;
 }
 
 
 int
 error_gen_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 			 int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                         struct iatt *postbuf)
+                         struct iatt *postbuf, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno,
-                             prebuf, postbuf);
-	return 0;
+                             prebuf, postbuf, xdata);
+        return 0;
 }
 
 
 int
 error_gen_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		     off_t offset)
+		     off_t offset, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp =NULL;
@@ -586,31 +587,30 @@ error_gen_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (ftruncate, frame, -1, op_errno,
-                                     NULL, NULL);
-		return 0;
+                                     NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_ftruncate_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->ftruncate,
-		    fd, offset);
-	return 0;
+		    fd, offset, xdata);
+        return 0;
 }
 
 
 int
 error_gen_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno)
+		      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (access, frame, op_ret, op_errno);
-
-	return 0;
+	STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		  int32_t mask)
+		  int32_t mask, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -624,31 +624,31 @@ error_gen_access (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (access, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (access, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_access_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->access,
-		    loc, mask);
-	return 0;
+		    loc, mask, xdata);
+        return 0;
 }
 
 
 int
 error_gen_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 			int32_t op_ret, int32_t op_errno,
-			const char *path, struct iatt *sbuf)
+			const char *path, struct iatt *sbuf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, sbuf);
-	return 0;
+	STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, sbuf, xdata);
+        return 0;
 }
 
 
 int
 error_gen_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    size_t size)
+                    size_t size, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -662,15 +662,15 @@ error_gen_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readlink, frame, -1, op_errno, NULL, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (readlink, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_readlink_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->readlink,
-		    loc, size);
-	return 0;
+		    loc, size, xdata);
+        return 0;
 }
 
 
@@ -678,18 +678,18 @@ int
 error_gen_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		     int32_t op_ret, int32_t op_errno, inode_t *inode,
                      struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent)
+                     struct iatt *postparent, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno,
                              inode, buf,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
 error_gen_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		 mode_t mode, dev_t rdev, dict_t *params)
+		 mode_t mode, dev_t rdev, mode_t umask, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -704,15 +704,15 @@ error_gen_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL);
-		return 0;
+                                     NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_mknod_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->mknod,
-		    loc, mode, rdev, params);
-	return 0;
+		    loc, mode, rdev, umask, xdata);
+        return 0;
 }
 
 
@@ -720,17 +720,17 @@ int
 error_gen_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		     int32_t op_ret, int32_t op_errno, inode_t *inode,
                      struct iatt *buf, struct iatt *preparent,
-                     struct iatt *postparent)
+                     struct iatt *postparent, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno,
                              inode, buf,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 int
 error_gen_mkdir (call_frame_t *frame, xlator_t *this,
-		 loc_t *loc, mode_t mode, dict_t *params)
+		 loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -745,31 +745,32 @@ error_gen_mkdir (call_frame_t *frame, xlator_t *this,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (mkdir, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL);
-		return 0;
+                                     NULL, NULL, xdata);
+                return 0;
 	}
 
 	STACK_WIND (frame, error_gen_mkdir_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->mkdir,
-		    loc, mode, params);
-	return 0;
+		    loc, mode, umask, xdata);
+        return 0;
 }
 
 
 int
 error_gen_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		      int32_t op_ret, int32_t op_errno,
-                      struct iatt *preparent, struct iatt *postparent)
+                      struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
-error_gen_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+error_gen_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+                  dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -783,31 +784,34 @@ error_gen_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL, NULL,
+                                     xdata);
+                return 0;
 	}
 
 	STACK_WIND (frame, error_gen_unlink_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->unlink,
-		    loc);
-	return 0;
+		    loc, xflag, xdata);
+        return 0;
 }
 
 
 int
 error_gen_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		     int32_t op_ret, int32_t op_errno,
-                     struct iatt *preparent, struct iatt *postparent)
+                     struct iatt *preparent, struct iatt *postparent,
+                     dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
-error_gen_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+error_gen_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+                 dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -821,15 +825,15 @@ error_gen_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_rmdir_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->rmdir,
-		    loc, flags);
-	return 0;
+		    loc, flags, xdata);
+        return 0;
 }
 
 
@@ -837,17 +841,17 @@ int
 error_gen_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		       int32_t op_ret, int32_t op_errno, inode_t *inode,
                        struct iatt *buf, struct iatt *preparent,
-                       struct iatt *postparent)
+                       struct iatt *postparent, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
 error_gen_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-		   loc_t *loc, dict_t *params)
+		   loc_t *loc, mode_t umask, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -862,15 +866,15 @@ error_gen_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (symlink, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL); /* pre & post parent attr */
+                                     NULL, NULL, NULL); /* pre & post parent attr */
 		return 0;
 	}
 
 	STACK_WIND (frame, error_gen_symlink_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->symlink,
-		    linkpath, loc, params);
-	return 0;
+		    linkpath, loc, umask, xdata);
+        return 0;
 }
 
 
@@ -878,18 +882,19 @@ int
 error_gen_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		      int32_t op_ret, int32_t op_errno, struct iatt *buf,
                       struct iatt *preoldparent, struct iatt *postoldparent,
-                      struct iatt *prenewparent, struct iatt *postnewparent)
+                      struct iatt *prenewparent, struct iatt *postnewparent,
+                      dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
                              preoldparent, postoldparent,
-                             prenewparent, postnewparent);
-	return 0;
+                             prenewparent, postnewparent, xdata);
+        return 0;
 }
 
 
 int
 error_gen_rename (call_frame_t *frame, xlator_t *this,
-		  loc_t *oldloc, loc_t *newloc)
+		  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -904,15 +909,15 @@ error_gen_rename (call_frame_t *frame, xlator_t *this,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL,
-                                     NULL, NULL, NULL, NULL); /* pre & post parent attr */
+                                     NULL, NULL, NULL, NULL, NULL);
 		return 0;
 	}
 
 	STACK_WIND (frame, error_gen_rename_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->rename,
-		    oldloc, newloc);
-	return 0;
+		    oldloc, newloc, xdata);
+        return 0;
 }
 
 
@@ -920,17 +925,17 @@ int
 error_gen_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		    int32_t op_ret, int32_t op_errno, inode_t *inode,
                     struct iatt *buf, struct iatt *preparent,
-                    struct iatt *postparent)
+                    struct iatt *postparent, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
 error_gen_link (call_frame_t *frame, xlator_t *this,
-		loc_t *oldloc, loc_t *newloc)
+		loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
 	int             op_errno = 0;
         eg_t            *egp = NULL;
@@ -945,15 +950,15 @@ error_gen_link (call_frame_t *frame, xlator_t *this,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (link, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL); /* pre & post parent attr */
+                                     NULL, NULL, NULL);
 		return 0;
 	}
 
 	STACK_WIND (frame, error_gen_link_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->link,
-		    oldloc, newloc);
-	return 0;
+		    oldloc, newloc, xdata);
+        return 0;
 }
 
 
@@ -961,17 +966,18 @@ int
 error_gen_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		      int32_t op_ret, int32_t op_errno,
 		      fd_t *fd, inode_t *inode, struct iatt *buf,
-                      struct iatt *preparent, struct iatt *postparent)
+                      struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
-	return 0;
+                             preparent, postparent, xdata);
+        return 0;
 }
 
 
 int
 error_gen_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		  int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
+		  int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+                  dict_t *xdata)
 {
 	int             op_errno = 0;
         eg_t            *egp = NULL;
@@ -986,30 +992,30 @@ error_gen_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, NULL); /* pre & post attr */
+                                     NULL, NULL, NULL, NULL);
 		return 0;
 	}
 
 	STACK_WIND (frame, error_gen_create_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->create,
-		    loc, flags, mode, fd, params);
-	return 0;
+		    loc, flags, mode, umask, fd, xdata);
+        return 0;
 }
 
 
 int
 error_gen_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		    int32_t op_ret, int32_t op_errno, fd_t *fd)
+		    int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
-	return 0;
+	STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+        return 0;
 }
 
 
 int
 error_gen_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		int32_t flags, fd_t *fd, int32_t wbflags)
+		int32_t flags, fd_t *fd, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1023,15 +1029,15 @@ error_gen_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (open, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (open, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_open_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->open,
-		    loc, flags, fd, wbflags);
-	return 0;
+		    loc, flags, fd, xdata);
+        return 0;
 }
 
 
@@ -1039,17 +1045,17 @@ int
 error_gen_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		     int32_t op_ret, int32_t op_errno,
 		     struct iovec *vector, int32_t count,
-		     struct iatt *stbuf, struct iobref *iobref)
+		     struct iatt *stbuf, struct iobref *iobref, dict_t *xdata)
 {
 	STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                             vector, count, stbuf, iobref);
-	return 0;
+                             vector, count, stbuf, iobref, xdata);
+        return 0;
 }
 
 
 int
 error_gen_readv (call_frame_t *frame, xlator_t *this,
-		 fd_t *fd, size_t size, off_t offset)
+		 fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1064,33 +1070,33 @@ error_gen_readv (call_frame_t *frame, xlator_t *this,
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
 		STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0,
-                                     NULL, NULL);
-		return 0;
+                                     NULL, NULL, xdata);
+        return 0;
 	}
 
 
 	STACK_WIND (frame, error_gen_readv_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->readv,
-		    fd, size, offset);
-	return 0;
+		    fd, size, offset, flags, xdata);
+        return 0;
 }
 
 
 int
 error_gen_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		      int32_t op_ret, int32_t op_errno,
-                      struct iatt *prebuf, struct iatt *postbuf)
+                      struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
+	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
 }
 
 
 int
 error_gen_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 		  struct iovec *vector, int32_t count,
-		  off_t off, struct iobref *iobref)
+		  off_t off, uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1102,31 +1108,47 @@ error_gen_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         if (enable)
                 op_errno = error_gen (this, GF_FOP_WRITE);
 
-	if (op_errno) {
-		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL);
+	if (op_errno == GF_ERROR_SHORT_WRITE) {
+		struct iovec *shortvec;
+
+		/*
+		 * A short write error returns some value less than what was
+		 * requested from a write. To simulate this, replace the vector
+		 * with one half the size;
+		 */
+		shortvec = iov_dup(vector, 1);
+		shortvec->iov_len /= 2;
+
+		STACK_WIND(frame, error_gen_writev_cbk, FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->writev, fd, shortvec, count,
+			   off, flags, iobref, xdata);
+		GF_FREE(shortvec);
 		return 0;
+	} else if (op_errno) {
+		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
+		STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL, xdata);
+        	return 0;
 	}
 
 	STACK_WIND (frame, error_gen_writev_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->writev,
-		    fd, vector, count, off, iobref);
-	return 0;
+		    fd, vector, count, off, flags, iobref, xdata);
+        return 0;
 }
 
 
 int
 error_gen_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		     int32_t op_ret, int32_t op_errno)
+		     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
-error_gen_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+error_gen_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1140,15 +1162,15 @@ error_gen_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (flush, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (flush, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_flush_cbk,
                     FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->flush,
-		    fd);
-	return 0;
+		    fd, xdata);
+        return 0;
 }
 
 
@@ -1156,15 +1178,15 @@ int
 error_gen_fsync_cbk (call_frame_t *frame, void *cookie,
 		     xlator_t *this, int32_t op_ret,
 		     int32_t op_errno, struct iatt *prebuf,
-                     struct iatt *postbuf)
+                     struct iatt *postbuf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf);
-	return 0;
+	STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
 }
 
 
 int
-error_gen_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+error_gen_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1178,29 +1200,29 @@ error_gen_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (fsync, frame, -1, op_errno, NULL, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fsync_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fsync,
-		    fd, flags);
-	return 0;
+		    fd, flags, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                     int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf);
-	return 0;
+	STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
 }
 
 
 int
-error_gen_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
+error_gen_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1214,29 +1236,29 @@ error_gen_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fstat, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (fstat, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fstat_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fstat,
-		    fd);
-	return 0;
+		    fd, xdata);
+        return 0;
 }
 
 
 int
 error_gen_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, fd_t *fd)
+		       int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd);
-	return 0;
+	STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
+        return 0;
 }
 
 
 int
-error_gen_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+error_gen_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1250,29 +1272,29 @@ error_gen_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (opendir, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (opendir, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_opendir_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->opendir,
-		    loc, fd);
-	return 0;
+		    loc, fd, xdata);
+        return 0;
 }
 
 int
 error_gen_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno)
+			int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    int32_t flags)
+                    int32_t flags, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1286,30 +1308,29 @@ error_gen_fsyncdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fsyncdir, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (fsyncdir, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fsyncdir_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fsyncdir,
-		    fd, flags);
-	return 0;
+		    fd, flags, xdata);
+        return 0;
 }
 
 
 int
 error_gen_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		      int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+		      int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf);
-
-	return 0;
+	STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
+        return 0;
 }
 
 
 int
-error_gen_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+error_gen_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1323,31 +1344,30 @@ error_gen_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (statfs, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (statfs, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_statfs_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->statfs,
-		    loc);
-	return 0;
+		    loc, xdata);
+        return 0;
 }
 
 
 int
 error_gen_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno)
+			int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
-
-	return 0;
+	STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    dict_t *dict, int32_t flags)
+		    dict_t *dict, int32_t flags, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1361,30 +1381,30 @@ error_gen_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_setxattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->setxattr,
-		    loc, dict, flags);
-	return 0;
+		    loc, dict, flags, xdata);
+        return 0;
 }
 
 
 int
 error_gen_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno, dict_t *dict)
+			int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
-	return 0;
+	STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 }
 
 
 int
 error_gen_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		    const char *name)
+		    const char *name, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1398,31 +1418,103 @@ error_gen_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (getxattr, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (getxattr, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_getxattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->getxattr,
-		    loc, name);
-	return 0;
+		    loc, name, xdata);
+        return 0;
+}
+
+int
+error_gen_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
-error_gen_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict)
+error_gen_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     dict_t *dict, int32_t flags, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict);
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
 
-	return 0;
+        egp = this->private;
+        enable = egp->enable[GF_FOP_FSETXATTR];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_FSETXATTR);
+
+	if (op_errno) {
+		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
+		STACK_UNWIND_STRICT (fsetxattr, frame, -1, op_errno, xdata);
+        return 0;
+	}
+
+	STACK_WIND (frame, error_gen_fsetxattr_cbk,
+		    FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->fsetxattr,
+		    fd, dict, flags, xdata);
+        return 0;
+}
+
+
+int
+error_gen_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
+}
+
+
+int
+error_gen_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                     const char *name, dict_t *xdata)
+{
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_FGETXATTR];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_FGETXATTR);
+
+	if (op_errno) {
+		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
+		STACK_UNWIND_STRICT (fgetxattr, frame, -1, op_errno, NULL, xdata);
+        return 0;
+	}
+
+	STACK_WIND (frame, error_gen_fgetxattr_cbk,
+		    FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->fgetxattr,
+		    fd, name, xdata);
+        return 0;
+}
+
+
+int
+error_gen_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 }
 
 
 int
 error_gen_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		   gf_xattrop_flags_t flags, dict_t *dict)
+		   gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1436,31 +1528,30 @@ error_gen_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (xattrop, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (xattrop, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_xattrop_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->xattrop,
-		    loc, flags, dict);
-	return 0;
+		    loc, flags, dict, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			int32_t op_ret, int32_t op_errno, dict_t *dict)
+			int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict);
-
-	return 0;
+	STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		    gf_xattrop_flags_t flags, dict_t *dict)
+		    gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1474,31 +1565,30 @@ error_gen_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fxattrop, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (fxattrop, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fxattrop_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fxattrop,
-		    fd, flags, dict);
-	return 0;
+		    fd, flags, dict, xdata);
+        return 0;
 }
 
 
 int
 error_gen_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-			   int32_t op_ret, int32_t op_errno)
+			   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno);
-
-	return 0;
+	STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-		       const char *name)
+		       const char *name, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1512,30 +1602,66 @@ error_gen_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (removexattr, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (removexattr, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_removexattr_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->removexattr,
-		    loc, name);
-	return 0;
+		    loc, name, xdata);
+        return 0;
+}
+
+int
+error_gen_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+			   int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+	STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+
+int
+error_gen_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        const char *name, dict_t *xdata)
+{
+	int              op_errno = 0;
+        eg_t            *egp = NULL;
+        int              enable = 1;
+
+        egp = this->private;
+        enable = egp->enable[GF_FOP_FREMOVEXATTR];
+
+        if (enable)
+                op_errno = error_gen (this, GF_FOP_FREMOVEXATTR);
+
+	if (op_errno) {
+		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
+		STACK_UNWIND_STRICT (fremovexattr, frame, -1, op_errno, xdata);
+        return 0;
+	}
+
+	STACK_WIND (frame, error_gen_fremovexattr_cbk,
+		    FIRST_CHILD(this),
+		    FIRST_CHILD(this)->fops->fremovexattr,
+		    fd, name, xdata);
+        return 0;
 }
 
 
 int
 error_gen_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		  int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+		  int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock);
-	return 0;
+	STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
+        return 0;
 }
 
 
 int
 error_gen_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
-	      struct gf_flock *lock)
+	      struct gf_flock *lock, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1549,32 +1675,31 @@ error_gen_lk (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t cmd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (lk, frame, -1, op_errno, NULL);
-		return 0;
+		STACK_UNWIND_STRICT (lk, frame, -1, op_errno, NULL, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_lk_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->lk,
-		    fd, cmd, lock);
-	return 0;
+		    fd, cmd, lock, xdata);
+        return 0;
 }
 
 
 int
-error_gen_inodelk_cbk (call_frame_t *frame, void *cookie,
-		       xlator_t *this, int32_t op_ret, int32_t op_errno)
-
+error_gen_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_inodelk (call_frame_t *frame, xlator_t *this,
 		   const char *volume, loc_t *loc, int32_t cmd,
-                   struct gf_flock *lock)
+                   struct gf_flock *lock, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1588,32 +1713,31 @@ error_gen_inodelk (call_frame_t *frame, xlator_t *this,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (inodelk, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (inodelk, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_inodelk_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->inodelk,
-		    volume, loc, cmd, lock);
-	return 0;
+		    volume, loc, cmd, lock, xdata);
+        return 0;
 }
 
 
 int
-error_gen_finodelk_cbk (call_frame_t *frame, void *cookie,
-			xlator_t *this, int32_t op_ret, int32_t op_errno)
-
+error_gen_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_finodelk (call_frame_t *frame, xlator_t *this,
 		    const char *volume, fd_t *fd, int32_t cmd,
-                    struct gf_flock *lock)
+                    struct gf_flock *lock, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1627,32 +1751,31 @@ error_gen_finodelk (call_frame_t *frame, xlator_t *this,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (finodelk, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (finodelk, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_finodelk_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->finodelk,
-		    volume, fd, cmd, lock);
-	return 0;
+		    volume, fd, cmd, lock, xdata);
+        return 0;
 }
 
 
 int
-error_gen_entrylk_cbk (call_frame_t *frame, void *cookie,
-		       xlator_t *this, int32_t op_ret, int32_t op_errno)
-
+error_gen_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_entrylk (call_frame_t *frame, xlator_t *this,
 		   const char *volume, loc_t *loc, const char *basename,
-		   entrylk_cmd cmd, entrylk_type type)
+		   entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1666,32 +1789,31 @@ error_gen_entrylk (call_frame_t *frame, xlator_t *this,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (entrylk, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (entrylk, frame, -1, op_errno, xdata);
+        return 0;
 	}
 
 	STACK_WIND (frame, error_gen_entrylk_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->entrylk,
-		    volume, loc, basename, cmd, type);
-	return 0;
+		    volume, loc, basename, cmd, type, xdata);
+        return 0;
 }
 
 
 int
-error_gen_fentrylk_cbk (call_frame_t *frame, void *cookie,
-			xlator_t *this, int32_t op_ret, int32_t op_errno)
-
+error_gen_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno);
-	return 0;
+	STACK_UNWIND_STRICT (fentrylk, frame, op_ret, op_errno, xdata);
+        return 0;
 }
 
 
 int
 error_gen_fentrylk (call_frame_t *frame, xlator_t *this,
 		    const char *volume, fd_t *fd, const char *basename,
-		    entrylk_cmd cmd, entrylk_type type)
+		    entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
 	int             op_errno = 0;
         eg_t            *egp = NULL;
@@ -1705,15 +1827,15 @@ error_gen_fentrylk (call_frame_t *frame, xlator_t *this,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (fentrylk, frame, -1, op_errno);
-		return 0;
+		STACK_UNWIND_STRICT (fentrylk, frame, -1, op_errno, xdata);
+                return 0;
 	}
 
 	STACK_WIND (frame, error_gen_fentrylk_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->fentrylk,
-		    volume, fd, basename, cmd, type);
-	return 0;
+		    volume, fd, basename, cmd, type, xdata);
+        return 0;
 }
 
 
@@ -1725,8 +1847,7 @@ error_gen_getspec_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 		       int32_t op_ret, int32_t op_errno, char *spec_data)
 {
 	STACK_UNWIND_STRICT (getspec, frame, op_ret, op_errno, spec_data);
-
-	return 0;
+        return 0;
 }
 
 
@@ -1760,16 +1881,17 @@ error_gen_getspec (call_frame_t *frame, xlator_t *this, const char *key,
 
 int
 error_gen_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-		       int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+		       int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                       dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries);
+	STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries, xdata);
 	return 0;
 }
 
 
 int
 error_gen_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-		   size_t size, off_t off)
+		   size_t size, off_t off, dict_t *xdata)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1783,30 +1905,31 @@ error_gen_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readdir, frame, -1, op_errno, NULL);
+		STACK_UNWIND_STRICT (readdir, frame, -1, op_errno, NULL, xdata);
 		return 0;
 	}
 
 	STACK_WIND (frame, error_gen_readdir_cbk,
 		    FIRST_CHILD(this),
 		    FIRST_CHILD(this)->fops->readdir,
-		    fd, size, off);
+		    fd, size, off, xdata);
 	return 0;
 }
 
 
 int
 error_gen_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, gf_dirent_t *entries)
+                        int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                        dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries);
+	STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
 	return 0;
 }
 
 
 int
 error_gen_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                    off_t off)
+                    off_t off, dict_t *dict)
 {
 	int              op_errno = 0;
         eg_t            *egp = NULL;
@@ -1820,44 +1943,153 @@ error_gen_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
 
 	if (op_errno) {
 		GF_ERROR(this, "unwind(-1, %s)", strerror (op_errno));
-		STACK_UNWIND_STRICT (readdirp, frame, -1, op_errno, NULL);
+		STACK_UNWIND_STRICT (readdirp, frame, -1, op_errno, NULL, NULL);
 		return 0;
 	}
 
 	STACK_WIND (frame, error_gen_readdirp_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readdirp,
-                    fd, size, off);
+                    fd, size, off, dict);
 	return 0;
 }
 
+static void
+error_gen_set_failure (eg_t *pvt, int percent)
+{
+        GF_ASSERT (pvt);
 
-int
-error_gen_closedir (xlator_t *this, fd_t *fd)
+        if (percent)
+                pvt->failure_iter_no = 100/percent;
+        else
+                pvt->failure_iter_no = 100/GF_FAILURE_DEFAULT;
+}
+
+static void
+error_gen_parse_fill_fops (eg_t *pvt, char *enable_fops)
 {
-	return 0;
+        char            *op_no_str = NULL;
+        int              op_no = -1;
+        int              i = 0;
+        xlator_t        *this = THIS;
+        char            *saveptr = NULL;
+
+        GF_ASSERT (pvt);
+        GF_ASSERT (this);
+
+        for (i = 0; i < GF_FOP_MAXVALUE; i++)
+                pvt->enable[i] = 0;
+
+        if (!enable_fops) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "All fops are enabled.");
+                for (i = 0; i < GF_FOP_MAXVALUE; i++)
+                        pvt->enable[i] = 1;
+        } else {
+                op_no_str = strtok_r (enable_fops, ",", &saveptr);
+                while (op_no_str) {
+                        op_no = get_fop_int (&op_no_str);
+                        if (op_no == -1) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Wrong option value %s", op_no_str);
+                        } else
+                                pvt->enable[op_no] = 1;
+
+                        op_no_str = strtok_r (NULL, ",", &saveptr);
+                }
+        }
 }
 
+int32_t
+error_gen_priv_dump (xlator_t *this)
+{
+        char            key_prefix[GF_DUMP_MAX_BUF_LEN];
+        int             ret = -1;
+        eg_t            *conf = NULL;
 
-int
-error_gen_close (xlator_t *this, fd_t *fd)
+        if (!this)
+            goto out;
+
+        conf = this->private;
+        if (!conf)
+            goto out;
+
+        ret = TRY_LOCK(&conf->lock);
+        if (ret != 0) {
+                return ret;
+        }
+
+        gf_proc_dump_add_section("xlator.debug.error-gen.%s.priv", this->name);
+        gf_proc_dump_build_key(key_prefix,"xlator.debug.error-gen","%s.priv",
+                               this->name);
+
+        gf_proc_dump_write("op_count", "%d", conf->op_count);
+        gf_proc_dump_write("failure_iter_no", "%d", conf->failure_iter_no);
+        gf_proc_dump_write("error_no", "%s", conf->error_no);
+        gf_proc_dump_write("random_failure", "%d", conf->random_failure);
+
+        UNLOCK(&conf->lock);
+out:
+        return ret;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
 {
-	return 0;
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_error_gen_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                        " failed");
+                return ret;
+        }
+
+        return ret;
 }
 
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        eg_t            *pvt = NULL;
+        int32_t          ret = 0;
+        char            *error_enable_fops = NULL;
+        int32_t          failure_percent_int = 0;
+
+        if (!this || !this->private)
+                goto out;
+
+        pvt = this->private;
+
+        GF_OPTION_RECONF ("error-no", pvt->error_no, options, str, out);
+
+        GF_OPTION_RECONF ("failure", failure_percent_int, options, int32,
+                          out);
+
+        GF_OPTION_RECONF ("enable", error_enable_fops, options, str, out);
+
+        GF_OPTION_RECONF ("random-failure", pvt->random_failure, options,
+                          bool, out);
+
+        error_gen_parse_fill_fops (pvt, error_enable_fops);
+        error_gen_set_failure (pvt, failure_percent_int);
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "reconfigure returning %d", ret);
+        return ret;
+}
 
 int
 init (xlator_t *this)
 {
         eg_t            *pvt = NULL;
-        data_t          *error_no = NULL;
-        data_t          *failure_percent = NULL;
-        data_t          *enable = NULL;
         int32_t          ret = 0;
         char            *error_enable_fops = NULL;
-        char            *op_no_str = NULL;
-        int              op_no = -1;
-        int              i = 0;
         int32_t          failure_percent_int = 0;
 
         if (!this->children || this->children->next) {
@@ -1872,74 +2104,34 @@ init (xlator_t *this)
                         "dangling volume. check volfile ");
         }
 
-        error_no = dict_get (this->options, "error-no");
-        failure_percent = dict_get (this->options, "failure");
-        enable = dict_get (this->options, "enable");
-
-        pvt = CALLOC (1, sizeof (eg_t));
+        pvt = GF_CALLOC (1, sizeof (eg_t), gf_error_gen_mt_eg_t);
 
         if (!pvt) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "out of memory.");
                 ret = -1;
                 goto out;
         }
 
         LOCK_INIT (&pvt->lock);
 
-        for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                pvt->enable[i] = 0;
-        if (!error_no) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "error-no not specified.");
-        } else {
-                pvt->error_no = data_to_str (error_no);
-        }
+        GF_OPTION_INIT ("error-no", pvt->error_no, str, out);
 
-        if (!failure_percent) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "failure percent not specified.");
-                pvt->failure_iter_no = 100/GF_FAILURE_DEFAULT;
-        } else {
-                failure_percent_int = data_to_int32 (failure_percent);
-                if (failure_percent_int)
-                        pvt->failure_iter_no = 100/failure_percent_int;
-                else
-                        pvt->failure_iter_no = 100/GF_FAILURE_DEFAULT;
-        }
+        GF_OPTION_INIT ("failure", failure_percent_int, int32, out);
+
+        GF_OPTION_INIT ("enable", error_enable_fops, str, out);
+
+        GF_OPTION_INIT ("random-failure", pvt->random_failure, bool, out);
+
+
+        error_gen_parse_fill_fops (pvt, error_enable_fops);
+        error_gen_set_failure (pvt, failure_percent_int);
 
-        if (!enable) {
-                gf_log (this->name, GF_LOG_WARNING,
-                        "All fops are enabled.");
-                for (i = 0; i < GF_FOP_MAXVALUE; i++)
-                        pvt->enable[i] = 1;
-        } else {
-                error_enable_fops = data_to_str (enable);
-                op_no_str = error_enable_fops;
-                while ((*error_enable_fops) != '\0') {
-                        error_enable_fops++;
-                        if (((*error_enable_fops) == ',') ||
-                            ((*error_enable_fops) == '\0')) {
-                                if ((*error_enable_fops) != '\0') {
-                                        (*error_enable_fops) = '\0';
-                                        error_enable_fops++;
-                                }
-                                op_no = get_fop_int (&op_no_str);
-                                if (op_no == -1) {
-                                        gf_log (this->name, GF_LOG_WARNING,
-                                                "Wrong option value %s",
-                                                op_no_str);
-                                } else
-                                        pvt->enable[op_no] = 1;
-                                op_no_str = error_enable_fops;
-                        }
-                }
-        }
         this->private = pvt;
 
         /* Give some seed value here */
         srand (time(NULL));
 out:
+        if (ret)
+                GF_FREE (pvt);
         return ret;
 }
 
@@ -1955,12 +2147,18 @@ fini (xlator_t *this)
 
         if (pvt) {
                 LOCK_DESTROY (&pvt->lock);
-                FREE (pvt);
+                GF_FREE (pvt);
                 gf_log (this->name, GF_LOG_DEBUG, "fini called");
         }
         return;
 }
 
+struct xlator_dumpops dumpops = {
+        .priv = error_gen_priv_dump,
+};
+
+struct xlator_fops cbks;
+
 struct xlator_fops fops = {
 	.lookup      = error_gen_lookup,
 	.stat        = error_gen_stat,
@@ -1983,6 +2181,9 @@ struct xlator_fops fops = {
 	.setxattr    = error_gen_setxattr,
 	.getxattr    = error_gen_getxattr,
 	.removexattr = error_gen_removexattr,
+	.fsetxattr    = error_gen_fsetxattr,
+	.fgetxattr    = error_gen_fgetxattr,
+	.fremovexattr = error_gen_fremovexattr,
 	.opendir     = error_gen_opendir,
 	.readdir     = error_gen_readdir,
 	.readdirp    = error_gen_readdirp,
@@ -2003,22 +2204,29 @@ struct xlator_fops fops = {
 	.getspec     = error_gen_getspec,
 };
 
-struct xlator_cbks cbks = {
-	.release = error_gen_close,
-	.releasedir = error_gen_closedir,
-};
-
 struct volume_options options[] = {
         { .key  = {"failure"},
-          .type = GF_OPTION_TYPE_INT },
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Percentage failure of operations when enabled.",
+        },
+
         { .key  = {"error-no"},
           .value = {"ENOENT","ENOTDIR","ENAMETOOLONG","EACCES","EBADF",
                     "EFAULT","ENOMEM","EINVAL","EIO","EEXIST","ENOSPC",
                     "EPERM","EROFS","EBUSY","EISDIR","ENOTEMPTY","EMLINK"
                     "ENODEV","EXDEV","EMFILE","ENFILE","ENOSYS","EINTR",
-                    "EFBIG","EAGAIN"},
-          .type = GF_OPTION_TYPE_STR },
+                    "EFBIG","EAGAIN","GF_ERROR_SHORT_WRITE"},
+          .type = GF_OPTION_TYPE_STR,
+        },
+
+        { .key = {"random-failure"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+        },
+
         { .key  = {"enable"},
-          .type = GF_OPTION_TYPE_STR },
+          .type = GF_OPTION_TYPE_STR,
+        },
+
         { .key  = {NULL} }
 };
diff --git a/xlators/debug/error-gen/src/error-gen.h b/xlators/debug/error-gen/src/error-gen.h
index a27c2de50..d92c23062 100644
--- a/xlators/debug/error-gen/src/error-gen.h
+++ b/xlators/debug/error-gen/src/error-gen.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _ERROR_GEN_H
 #define _ERROR_GEN_H
 
@@ -25,13 +15,28 @@
 #include "config.h"
 #endif
 
+#include "error-gen-mem-types.h"
+
 #define GF_FAILURE_DEFAULT 10
 
+/*
+ * Pseudo-errors refer to errors beyond the scope of traditional <-1, op_errno>
+ * returns. This facilitates the ability to return unexpected, but not -1 values
+ * and/or to inject operations that lead to implicit error conditions. The range
+ * for pseudo errors resides at a high value to avoid conflicts with the errno
+ * range.
+ */
+enum GF_PSEUDO_ERRORS {
+	GF_ERROR_SHORT_WRITE = 1000,	/* short writev return value */
+	GF_ERROR_MAX
+};
+
 typedef struct {
         int enable[GF_FOP_MAXVALUE];
         int op_count;
         int failure_iter_no;
         char *error_no;
+        gf_boolean_t random_failure;
         gf_lock_t lock;
 } eg_t;
 
diff --git a/xlators/debug/io-stats/src/Makefile.am b/xlators/debug/io-stats/src/Makefile.am
index b894e79c3..dff294cd8 100644
--- a/xlators/debug/io-stats/src/Makefile.am
+++ b/xlators/debug/io-stats/src/Makefile.am
@@ -2,14 +2,17 @@
 xlator_LTLIBRARIES = io-stats.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
 
-io_stats_la_LDFLAGS = -module -avoidversion
+io_stats_la_LDFLAGS = -module -avoid-version
 
 io_stats_la_SOURCES = io-stats.c
 io_stats_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
 noinst_HEADERS = io-stats-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/rpc/xdr/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES =
diff --git a/xlators/debug/io-stats/src/io-stats-mem-types.h b/xlators/debug/io-stats/src/io-stats-mem-types.h
index e53d2d22a..c30dfb17e 100644
--- a/xlators/debug/io-stats/src/io-stats-mem-types.h
+++ b/xlators/debug/io-stats/src/io-stats-mem-types.h
@@ -1,24 +1,13 @@
-
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __IO_STATS_MEM_TYPES_H__
 #define __IO_STATS_MEM_TYPES_H__
 
@@ -27,6 +16,8 @@
 enum gf_io_stats_mem_types_ {
         gf_io_stats_mt_ios_conf = gf_common_mt_end + 1,
         gf_io_stats_mt_ios_fd,
+        gf_io_stats_mt_ios_stat,
+        gf_io_stats_mt_ios_stat_list,
         gf_io_stats_mt_end
 };
 #endif
diff --git a/xlators/debug/io-stats/src/io-stats.c b/xlators/debug/io-stats/src/io-stats.c
index c72545bd3..5b4c833fb 100644
--- a/xlators/debug/io-stats/src/io-stats.c
+++ b/xlators/debug/io-stats/src/io-stats.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2009 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -43,6 +33,57 @@
 #include "glusterfs.h"
 #include "xlator.h"
 #include "io-stats-mem-types.h"
+#include <stdarg.h>
+#include "defaults.h"
+#include "logging.h"
+#include "cli1-xdr.h"
+
+#define MAX_LIST_MEMBERS 100
+
+typedef enum {
+        IOS_STATS_TYPE_NONE,
+        IOS_STATS_TYPE_OPEN,
+        IOS_STATS_TYPE_READ,
+        IOS_STATS_TYPE_WRITE,
+        IOS_STATS_TYPE_OPENDIR,
+        IOS_STATS_TYPE_READDIRP,
+        IOS_STATS_TYPE_READ_THROUGHPUT,
+        IOS_STATS_TYPE_WRITE_THROUGHPUT,
+        IOS_STATS_TYPE_MAX
+}ios_stats_type_t;
+
+typedef enum {
+        IOS_STATS_THRU_READ,
+        IOS_STATS_THRU_WRITE,
+        IOS_STATS_THRU_MAX,
+}ios_stats_thru_t;
+
+struct ios_stat_lat {
+        struct timeval  time;
+        double          throughput;
+};
+
+struct ios_stat {
+        gf_lock_t       lock;
+        uuid_t          gfid;
+        char           *filename;
+        uint64_t        counters [IOS_STATS_TYPE_MAX];
+        struct ios_stat_lat thru_counters [IOS_STATS_THRU_MAX];
+        int             refcnt;
+};
+
+struct ios_stat_list {
+        struct list_head  list;
+        struct ios_stat  *iosstat;
+        double            value;
+};
+
+struct ios_stat_head {
+       gf_lock_t                lock;
+       double                   min_cnt;
+       uint64_t                 members;
+       struct ios_stat_list    *iosstats;
+};
 
 struct ios_lat {
         double  min;
@@ -58,6 +99,9 @@ struct ios_global_stats {
         uint64_t        fop_hits[GF_FOP_MAXVALUE];
         struct timeval  started_at;
         struct ios_lat  latency[GF_FOP_MAXVALUE];
+        uint64_t        nr_opens;
+        uint64_t        max_nr_opens;
+        struct timeval  max_openfd_time;
 };
 
 
@@ -67,7 +111,10 @@ struct ios_conf {
         uint64_t                  increment;
         struct ios_global_stats   incremental;
         gf_boolean_t              dump_fd_stats;
-        int                       measure_latency;
+        gf_boolean_t              count_fop_hits;
+        gf_boolean_t              measure_latency;
+        struct ios_stat_head      list[IOS_STATS_TYPE_MAX];
+        struct ios_stat_head      thru_list[IOS_STATS_THRU_MAX];
 };
 
 
@@ -80,31 +127,60 @@ struct ios_fd {
         struct timeval  opened_at;
 };
 
+typedef enum {
+        IOS_DUMP_TYPE_NONE = 0,
+        IOS_DUMP_TYPE_FILE = 1,
+        IOS_DUMP_TYPE_DICT = 2,
+        IOS_DUMP_TYPE_MAX  = 3
+} ios_dump_type_t;
+
+struct ios_dump_args {
+        ios_dump_type_t type;
+        union {
+                FILE *logfp;
+                dict_t *dict;
+        } u;
+};
+
+typedef int (*block_dump_func) (xlator_t *, struct ios_dump_args*,
+                                    int , int , uint64_t ) ;
 
 struct ios_local {
         struct timeval  wind_at;
         struct timeval  unwind_at;
 };
 
+struct volume_options options[];
+
+inline static int
+is_fop_latency_started (call_frame_t *frame)
+{
+        GF_ASSERT (frame);
+        struct timeval epoch = {0,};
+        return memcmp (&frame->begin, &epoch, sizeof (epoch));
+}
+
 #define END_FOP_LATENCY(frame, op)                                      \
         do {                                                            \
                 struct ios_conf  *conf = NULL;                          \
                                                                         \
                 conf = this->private;                                   \
-                if (conf->measure_latency) {                            \
-                        gettimeofday (&frame->end, NULL);        \
+                if (conf && conf->measure_latency) {                    \
+                        gettimeofday (&frame->end, NULL);               \
                         update_ios_latency (conf, frame, GF_FOP_##op);  \
                 }                                                       \
         } while (0)
 
-#define START_FOP_LATENCY(frame)                                        \
-        do {                                                            \
-                struct ios_conf  *conf = NULL;                          \
-                                                                        \
-                conf = this->private;                                   \
-                if (conf->measure_latency) {                            \
-                        gettimeofday (&frame->begin, NULL);      \
-                }                                                       \
+#define START_FOP_LATENCY(frame)                                         \
+        do {                                                             \
+                struct ios_conf  *conf = NULL;                           \
+                                                                         \
+                conf = this->private;                                    \
+                if (conf && conf->measure_latency) {                     \
+                        gettimeofday (&frame->begin, NULL);              \
+                } else {                                                 \
+                        memset (&frame->begin, 0, sizeof (frame->begin));\
+                }                                                        \
         } while (0)
 
 
@@ -113,14 +189,30 @@ struct ios_local {
                 struct ios_conf  *conf = NULL;                          \
                                                                         \
                 conf = this->private;                                   \
-                LOCK (&conf->lock);                                     \
-                {                                                       \
-                        conf->cumulative.fop_hits[GF_FOP_##op]++;       \
-                        conf->incremental.fop_hits[GF_FOP_##op]++;      \
-                }                                                       \
-                UNLOCK (&conf->lock);                                   \
+                if (!conf)                                              \
+                        break;                                          \
+                conf->cumulative.fop_hits[GF_FOP_##op]++;               \
+                conf->incremental.fop_hits[GF_FOP_##op]++;              \
         } while (0)
 
+#define UPDATE_PROFILE_STATS(frame, op)                                       \
+        do {                                                                  \
+                struct ios_conf  *conf = NULL;                                \
+                                                                              \
+                if (!is_fop_latency_started (frame))                          \
+                        break;                                                \
+                conf = this->private;                                         \
+                LOCK (&conf->lock);                                           \
+                {                                                             \
+                        if (conf && conf->measure_latency &&                  \
+                            conf->count_fop_hits) {                           \
+                                BUMP_FOP(op);                                 \
+                                gettimeofday (&frame->end, NULL);             \
+                                update_ios_latency (conf, frame, GF_FOP_##op);\
+                        }                                                     \
+                }                                                             \
+                UNLOCK (&conf->lock);                                         \
+        } while (0)
 
 #define BUMP_READ(fd, len)                                              \
         do {                                                            \
@@ -131,6 +223,8 @@ struct ios_local {
                 conf = this->private;                                   \
                 lb2 = log_base2 (len);                                  \
                 ios_fd_ctx_get (fd, this, &iosfd);                      \
+                if (!conf)                                              \
+                        break;                                          \
                                                                         \
                 LOCK (&conf->lock);                                     \
                 {                                                       \
@@ -157,6 +251,8 @@ struct ios_local {
                 conf = this->private;                                   \
                 lb2 = log_base2 (len);                                  \
                 ios_fd_ctx_get (fd, this, &iosfd);                      \
+                if (!conf)                                              \
+                        break;                                          \
                                                                         \
                 LOCK (&conf->lock);                                     \
                 {                                                       \
@@ -174,6 +270,58 @@ struct ios_local {
         } while (0)
 
 
+#define BUMP_STATS(iosstat, type)                                               \
+        do {                                                                    \
+                struct ios_conf         *conf = NULL;                           \
+                uint64_t                 value = 0;                             \
+                                                                                \
+                conf = this->private;                                           \
+                                                                                \
+                LOCK(&iosstat->lock);                                           \
+                {                                                               \
+                        iosstat->counters[type]++;                              \
+                        value = iosstat->counters[type];                        \
+                }                                                               \
+                UNLOCK (&iosstat->lock);                                        \
+                ios_stat_add_to_list (&conf->list[type],                        \
+                                     value, iosstat);                           \
+                                                                                \
+        } while (0)
+
+
+#define BUMP_THROUGHPUT(iosstat, type)						\
+        do {									\
+                struct ios_conf         *conf = NULL;				\
+                double                   elapsed;				\
+                struct timeval          *begin, *end;				\
+                double                   throughput;				\
+               int                      flag = 0;                              \
+                                                                                \
+                begin = &frame->begin;						\
+                end   = &frame->end;						\
+                                                                                \
+                elapsed = (end->tv_sec - begin->tv_sec) * 1e6			\
+                        + (end->tv_usec - begin->tv_usec);			\
+                throughput = op_ret / elapsed;					\
+                                                                                \
+                conf = this->private;						\
+                LOCK(&iosstat->lock);						\
+                {								\
+                        if (iosstat->thru_counters[type].throughput             \
+                                <= throughput) {                                \
+                                iosstat->thru_counters[type].throughput =       \
+                                                                throughput;     \
+                                gettimeofday (&iosstat->                        \
+                                             thru_counters[type].time, NULL);   \
+                               flag = 1;                                       \
+                        }							\
+                }								\
+                UNLOCK (&iosstat->lock);					\
+               if (flag)                                                       \
+                       ios_stat_add_to_list (&conf->thru_list[type],           \
+                                               throughput, iosstat);           \
+        } while (0)
+
 int
 ios_fd_ctx_get (fd_t *fd, xlator_t *this, struct ios_fd **iosfd)
 {
@@ -203,6 +351,183 @@ ios_fd_ctx_set (fd_t *fd, xlator_t *this, struct ios_fd *iosfd)
         return ret;
 }
 
+int
+ios_stat_ref (struct ios_stat *iosstat)
+{
+        LOCK (&iosstat->lock);
+        {
+                iosstat->refcnt++;
+        }
+        UNLOCK (&iosstat->lock);
+
+        return iosstat->refcnt;
+}
+
+int
+ios_stat_unref (struct ios_stat *iosstat)
+{
+        int cleanup = 0;
+        LOCK (&iosstat->lock);
+        {
+                iosstat->refcnt--;
+                if (iosstat->refcnt == 0) {
+                        if (iosstat->filename) {
+                                GF_FREE (iosstat->filename);
+                                iosstat->filename = NULL;
+                        }
+                        cleanup = 1;
+                }
+        }
+        UNLOCK (&iosstat->lock);
+
+        if (cleanup) {
+                GF_FREE (iosstat);
+                iosstat = NULL;
+        }
+
+        return 0;
+}
+
+int
+ios_inode_ctx_set (inode_t *inode, xlator_t *this, struct ios_stat *iosstat)
+{
+        uint64_t   iosstat64 = 0;
+        int        ret     = 0;
+
+        ios_stat_ref (iosstat);
+        iosstat64 = (unsigned long )iosstat;
+        ret = inode_ctx_put (inode, this, iosstat64);
+        return ret;
+}
+
+int
+ios_inode_ctx_get (inode_t *inode, xlator_t *this, struct ios_stat **iosstat)
+{
+        uint64_t      iosstat64 = 0;
+        unsigned long iosstatlong = 0;
+        int           ret = 0;
+
+        ret = inode_ctx_get (inode, this, &iosstat64);
+        iosstatlong = iosstat64;
+        if (ret != -1)
+                *iosstat = (void *) iosstatlong;
+
+        return ret;
+
+}
+
+int
+ios_stat_add_to_list (struct ios_stat_head *list_head, uint64_t value,
+                            struct ios_stat *iosstat)
+{
+        struct ios_stat_list *new = NULL;
+        struct ios_stat_list *entry = NULL;
+        struct ios_stat_list *t = NULL;
+        struct ios_stat_list *list_entry = NULL;
+        struct ios_stat_list *tmp = NULL;
+        struct ios_stat_list *last = NULL;
+        struct ios_stat      *stat = NULL;
+        int                   cnt = 0;
+        int                   found = 0;
+        int                   reposition = 0;
+        double                min_count = 0;
+
+        LOCK (&list_head->lock);
+        {
+
+                if (list_head->min_cnt == 0)
+                        list_head->min_cnt = value;
+                if ((list_head->members == MAX_LIST_MEMBERS) &&
+                     (list_head->min_cnt > value))
+                        goto out;
+
+                list_for_each_entry_safe (entry, t,
+                              &list_head->iosstats->list, list) {
+                        cnt++;
+                        if (cnt == list_head->members)
+                                last = entry;
+
+                        if (!uuid_compare (iosstat->gfid,
+                            entry->iosstat->gfid)) {
+                                list_entry = entry;
+                                found = cnt;
+                                entry->value = value;
+                                if (!reposition) {
+                                        if (cnt == list_head->members)
+                                                list_head->min_cnt = value;
+                                        goto out;
+                                }
+                                break;
+                        } else if (entry->value <= value && !reposition) {
+                                reposition = cnt;
+                                tmp = entry;
+                                if (cnt == list_head->members - 1)
+                                        min_count = entry->value;
+                        }
+                }
+                if (found) {
+                        list_del (&list_entry->list);
+                        list_add_tail (&list_entry->list, &tmp->list);
+                        if (min_count)
+                                list_head->min_cnt = min_count;
+                        goto out;
+                } else if (list_head->members == MAX_LIST_MEMBERS && reposition) {
+                        new = GF_CALLOC (1, sizeof (*new),
+                                        gf_io_stats_mt_ios_stat_list);
+                        new->iosstat = iosstat;
+                        new->value = value;
+                        ios_stat_ref (iosstat);
+                        list_add_tail (&new->list, &tmp->list);
+                        stat = last->iosstat;
+                        last->iosstat = NULL;
+                        ios_stat_unref (stat);
+                        list_del (&last->list);
+                        GF_FREE (last);
+                        if (reposition == MAX_LIST_MEMBERS)
+                                list_head->min_cnt = value;
+                        else if (min_count) {
+                                list_head->min_cnt = min_count;
+                        }
+                } else if (list_head->members < MAX_LIST_MEMBERS) {
+                        new  = GF_CALLOC (1, sizeof (*new),
+                                          gf_io_stats_mt_ios_stat_list);
+                        new->iosstat = iosstat;
+                        new->value = value;
+                        ios_stat_ref (iosstat);
+                        if (reposition) {
+                                list_add_tail (&new->list, &tmp->list);
+                        } else {
+                                list_add_tail (&new->list, &entry->list);
+                        }
+                        list_head->members++;
+                        if (list_head->min_cnt > value)
+                                list_head->min_cnt = value;
+                }
+        }
+out:
+        UNLOCK (&list_head->lock);
+        return 0;
+}
+
+static inline int
+ios_stats_cleanup (xlator_t *this, inode_t *inode)
+{
+
+        struct ios_stat *iosstat = NULL;
+        uint64_t         iosstat64 = 0;
+
+        inode_ctx_del (inode, this, &iosstat64);
+        if (!iosstat64) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "could not get inode ctx");
+                return 0;
+        }
+        iosstat = (void *) (long)iosstat64;
+        if (iosstat) {
+                ios_stat_unref (iosstat);
+        }
+        return 0;
+}
 
 #define ios_log(this, logfp, fmt ...)                           \
         do {                                                    \
@@ -210,88 +535,445 @@ ios_fd_ctx_set (fd_t *fd, xlator_t *this, struct ios_fd *iosfd)
                         fprintf (logfp, fmt);                   \
                         fprintf (logfp, "\n");                  \
                 }                                               \
-                gf_log (this->name, GF_LOG_NORMAL, fmt);        \
+                gf_log (this->name, GF_LOG_INFO, fmt);        \
         } while (0)
 
+int
+ios_dump_file_stats (struct ios_stat_head *list_head, xlator_t *this, FILE* logfp)
+{
+        struct ios_stat_list *entry = NULL;
+
+        LOCK (&list_head->lock);
+        {
+                list_for_each_entry (entry, &list_head->iosstats->list, list) {
+                        ios_log (this, logfp, "%-12.0f %s",
+                                entry->value, entry->iosstat->filename);
+                }
+        }
+        UNLOCK (&list_head->lock);
+        return 0;
+}
+
+int
+ios_dump_throughput_stats (struct ios_stat_head *list_head, xlator_t *this,
+                            FILE* logfp, ios_stats_type_t type)
+{
+        struct ios_stat_list *entry = NULL;
+        struct timeval        time  = {0, };
+        char                  timestr[256] = {0, };
+
+        LOCK (&list_head->lock);
+        {
+                list_for_each_entry (entry, &list_head->iosstats->list, list) {
+                        gf_time_fmt (timestr, sizeof timestr,
+                                     entry->iosstat->thru_counters[type].time.tv_sec,
+                                     gf_timefmt_FT);
+                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
+                          ".%"GF_PRI_SUSECONDS, time.tv_usec);
+
+                        ios_log (this, logfp, "%s \t %-10.2f  \t  %s",
+                                 timestr, entry->value, entry->iosstat->filename);
+                }
+        }
+        UNLOCK (&list_head->lock);
+        return 0;
+}
 
 int
-io_stats_dump_global (xlator_t *this, struct ios_global_stats *stats,
-                      struct timeval *now, int interval, FILE *logfp)
+io_stats_dump_global_to_logfp (xlator_t *this, struct ios_global_stats *stats,
+                               struct timeval *now, int interval, FILE* logfp)
 {
-        int    i = 0;
+        int                   i = 0;
+        int                   per_line = 0;
+        int                   index = 0;
+        struct ios_stat_head *list_head = NULL;
+        struct ios_conf      *conf = NULL;
+        char                  timestr[256] = {0, };
+        char                  str_header[128] = {0};
+        char                  str_read[128] = {0};
+        char                  str_write[128] = {0};
+
+        conf = this->private;
 
         if (interval == -1)
-                ios_log (this, logfp, "=== Cumulative stats ===");
+                ios_log (this, logfp, "\n=== Cumulative stats ===");
         else
-                ios_log (this, logfp, "=== Interval %d stats ===",
+                ios_log (this, logfp, "\n=== Interval %d stats ===",
                          interval);
-        ios_log (this, logfp, "      Duration : %"PRId64"secs",
+        ios_log (this, logfp, "      Duration : %"PRId64" secs",
                  (uint64_t) (now->tv_sec - stats->started_at.tv_sec));
         ios_log (this, logfp, "     BytesRead : %"PRId64,
                  stats->data_read);
-        ios_log (this, logfp, "  BytesWritten : %"PRId64,
+        ios_log (this, logfp, "  BytesWritten : %"PRId64"\n",
                  stats->data_written);
 
+        snprintf (str_header, sizeof (str_header), "%-12s %c", "Block Size", ':');
+        snprintf (str_read, sizeof (str_read), "%-12s %c", "Read Count", ':');
+        snprintf (str_write, sizeof (str_write), "%-12s %c", "Write Count", ':');
+        index = 14;
         for (i = 0; i < 32; i++) {
+                if ((stats->block_count_read[i] == 0) &&
+                    (stats->block_count_write[i] == 0))
+                        continue;
+                per_line++;
+
+                snprintf (str_header+index, sizeof (str_header)-index,
+                          "%16dB+", (1<<i));
                 if (stats->block_count_read[i])
-                        ios_log (this, logfp, " Read %06db+ : %"PRId64,
-                                 (1 << i), stats->block_count_read[i]);
+                        snprintf (str_read+index, sizeof (str_read)-index,
+                                  "%18"PRId64, stats->block_count_read[i]);
+                else    snprintf (str_read+index, sizeof (str_read)-index,
+                                  "%18s", "0");
+                if (stats->block_count_write[i])
+                        snprintf (str_write+index, sizeof (str_write)-index,
+                                  "%18"PRId64, stats->block_count_write[i]);
+                else    snprintf (str_write+index, sizeof (str_write)-index,
+                                  "%18s", "0");
+
+                index += 18;
+                if (per_line == 3) {
+                        ios_log (this, logfp, "%s", str_header);
+                        ios_log (this, logfp, "%s", str_read);
+                        ios_log (this, logfp, "%s\n", str_write);
+
+                        memset (str_header, 0, sizeof (str_header));
+                        memset (str_read, 0, sizeof (str_read));
+                        memset (str_write, 0, sizeof (str_write));
+
+                        snprintf (str_header, sizeof (str_header), "%-12s %c",
+                                  "Block Size", ':');
+                        snprintf (str_read, sizeof (str_read), "%-12s %c",
+                                  "Read Count", ':');
+                        snprintf (str_write, sizeof (str_write), "%-12s %c",
+                                  "Write Count", ':');
+
+                        index = 14;
+                        per_line = 0;
+                }
         }
 
-        for (i = 0; i < 32; i++) {
-                if (stats->block_count_write[i])
-                        ios_log (this, logfp, "Write %06db+ : %"PRId64,
-                                 (1 << i), stats->block_count_write[i]);
+        if (per_line != 0) {
+                ios_log (this, logfp, "%s", str_header);
+                ios_log (this, logfp, "%s", str_read);
+                ios_log (this, logfp, "%s\n", str_write);
         }
 
+        ios_log (this, logfp, "%-13s %10s %14s %14s %14s", "Fop",
+                 "Call Count", "Avg-Latency", "Min-Latency",
+                 "Max-Latency");
+        ios_log (this, logfp, "%-13s %10s %14s %14s %14s", "---", "----------",
+                 "-----------", "-----------", "-----------");
+
         for (i = 0; i < GF_FOP_MAXVALUE; i++) {
                 if (stats->fop_hits[i] && !stats->latency[i].avg)
-                        ios_log (this, logfp, "%14s : %"PRId64,
-                                 gf_fop_list[i], stats->fop_hits[i]);
+                        ios_log (this, logfp, "%-13s %10"PRId64" %11s "
+                                 "us %11s us %11s us", gf_fop_list[i],
+                                 stats->fop_hits[i], "0", "0", "0");
                 else if (stats->fop_hits[i] && stats->latency[i].avg)
-                        ios_log (this, logfp, "%14s : %"PRId64 ", latency"
-                                 "(avg: %f, min: %f, max: %f)",
-                                 gf_fop_list[i], stats->fop_hits[i],
-                                 stats->latency[i].avg, stats->latency[i].min,
-                                 stats->latency[i].max);
+                        ios_log (this, logfp, "%-13s %10"PRId64" %11.2lf us "
+                                 "%11.2lf us %11.2lf us", gf_fop_list[i],
+                                 stats->fop_hits[i], stats->latency[i].avg,
+                                 stats->latency[i].min, stats->latency[i].max);
+        }
+        ios_log (this, logfp, "------ ----- ----- ----- ----- ----- ----- ----- "
+                 " ----- ----- ----- -----\n");
+
+        if (interval == -1) {
+                LOCK (&conf->lock);
+                {
+                        gf_time_fmt (timestr, sizeof timestr,
+                                     conf->cumulative.max_openfd_time.tv_sec,
+                                     gf_timefmt_FT);
+                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
+                                  ".%"GF_PRI_SUSECONDS,
+                                  conf->cumulative.max_openfd_time.tv_usec);
+                        ios_log (this, logfp, "Current open fd's: %"PRId64
+                                 " Max open fd's: %"PRId64" time %s",
+                                 conf->cumulative.nr_opens,
+                                 conf->cumulative.max_nr_opens, timestr);
+                }
+                UNLOCK (&conf->lock);
+                ios_log (this, logfp, "\n==========Open File Stats========");
+                ios_log (this, logfp, "\nCOUNT:  \t  FILE NAME");
+                list_head = &conf->list[IOS_STATS_TYPE_OPEN];
+                ios_dump_file_stats (list_head, this, logfp);
+
+
+                ios_log (this, logfp, "\n==========Read File Stats========");
+                ios_log (this, logfp, "\nCOUNT:  \t  FILE NAME");
+                list_head = &conf->list[IOS_STATS_TYPE_READ];
+                ios_dump_file_stats (list_head, this, logfp);
+
+                ios_log (this, logfp, "\n==========Write File Stats========");
+                ios_log (this, logfp, "\nCOUNT:  \t  FILE NAME");
+                list_head = &conf->list[IOS_STATS_TYPE_WRITE];
+                ios_dump_file_stats (list_head, this, logfp);
+
+                ios_log (this, logfp, "\n==========Directory open stats========");
+                ios_log (this, logfp, "\nCOUNT:  \t  DIRECTORY NAME");
+                list_head = &conf->list[IOS_STATS_TYPE_OPENDIR];
+                ios_dump_file_stats (list_head, this, logfp);
+
+                ios_log (this, logfp, "\n========Directory readdirp Stats=======");
+                ios_log (this, logfp, "\nCOUNT:  \t  DIRECTORY NAME");
+                list_head = &conf->list[IOS_STATS_TYPE_READDIRP];
+                ios_dump_file_stats (list_head, this, logfp);
+
+                ios_log (this, logfp, "\n========Read Throughput File Stats=====");
+                ios_log (this, logfp, "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
+                         "\tFILE NAME");
+                list_head = &conf->thru_list[IOS_STATS_THRU_READ];
+                ios_dump_throughput_stats(list_head, this, logfp, IOS_STATS_THRU_READ);
+
+                ios_log (this, logfp, "\n======Write Throughput File Stats======");
+                ios_log (this, logfp, "\nTIMESTAMP \t\t\t THROUGHPUT(KBPS)"
+                         "\tFILE NAME");
+                list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
+                ios_dump_throughput_stats (list_head, this, logfp, IOS_STATS_THRU_WRITE);
         }
-
         return 0;
 }
 
+int
+io_stats_dump_global_to_dict (xlator_t *this, struct ios_global_stats *stats,
+                              struct timeval *now, int interval, dict_t *dict)
+{
+        int             ret = 0;
+        char            key[256] = {0};
+        uint64_t        sec = 0;
+        int             i = 0;
+        uint64_t        count = 0;
+
+        GF_ASSERT (stats);
+        GF_ASSERT (now);
+        GF_ASSERT (dict);
+        GF_ASSERT (this);
+
+        if (interval == -1)
+                snprintf (key, sizeof (key), "cumulative");
+        else
+                snprintf (key, sizeof (key), "interval");
+        ret = dict_set_int32 (dict, key, interval);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "failed to set "
+                        "interval %d", interval);
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%d-duration", interval);
+        sec = (uint64_t) (now->tv_sec - stats->started_at.tv_sec);
+        ret = dict_set_uint64 (dict, key, sec);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to set "
+                        "duration(%d) - %"PRId64, interval, sec);
+                goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%d-total-read", interval);
+        ret = dict_set_uint64 (dict, key, stats->data_read);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to set total "
+                       "read(%d) - %"PRId64, interval, stats->data_read);
+                goto out;
+        }
+
+        memset (key, 0, sizeof (key));
+        snprintf (key, sizeof (key), "%d-total-write", interval);
+        ret = dict_set_uint64 (dict, key, stats->data_written);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to set total "
+                        "write(%d) - %"PRId64, interval, stats->data_written);
+                goto out;
+        }
+        for (i = 0; i < 32; i++) {
+                if (stats->block_count_read[i]) {
+                        memset (key, 0, sizeof (key));
+                        snprintf (key, sizeof (key), "%d-read-%d", interval,
+                                  (1 << i));
+                        count = stats->block_count_read[i];
+                        ret = dict_set_uint64 (dict, key, count);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                        "set read-%db+, with: %"PRId64,
+                                        (1<<i), count);
+                                goto out;
+                        }
+                }
+        }
+
+        for (i = 0; i < 32; i++) {
+                if (stats->block_count_write[i]) {
+                        snprintf (key, sizeof (key), "%d-write-%d", interval,
+                                  (1<<i));
+                        count = stats->block_count_write[i];
+                        ret = dict_set_uint64 (dict, key, count);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                        "set write-%db+, with: %"PRId64,
+                                        (1<<i), count);
+                                goto out;
+                        }
+                }
+        }
+
+        for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+                if (stats->fop_hits[i] == 0)
+                        continue;
+                snprintf (key, sizeof (key), "%d-%d-hits", interval, i);
+                ret = dict_set_uint64 (dict, key, stats->fop_hits[i]);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to "
+                                "set %s-fop-hits: %"PRIu64, gf_fop_list[i],
+                                stats->fop_hits[i]);
+                        goto out;
+                }
+
+                if (stats->latency[i].avg == 0)
+                        continue;
+                snprintf (key, sizeof (key), "%d-%d-avglatency", interval, i);
+                ret = dict_set_double (dict, key, stats->latency[i].avg);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to set %s "
+                                "avglatency(%d) with %f", gf_fop_list[i],
+                                interval, stats->latency[i].avg);
+                        goto out;
+                }
+                snprintf (key, sizeof (key), "%d-%d-minlatency", interval, i);
+                ret = dict_set_double (dict, key, stats->latency[i].min);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to set %s "
+                                "minlatency(%d) with %f", gf_fop_list[i],
+                                interval, stats->latency[i].min);
+                        goto out;
+                }
+                snprintf (key, sizeof (key), "%d-%d-maxlatency", interval, i);
+                ret = dict_set_double (dict, key, stats->latency[i].max);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to set %s "
+                                "maxlatency(%d) with %f", gf_fop_list[i],
+                                interval, stats->latency[i].max);
+                        goto out;
+                }
+        }
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "returning %d", ret);
+        return ret;
+}
+
+int
+io_stats_dump_global (xlator_t *this, struct ios_global_stats *stats,
+                      struct timeval *now, int interval,
+                      struct ios_dump_args *args)
+{
+        int     ret = -1;
+
+        GF_ASSERT (args);
+        GF_ASSERT (now);
+        GF_ASSERT (stats);
+        GF_ASSERT (this);
+
+
+
+        switch (args->type) {
+        case IOS_DUMP_TYPE_FILE:
+                ret = io_stats_dump_global_to_logfp (this, stats, now,
+                                                     interval, args->u.logfp);
+        break;
+        case IOS_DUMP_TYPE_DICT:
+                ret = io_stats_dump_global_to_dict (this, stats, now,
+                                                    interval, args->u.dict);
+        break;
+        default:
+                GF_ASSERT (0);
+                ret = -1;
+        break;
+        }
+        return ret;
+}
+
+int
+ios_dump_args_init (struct ios_dump_args *args, ios_dump_type_t type,
+                    void *output)
+{
+        int             ret = 0;
+
+        GF_ASSERT (args);
+        GF_ASSERT (type > IOS_DUMP_TYPE_NONE && type < IOS_DUMP_TYPE_MAX);
+        GF_ASSERT (output);
+
+        args->type = type;
+        switch (args->type) {
+        case IOS_DUMP_TYPE_FILE:
+                args->u.logfp = output;
+                break;
+        case IOS_DUMP_TYPE_DICT:
+                args->u.dict = output;
+                break;
+        default:
+                GF_ASSERT (0);
+                ret = -1;
+        }
+
+        return ret;
+}
+
+static void
+ios_global_stats_clear (struct ios_global_stats *stats, struct timeval *now)
+{
+        GF_ASSERT (stats);
+        GF_ASSERT (now);
+
+        memset (stats, 0, sizeof (*stats));
+        stats->started_at = *now;
+}
 
 int
-io_stats_dump (xlator_t *this, char *filename, inode_t *inode,
-               const char *path)
+io_stats_dump (xlator_t *this, struct ios_dump_args *args,
+               gf1_cli_info_op op, gf_boolean_t is_peek)
 {
         struct ios_conf         *conf = NULL;
         struct ios_global_stats  cumulative = {0, };
         struct ios_global_stats  incremental = {0, };
         int                      increment = 0;
         struct timeval           now;
-        FILE                    *logfp = NULL;
+
+        GF_ASSERT (this);
+        GF_ASSERT (args);
+        GF_ASSERT (args->type > IOS_DUMP_TYPE_NONE);
+        GF_ASSERT (args->type < IOS_DUMP_TYPE_MAX);
 
         conf = this->private;
 
         gettimeofday (&now, NULL);
         LOCK (&conf->lock);
         {
-                cumulative  = conf->cumulative;
-                incremental = conf->incremental;
+                if (op == GF_CLI_INFO_ALL ||
+                    op == GF_CLI_INFO_CUMULATIVE)
+                        cumulative  = conf->cumulative;
 
-                increment = conf->increment++;
+                if (op == GF_CLI_INFO_ALL ||
+                    op == GF_CLI_INFO_INCREMENTAL) {
+                        incremental = conf->incremental;
+                        increment = conf->increment;
 
-                memset (&conf->incremental, 0, sizeof (conf->incremental));
-                conf->incremental.started_at = now;
+                        if (!is_peek) {
+                                increment = conf->increment++;
+
+                                ios_global_stats_clear (&conf->incremental,
+                                                        &now);
+                        }
+                }
         }
         UNLOCK (&conf->lock);
 
-        logfp = fopen (filename, "w+");
-        io_stats_dump_global (this, &cumulative, &now, -1, logfp);
-        io_stats_dump_global (this, &incremental, &now, increment, logfp);
+        if (op == GF_CLI_INFO_ALL ||
+            op == GF_CLI_INFO_CUMULATIVE)
+                io_stats_dump_global (this, &cumulative, &now, -1, args);
+
+        if (op == GF_CLI_INFO_ALL ||
+            op == GF_CLI_INFO_INCREMENTAL)
+                io_stats_dump_global (this, &incremental, &now, increment, args);
 
-        if (logfp)
-                fclose (logfp);
         return 0;
 }
 
@@ -323,51 +1005,70 @@ io_stats_dump_fd (xlator_t *this, struct ios_fd *iosfd)
         sec = now.tv_sec - iosfd->opened_at.tv_sec;
         usec = now.tv_usec - iosfd->opened_at.tv_usec;
 
-        gf_log (this->name, GF_LOG_NORMAL,
+        gf_log (this->name, GF_LOG_INFO,
                 "--- fd stats ---");
 
         if (iosfd->filename)
-                gf_log (this->name, GF_LOG_NORMAL,
+                gf_log (this->name, GF_LOG_INFO,
                         "      Filename : %s",
                         iosfd->filename);
 
         if (sec)
-                gf_log (this->name, GF_LOG_NORMAL,
+                gf_log (this->name, GF_LOG_INFO,
                         "      Lifetime : %"PRId64"secs, %"PRId64"usecs",
                         sec, usec);
 
         if (iosfd->data_read)
-                gf_log (this->name, GF_LOG_NORMAL,
+                gf_log (this->name, GF_LOG_INFO,
                         "     BytesRead : %"PRId64" bytes",
                         iosfd->data_read);
 
         if (iosfd->data_written)
-                gf_log (this->name, GF_LOG_NORMAL,
+                gf_log (this->name, GF_LOG_INFO,
                         "  BytesWritten : %"PRId64" bytes",
                         iosfd->data_written);
 
         for (i = 0; i < 32; i++) {
                 if (iosfd->block_count_read[i])
-                        gf_log (this->name, GF_LOG_NORMAL,
+                        gf_log (this->name, GF_LOG_INFO,
                                 " Read %06db+ : %"PRId64,
                                 (1 << i), iosfd->block_count_read[i]);
         }
         for (i = 0; i < 32; i++) {
                 if (iosfd->block_count_write[i])
-                        gf_log (this->name, GF_LOG_NORMAL,
+                        gf_log (this->name, GF_LOG_INFO,
                                 "Write %06db+ : %"PRId64,
                                 (1 << i), iosfd->block_count_write[i]);
         }
         return 0;
 }
 
+static void
+update_ios_latency_stats (struct ios_global_stats   *stats, double elapsed,
+                          glusterfs_fop_t op)
+{
+        double avg;
+
+        GF_ASSERT (stats);
+
+        if (!stats->latency[op].min)
+                stats->latency[op].min = elapsed;
+        if (stats->latency[op].min > elapsed)
+                stats->latency[op].min = elapsed;
+        if (stats->latency[op].max < elapsed)
+                stats->latency[op].max = elapsed;
+
+        avg = stats->latency[op].avg;
+
+        stats->latency[op].avg = avg + (elapsed - avg) / stats->fop_hits[op];
+}
+
 int
 update_ios_latency (struct ios_conf *conf, call_frame_t *frame,
                     glusterfs_fop_t op)
 {
         double elapsed;
         struct timeval *begin, *end;
-        double avg;
 
         begin = &frame->begin;
         end   = &frame->end;
@@ -375,43 +1076,143 @@ update_ios_latency (struct ios_conf *conf, call_frame_t *frame,
         elapsed = (end->tv_sec - begin->tv_sec) * 1e6
                 + (end->tv_usec - begin->tv_usec);
 
-        /* Cumulative */
-        if (!conf->cumulative.latency[op].min)
-                conf->cumulative.latency[op].min = elapsed;
-        if (conf->cumulative.latency[op].min > elapsed)
-                conf->cumulative.latency[op].min = elapsed;
-        if (conf->cumulative.latency[op].max < elapsed)
-                conf->cumulative.latency[op].max = elapsed;
-
-        avg = conf->cumulative.latency[op].avg;
+        update_ios_latency_stats (&conf->cumulative, elapsed, op);
+        update_ios_latency_stats (&conf->incremental, elapsed, op);
 
-        conf->cumulative.latency[op].avg =
-                avg + (elapsed - avg) / conf->cumulative.fop_hits[op];
+        return 0;
+}
 
-        /* Incremental */
-        if (!conf->incremental.latency[op].min)
-                conf->incremental.latency[op].min = elapsed;
-        if (conf->incremental.latency[op].min > elapsed)
-                conf->incremental.latency[op].min = elapsed;
-        if (conf->incremental.latency[op].max < elapsed)
-                conf->incremental.latency[op].max = elapsed;
+int32_t
+io_stats_dump_stats_to_dict (xlator_t *this, dict_t *resp,
+                             ios_stats_type_t flags, int32_t list_cnt)
+{
+        struct ios_conf         *conf = NULL;
+        int                      cnt  = 0;
+        char                     key[256];
+        struct ios_stat_head    *list_head = NULL;
+        struct ios_stat_list    *entry = NULL;
+        int                      ret = -1;
+        ios_stats_thru_t         index = IOS_STATS_THRU_MAX;
+        char                     timestr[256] = {0, };
+        char                    *dict_timestr = NULL;
 
-        avg = conf->incremental.latency[op].avg;
+        conf = this->private;
 
-        conf->incremental.latency[op].avg =
-                avg + (elapsed - avg) / conf->incremental.fop_hits[op];
+        switch (flags) {
+                case IOS_STATS_TYPE_OPEN:
+                        list_head = &conf->list[IOS_STATS_TYPE_OPEN];
+                        LOCK (&conf->lock);
+                        {
+                                ret = dict_set_uint64 (resp, "current-open",
+                                                     conf->cumulative.nr_opens);
+                                if (ret)
+                                        goto unlock;
+                                ret = dict_set_uint64 (resp, "max-open",
+                                                       conf->cumulative.max_nr_opens);
+
+                                gf_time_fmt (timestr, sizeof timestr,
+                                             conf->cumulative.max_openfd_time.tv_sec,
+                                             gf_timefmt_FT);
+                                if (conf->cumulative.max_openfd_time.tv_sec)
+                                        snprintf (timestr + strlen (timestr), sizeof timestr - strlen (timestr),
+                                                  ".%"GF_PRI_SUSECONDS,
+                                                  conf->cumulative.max_openfd_time.tv_usec);
+
+                                dict_timestr = gf_strdup (timestr);
+                                if (!dict_timestr)
+                                        goto unlock;
+                                ret = dict_set_dynstr (resp, "max-openfd-time",
+                                                       dict_timestr);
+                                if (ret)
+                                        goto unlock;
+                        }
+        unlock:
+                        UNLOCK (&conf->lock);
+                        /* Do not proceed if we came here because of some error
+                         * during the dict operation */
+                        if (ret)
+                                goto out;
+                        break;
+                case IOS_STATS_TYPE_READ:
+                        list_head = &conf->list[IOS_STATS_TYPE_READ];
+                        break;
+                case IOS_STATS_TYPE_WRITE:
+                        list_head = &conf->list[IOS_STATS_TYPE_WRITE];
+                        break;
+                case IOS_STATS_TYPE_OPENDIR:
+                        list_head = &conf->list[IOS_STATS_TYPE_OPENDIR];
+                        break;
+                case IOS_STATS_TYPE_READDIRP:
+                        list_head = &conf->list[IOS_STATS_TYPE_READDIRP];
+                        break;
+                case IOS_STATS_TYPE_READ_THROUGHPUT:
+                        list_head = &conf->thru_list[IOS_STATS_THRU_READ];
+                        index = IOS_STATS_THRU_READ;
+                        break;
+                case IOS_STATS_TYPE_WRITE_THROUGHPUT:
+                        list_head = &conf->thru_list[IOS_STATS_THRU_WRITE];
+                        index = IOS_STATS_THRU_WRITE;
+                        break;
+
+                default:
+                       goto out;
+        }
+        ret = dict_set_int32 (resp, "top-op", flags);
+        if (!list_cnt)
+                goto out;
+        LOCK (&list_head->lock);
+        {
+                list_for_each_entry (entry, &list_head->iosstats->list, list) {
+
+                        cnt++;
+                        snprintf (key, 256, "%s-%d", "filename", cnt);
+                        ret = dict_set_str (resp, key, entry->iosstat->filename);
+                        if (ret)
+                                goto unlock_list_head;
+                         snprintf (key, 256, "%s-%d", "value",cnt);
+                         ret = dict_set_uint64 (resp, key, entry->value);
+                         if (ret)
+                                 goto unlock_list_head;
+                         if (index != IOS_STATS_THRU_MAX) {
+                                 snprintf (key, 256, "%s-%d", "time-sec", cnt);
+                                 ret = dict_set_int32 (resp, key,
+                                         entry->iosstat->thru_counters[index].time.tv_sec);
+                                 if (ret)
+                                         goto unlock_list_head;
+                                 snprintf (key, 256, "%s-%d", "time-usec", cnt);
+                                 ret = dict_set_int32 (resp, key,
+                                         entry->iosstat->thru_counters[index].time.tv_usec);
+                                 if (ret)
+                                         goto unlock_list_head;
+                         }
+                         if (cnt == list_cnt)
+                                 break;
 
-        return 0;
+                }
+        }
+unlock_list_head:
+        UNLOCK (&list_head->lock);
+        /* ret is !=0 if some dict operation in the above critical region
+         * failed. */
+        if (ret)
+                goto out;
+        ret = dict_set_int32 (resp, "members", cnt);
+ out:
+        return ret;
 }
 
 int
 io_stats_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, fd_t *fd,
                      inode_t *inode, struct iatt *buf,
-                     struct iatt *preparent, struct iatt *postparent)
+                     struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         struct ios_fd *iosfd = NULL;
         char          *path = NULL;
+        struct ios_stat *iosstat = NULL;
+        struct ios_conf   *conf = NULL;
+
+        conf = this->private;
 
         path = frame->local;
         frame->local = NULL;
@@ -434,22 +1235,44 @@ io_stats_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         gettimeofday (&iosfd->opened_at, NULL);
 
         ios_fd_ctx_set (fd, this, iosfd);
+        LOCK (&conf->lock);
+        {
+                conf->cumulative.nr_opens++;
+                if (conf->cumulative.nr_opens > conf->cumulative.max_nr_opens) {
+                        conf->cumulative.max_nr_opens = conf->cumulative.nr_opens;
+                        conf->cumulative.max_openfd_time = iosfd->opened_at;
+                }
+        }
+        UNLOCK (&conf->lock);
+
+        iosstat = GF_CALLOC (1, sizeof (*iosstat), gf_io_stats_mt_ios_stat);
+        if (!iosstat) {
+                GF_FREE (path);
+                goto unwind;
+        }
+        iosstat->filename = gf_strdup (path);
+        uuid_copy (iosstat->gfid, buf->ia_gfid);
+        LOCK_INIT (&iosstat->lock);
+        ios_inode_ctx_set (fd->inode, this, iosstat);
 
 unwind:
-        END_FOP_LATENCY (frame, CREATE);
+        UPDATE_PROFILE_STATS (frame, CREATE);
         STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 io_stats_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd)
+                   int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         struct ios_fd *iosfd = NULL;
         char          *path = NULL;
+        struct   ios_stat *iosstat = NULL;
+        struct ios_conf   *conf = NULL;
 
+        conf = this->private;
         path = frame->local;
         frame->local = NULL;
 
@@ -472,19 +1295,46 @@ io_stats_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         ios_fd_ctx_set (fd, this, iosfd);
 
+        ios_inode_ctx_get (fd->inode, this, &iosstat);
+        if (!iosstat) {
+                iosstat = GF_CALLOC (1, sizeof (*iosstat),
+                                     gf_io_stats_mt_ios_stat);
+                if (iosstat) {
+                        iosstat->filename = gf_strdup (path);
+                        uuid_copy (iosstat->gfid, fd->inode->gfid);
+                        LOCK_INIT (&iosstat->lock);
+                        ios_inode_ctx_set (fd->inode, this, iosstat);
+                }
+        }
+
+        LOCK (&conf->lock);
+        {
+                conf->cumulative.nr_opens++;
+                if (conf->cumulative.nr_opens > conf->cumulative.max_nr_opens) {
+                        conf->cumulative.max_nr_opens = conf->cumulative.nr_opens;
+                        conf->cumulative.max_openfd_time = iosfd->opened_at;
+                }
+        }
+        UNLOCK (&conf->lock);
+        if (iosstat) {
+              BUMP_STATS (iosstat, IOS_STATS_TYPE_OPEN);
+              iosstat = NULL;
+        }
 unwind:
-        END_FOP_LATENCY (frame, OPEN);
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+        UPDATE_PROFILE_STATS (frame, OPEN);
+
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
         return 0;
+
 }
 
 
 int
 io_stats_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, STAT);
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf);
+        UPDATE_PROFILE_STATS (frame, STAT);
+        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
@@ -493,10 +1343,11 @@ int
 io_stats_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
                     struct iovec *vector, int32_t count,
-                    struct iatt *buf, struct iobref *iobref)
+                    struct iatt *buf, struct iobref *iobref, dict_t *xdata)
 {
         int              len = 0;
         fd_t            *fd = NULL;
+        struct ios_stat *iosstat = NULL;
 
         fd = frame->local;
         frame->local = NULL;
@@ -506,21 +1357,46 @@ io_stats_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 BUMP_READ (fd, len);
         }
 
-        END_FOP_LATENCY (frame, READ);
+        UPDATE_PROFILE_STATS (frame, READ);
+        ios_inode_ctx_get (fd->inode, this, &iosstat);
+
+        if (iosstat) {
+              BUMP_STATS (iosstat, IOS_STATS_TYPE_READ);
+              BUMP_THROUGHPUT (iosstat, IOS_STATS_THRU_READ);
+              iosstat = NULL;
+        }
+
         STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                             vector, count, buf, iobref);
+                             vector, count, buf, iobref, xdata);
         return 0;
+
 }
 
 
 int
 io_stats_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
-                     struct iatt *prebuf, struct iatt *postbuf)
-{
-        END_FOP_LATENCY (frame, WRITE);
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
+{
+        struct ios_stat *iosstat = NULL;
+        inode_t         *inode   = NULL;
+
+        UPDATE_PROFILE_STATS (frame, WRITE);
+        if (frame->local){
+                inode = frame->local;
+                frame->local = NULL;
+                ios_inode_ctx_get (inode, this, &iosstat);
+                if (iosstat) {
+                        BUMP_STATS (iosstat, IOS_STATS_TYPE_WRITE);
+                        BUMP_THROUGHPUT (iosstat, IOS_STATS_THRU_WRITE);
+                        inode = NULL;
+                        iosstat = NULL;
+                }
+        }
+
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
         return 0;
+
 }
 
 
@@ -528,20 +1404,33 @@ io_stats_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 io_stats_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, gf_dirent_t *buf)
+                       int32_t op_ret, int32_t op_errno, gf_dirent_t *buf, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, READDIRP);
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, buf);
+        struct ios_stat *iosstat = NULL;
+        inode_t         *inode   = frame->local;
+
+        frame->local = NULL;
+
+        UPDATE_PROFILE_STATS (frame, READDIRP);
+
+        ios_inode_ctx_get (inode, this, &iosstat);
+
+        if (iosstat) {
+              BUMP_STATS (iosstat, IOS_STATS_TYPE_READDIRP);
+              iosstat = NULL;
+        }
+
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
 
 int
 io_stats_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, gf_dirent_t *buf)
+                      int32_t op_ret, int32_t op_errno, gf_dirent_t *buf, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, READDIR);
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, buf);
+        UPDATE_PROFILE_STATS (frame, READDIR);
+        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
@@ -549,10 +1438,10 @@ io_stats_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
-                    struct iatt *prebuf, struct iatt *postbuf)
+                    struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, FSYNC);
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf);
+        UPDATE_PROFILE_STATS (frame, FSYNC);
+        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf, xdata);
         return 0;
 }
 
@@ -560,10 +1449,10 @@ io_stats_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno,
-                      struct iatt *preop, struct iatt *postop)
+                      struct iatt *preop, struct iatt *postop, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, SETATTR);
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop);
+        UPDATE_PROFILE_STATS (frame, SETATTR);
+        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, preop, postop, xdata);
         return 0;
 }
 
@@ -571,12 +1460,13 @@ io_stats_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
-                     struct iatt *preparent, struct iatt *postparent)
+                     struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, UNLINK);
+        UPDATE_PROFILE_STATS (frame, UNLINK);
         STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
+
 }
 
 
@@ -584,12 +1474,12 @@ int
 io_stats_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno, struct iatt *buf,
                      struct iatt *preoldparent, struct iatt *postoldparent,
-                     struct iatt *prenewparent, struct iatt *postnewparent)
+                     struct iatt *prenewparent, struct iatt *postnewparent, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, RENAME);
+        UPDATE_PROFILE_STATS (frame, RENAME);
         STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
                              preoldparent, postoldparent,
-                             prenewparent, postnewparent);
+                             prenewparent, postnewparent, xdata);
         return 0;
 }
 
@@ -597,10 +1487,10 @@ io_stats_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno, const char *buf,
-                       struct iatt *sbuf)
+                       struct iatt *sbuf, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, READLINK);
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, buf, sbuf);
+        UPDATE_PROFILE_STATS (frame, READLINK);
+        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, buf, sbuf, xdata);
         return 0;
 }
 
@@ -609,10 +1499,10 @@ int
 io_stats_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
                      inode_t *inode, struct iatt *buf,
-                     dict_t *xattr, struct iatt *postparent)
+                     dict_t *xdata, struct iatt *postparent)
 {
-        END_FOP_LATENCY (frame, LOOKUP);
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf, xattr,
+        UPDATE_PROFILE_STATS (frame, LOOKUP);
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf, xdata,
                              postparent);
         return 0;
 }
@@ -622,11 +1512,11 @@ int
 io_stats_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                       int32_t op_ret, int32_t op_errno,
                       inode_t *inode, struct iatt *buf,
-                      struct iatt *preparent, struct iatt *postparent)
+                      struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, SYMLINK);
+        UPDATE_PROFILE_STATS (frame, SYMLINK);
         STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
@@ -635,11 +1525,11 @@ int
 io_stats_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
                     inode_t *inode, struct iatt *buf,
-                    struct iatt *preparent, struct iatt *postparent)
+                    struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, MKNOD);
+        UPDATE_PROFILE_STATS (frame, MKNOD);
         STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
@@ -648,11 +1538,30 @@ int
 io_stats_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
                     inode_t *inode, struct iatt *buf,
-                    struct iatt *preparent, struct iatt *postparent)
+                    struct iatt *preparent, struct iatt *postparent,
+                    dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, MKDIR);
+        struct ios_stat *iosstat = NULL;
+        char   *path = frame->local;
+
+        UPDATE_PROFILE_STATS (frame, MKDIR);
+        if (op_ret < 0)
+                goto unwind;
+
+        iosstat = GF_CALLOC (1, sizeof (*iosstat), gf_io_stats_mt_ios_stat);
+        if (iosstat) {
+                LOCK_INIT (&iosstat->lock);
+                iosstat->filename = gf_strdup(path);
+                uuid_copy (iosstat->gfid, buf->ia_gfid);
+                ios_inode_ctx_set (inode, this, iosstat);
+        }
+
+unwind:
+        /* local is assigned with path */
+        GF_FREE (frame->local);
+        frame->local = NULL;
         STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
@@ -661,34 +1570,44 @@ int
 io_stats_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno,
                    inode_t *inode, struct iatt *buf,
-                   struct iatt *preparent, struct iatt *postparent)
+                   struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, LINK);
+        UPDATE_PROFILE_STATS (frame, LINK);
         STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
 
 int
 io_stats_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, FLUSH);
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
+        UPDATE_PROFILE_STATS (frame, FLUSH);
+        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 io_stats_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, fd_t *fd)
+                      int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
-        if (op_ret >= 0)
-                ios_fd_ctx_set (fd, this, 0);
+        struct ios_stat *iosstat = NULL;
+        int              ret     = -1;
+
+        UPDATE_PROFILE_STATS (frame, OPENDIR);
+        if (op_ret < 0)
+                goto unwind;
+
+        ios_fd_ctx_set (fd, this, 0);
 
-        END_FOP_LATENCY (frame, OPENDIR);
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd);
+        ret = ios_inode_ctx_get (fd->inode, this, &iosstat);
+        if (!ret)
+                BUMP_STATS (iosstat, IOS_STATS_TYPE_OPENDIR);
+
+unwind:
+        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
         return 0;
 }
 
@@ -696,11 +1615,13 @@ io_stats_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
-                    struct iatt *preparent, struct iatt *postparent)
+                    struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, RMDIR);
+
+        UPDATE_PROFILE_STATS (frame, RMDIR);
+
         STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
         return 0;
 }
 
@@ -708,71 +1629,100 @@ io_stats_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                        int32_t op_ret, int32_t op_errno,
-                       struct iatt *prebuf, struct iatt *postbuf)
+                       struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, TRUNCATE);
+        UPDATE_PROFILE_STATS (frame, TRUNCATE);
         STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno,
-                             prebuf, postbuf);
+                             prebuf, postbuf, xdata);
         return 0;
 }
 
 
 int
 io_stats_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+                     int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, STATFS);
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf);
+        UPDATE_PROFILE_STATS (frame, STATFS);
+        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
 
 int
 io_stats_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, SETXATTR);
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
+        UPDATE_PROFILE_STATS (frame, SETXATTR);
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 io_stats_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict)
+                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, GETXATTR);
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+        UPDATE_PROFILE_STATS (frame, GETXATTR);
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
 io_stats_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                          int32_t op_ret, int32_t op_errno)
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, REMOVEXATTR);
+        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int
+io_stats_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, FSETXATTR);
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, REMOVEXATTR);
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno);
+        UPDATE_PROFILE_STATS (frame, FGETXATTR);
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS (frame, FREMOVEXATTR);
+        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 io_stats_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, FSYNCDIR);
-        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno);
+        UPDATE_PROFILE_STATS (frame, FSYNCDIR);
+        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 io_stats_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno)
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, ACCESS);
-        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno);
+        UPDATE_PROFILE_STATS (frame, ACCESS);
+        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
@@ -780,376 +1730,364 @@ io_stats_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 io_stats_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                         int32_t op_ret, int32_t op_errno,
-                        struct iatt *prebuf, struct iatt *postbuf)
+                        struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, FTRUNCATE);
+        UPDATE_PROFILE_STATS (frame, FTRUNCATE);
         STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno,
-                             prebuf, postbuf);
+                             prebuf, postbuf, xdata);
         return 0;
 }
 
 
 int
 io_stats_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                    int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, FSTAT);
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf);
+        UPDATE_PROFILE_STATS (frame, FSTAT);
+        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
 
 int
+io_stats_fallocate_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+		       struct iatt *postbuf, dict_t *xdata)
+{
+	UPDATE_PROFILE_STATS(frame, FALLOCATE);
+	STACK_UNWIND_STRICT(fallocate, frame, op_ret, op_errno, prebuf, postbuf,
+			    xdata);
+	return 0;
+}
+
+
+int
+io_stats_discard_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+		     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+		     struct iatt *postbuf, dict_t *xdata)
+{
+	UPDATE_PROFILE_STATS(frame, DISCARD);
+	STACK_UNWIND_STRICT(discard, frame, op_ret, op_errno, prebuf, postbuf,
+			    xdata);
+	return 0;
+}
+
+int
+io_stats_zerofill_cbk(call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                     struct iatt *postbuf, dict_t *xdata)
+{
+        UPDATE_PROFILE_STATS(frame, ZEROFILL);
+        STACK_UNWIND_STRICT(zerofill, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
+        return 0;
+}
+
+int
 io_stats_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+                 int32_t op_ret, int32_t op_errno, struct gf_flock *lock, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, LK);
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock);
+        UPDATE_PROFILE_STATS (frame, LK);
+        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock, xdata);
         return 0;
 }
 
 
 int
 io_stats_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, ENTRYLK);
-        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno);
+        UPDATE_PROFILE_STATS (frame, ENTRYLK);
+        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
 io_stats_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict)
+                      int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, XATTROP);
-        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict);
+        UPDATE_PROFILE_STATS (frame, XATTROP);
+        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
 io_stats_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, dict_t *dict)
+                       int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, FXATTROP);
-        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict);
+        UPDATE_PROFILE_STATS (frame, FXATTROP);
+        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
 
 int
 io_stats_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        END_FOP_LATENCY (frame, INODELK);
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno);
+        UPDATE_PROFILE_STATS (frame, INODELK);
+        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 io_stats_entrylk (call_frame_t *frame, xlator_t *this,
                   const char *volume, loc_t *loc, const char *basename,
-                  entrylk_cmd cmd, entrylk_type type)
+                  entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
-        BUMP_FOP (ENTRYLK);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_entrylk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->entrylk,
-                    volume, loc, basename, cmd, type);
+                    volume, loc, basename, cmd, type, xdata);
         return 0;
 }
 
 
 int
 io_stats_inodelk (call_frame_t *frame, xlator_t *this,
-                  const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock)
+                  const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
 
-        BUMP_FOP (INODELK);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_inodelk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->inodelk,
-                    volume, loc, cmd, flock);
+                    volume, loc, cmd, flock, xdata);
         return 0;
 }
 
 
 int
 io_stats_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
 
-        END_FOP_LATENCY (frame, FINODELK);
-        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno);
+        UPDATE_PROFILE_STATS (frame, FINODELK);
+        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
 
 int
-io_stats_finodelk (call_frame_t *frame, xlator_t *this,
-                   const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock)
+io_stats_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
+                   fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        BUMP_FOP (FINODELK);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_finodelk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->finodelk,
-                    volume, fd, cmd, flock);
+                    volume, fd, cmd, flock, xdata);
         return 0;
 }
 
 
 int
-io_stats_xattrop (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, gf_xattrop_flags_t flags, dict_t *dict)
+io_stats_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        BUMP_FOP (XATTROP);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_xattrop_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->xattrop,
-                    loc, flags, dict);
-
+                    loc, flags, dict, xdata);
         return 0;
 }
 
 
 int
-io_stats_fxattrop (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict)
+io_stats_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
-        BUMP_FOP (FXATTROP);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_fxattrop_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fxattrop,
-                    fd, flags, dict);
-
+                    fd, flags, dict, xdata);
         return 0;
 }
 
 
 int
 io_stats_lookup (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, dict_t *xattr_req)
+                 loc_t *loc, dict_t *xdata)
 {
-        BUMP_FOP (LOOKUP);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_lookup_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->lookup,
-                    loc, xattr_req);
-
+                    loc, xdata);
         return 0;
 }
 
 
 int
-io_stats_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+io_stats_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
-        BUMP_FOP (STAT);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_stat_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->stat,
-                    loc);
-
+                    loc, xdata);
         return 0;
 }
 
 
 int
 io_stats_readlink (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, size_t size)
+                   loc_t *loc, size_t size, dict_t *xdata)
 {
-        BUMP_FOP (READLINK);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_readlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readlink,
-                    loc, size);
-
+                    loc, size, xdata);
         return 0;
 }
 
 
 int
-io_stats_mknod (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, mode_t mode, dev_t dev, dict_t *params)
+io_stats_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
 {
-        BUMP_FOP (MKNOD);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_mknod_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, dev, params);
-
+                    loc, mode, dev, umask, xdata);
         return 0;
 }
 
 
 int
 io_stats_mkdir (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, mode_t mode, dict_t *params)
+                loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
 {
-        BUMP_FOP (MKDIR);
+        frame->local = gf_strdup (loc->path);
 
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_mkdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->mkdir,
-                    loc, mode, params);
+                    loc, mode, umask, xdata);
         return 0;
 }
 
 
 int
 io_stats_unlink (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc)
+                 loc_t *loc, int xflag, dict_t *xdata)
 {
-        BUMP_FOP (UNLINK);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_unlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->unlink,
-                    loc);
+                    loc, xflag, xdata);
         return 0;
 }
 
 
 int
 io_stats_rmdir (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, int flags)
+                loc_t *loc, int flags, dict_t *xdata)
 {
-        BUMP_FOP (RMDIR);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_rmdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->rmdir,
-                    loc, flags);
-
+                    loc, flags, xdata);
         return 0;
 }
 
 
 int
-io_stats_symlink (call_frame_t *frame, xlator_t *this,
-                  const char *linkpath, loc_t *loc, dict_t *params)
+io_stats_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
+                  loc_t *loc, mode_t umask, dict_t *xdata)
 {
-        BUMP_FOP (SYMLINK);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_symlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->symlink,
-                    linkpath, loc, params);
-
+                    linkpath, loc, umask, xdata);
         return 0;
 }
 
 
 int
 io_stats_rename (call_frame_t *frame, xlator_t *this,
-                 loc_t *oldloc, loc_t *newloc)
+                 loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
-        BUMP_FOP (RENAME);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_rename_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->rename,
-                    oldloc, newloc);
-
+                    oldloc, newloc, xdata);
         return 0;
 }
 
 
 int
 io_stats_link (call_frame_t *frame, xlator_t *this,
-               loc_t *oldloc, loc_t *newloc)
+               loc_t *oldloc, loc_t *newloc, dict_t *xdata)
 {
-        BUMP_FOP (LINK);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_link_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->link,
-                    oldloc, newloc);
+                    oldloc, newloc, xdata);
         return 0;
 }
 
 
 int
 io_stats_setattr (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, struct iatt *stbuf, int32_t valid)
+                  loc_t *loc, struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        BUMP_FOP (SETATTR);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_setattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setattr,
-                    loc, stbuf, valid);
-
+                    loc, stbuf, valid, xdata);
         return 0;
 }
 
 
 int
 io_stats_truncate (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, off_t offset)
+                   loc_t *loc, off_t offset, dict_t *xdata)
 {
-        BUMP_FOP (TRUNCATE);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_truncate_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->truncate,
-                    loc, offset);
-
+                    loc, offset, xdata);
         return 0;
 }
 
 
 int
-io_stats_open (call_frame_t *frame, xlator_t *this,
-               loc_t *loc, int32_t flags, fd_t *fd, int32_t wbflags)
+io_stats_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               int32_t flags, fd_t *fd, dict_t *xdata)
 {
-        BUMP_FOP (OPEN);
-
         frame->local = gf_strdup (loc->path);
 
         START_FOP_LATENCY (frame);
@@ -1157,7 +2095,7 @@ io_stats_open (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, io_stats_open_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->open,
-                    loc, flags, fd, wbflags);
+                    loc, flags, fd, xdata);
         return 0;
 }
 
@@ -1165,10 +2103,8 @@ io_stats_open (call_frame_t *frame, xlator_t *this,
 int
 io_stats_create (call_frame_t *frame, xlator_t *this,
                  loc_t *loc, int32_t flags, mode_t mode,
-                 fd_t *fd, dict_t *params)
+                 mode_t umask, fd_t *fd, dict_t *xdata)
 {
-        BUMP_FOP (CREATE);
-
         frame->local = gf_strdup (loc->path);
 
         START_FOP_LATENCY (frame);
@@ -1176,17 +2112,15 @@ io_stats_create (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, io_stats_create_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, xdata);
         return 0;
 }
 
 
 int
 io_stats_readv (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, size_t size, off_t offset)
+                fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
-        BUMP_FOP (READ);
-
         frame->local = fd;
 
         START_FOP_LATENCY (frame);
@@ -1194,7 +2128,7 @@ io_stats_readv (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, io_stats_readv_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readv,
-                    fd, size, offset);
+                    fd, size, offset, flags, xdata);
         return 0;
 }
 
@@ -1203,75 +2137,69 @@ int
 io_stats_writev (call_frame_t *frame, xlator_t *this,
                  fd_t *fd, struct iovec *vector,
                  int32_t count, off_t offset,
-                 struct iobref *iobref)
+                 uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         int                 len = 0;
 
-
+        if (fd->inode)
+                frame->local = fd->inode;
         len = iov_length (vector, count);
 
-        BUMP_FOP (WRITE);
         BUMP_WRITE (fd, len);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_writev_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, offset, iobref);
+                    fd, vector, count, offset, flags, iobref, xdata);
         return 0;
+
 }
 
 
 int
 io_stats_statfs (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc)
+                 loc_t *loc, dict_t *xdata)
 {
-        BUMP_FOP (STATFS);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_statfs_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->statfs,
-                    loc);
+                    loc, xdata);
         return 0;
 }
 
 
 int
 io_stats_flush (call_frame_t *frame, xlator_t *this,
-                fd_t *fd)
+                fd_t *fd, dict_t *xdata)
 {
-        BUMP_FOP (FLUSH);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_flush_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->flush,
-                    fd);
+                    fd, xdata);
         return 0;
 }
 
 
 int
 io_stats_fsync (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, int32_t flags)
+                fd_t *fd, int32_t flags, dict_t *xdata)
 {
-        BUMP_FOP (FSYNC);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_fsync_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsync,
-                    fd, flags);
+                    fd, flags, xdata);
         return 0;
 }
 
 
-void
+int
 conditional_dump (dict_t *dict, char *key, data_t *value, void *data)
 {
         struct {
@@ -1279,30 +2207,43 @@ conditional_dump (dict_t *dict, char *key, data_t *value, void *data)
                 inode_t        *inode;
                 const char     *path;
         } *stub;
-        xlator_t   *this = NULL;
-        inode_t    *inode = NULL;
-        const char *path = NULL;
-        char       *filename = NULL;
+        xlator_t             *this = NULL;
+        char                 *filename = NULL;
+        FILE                 *logfp = NULL;
+        struct ios_dump_args args = {0};
 
         stub  = data;
         this  = stub->this;
-        inode = stub->inode;
-        path  = stub->path;
 
         filename = alloca (value->len + 1);
         memset (filename, 0, value->len + 1);
         memcpy (filename, data_to_str (value), value->len);
 
         if (fnmatch ("*io*stat*dump", key, 0) == 0) {
-                io_stats_dump (this, filename, inode, path);
+
+                if (!strncmp (filename, "", 1)) {
+                        gf_log (this->name, GF_LOG_ERROR, "No filename given");
+                        return -1;
+                }
+                logfp = fopen (filename, "w+");
+                if (!logfp) {
+                        gf_log (this->name, GF_LOG_ERROR, "failed to open %s "
+                                "for writing", filename);
+                        return -1;
+                    }
+                (void) ios_dump_args_init (&args, IOS_DUMP_TYPE_FILE,
+                                           logfp);
+                io_stats_dump (this, &args, GF_CLI_INFO_ALL, _gf_false);
+                fclose (logfp);
         }
+        return 0;
 }
 
 
 int
 io_stats_setxattr (call_frame_t *frame, xlator_t *this,
                    loc_t *loc, dict_t *dict,
-                   int32_t flags)
+                   int32_t flags, dict_t *xdata)
 {
         struct {
                 xlator_t     *this;
@@ -1310,8 +2251,6 @@ io_stats_setxattr (call_frame_t *frame, xlator_t *this,
                 const char   *path;
         } stub;
 
-        BUMP_FOP (SETXATTR);
-
         stub.this  = this;
         stub.inode = loc->inode;
         stub.path  = loc->path;
@@ -1323,186 +2262,244 @@ io_stats_setxattr (call_frame_t *frame, xlator_t *this,
         STACK_WIND (frame, io_stats_setxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setxattr,
-                    loc, dict, flags);
+                    loc, dict, flags, xdata);
         return 0;
 }
 
 
 int
 io_stats_getxattr (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, const char *name)
+                   loc_t *loc, const char *name, dict_t *xdata)
 {
-        BUMP_FOP (GETXATTR);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_getxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->getxattr,
-                    loc, name);
+                    loc, name, xdata);
         return 0;
 }
 
 
 int
 io_stats_removexattr (call_frame_t *frame, xlator_t *this,
-                      loc_t *loc, const char *name)
+                      loc_t *loc, const char *name, dict_t *xdata)
 {
-        BUMP_FOP (REMOVEXATTR);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_removexattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->removexattr,
-                    loc, name);
+                    loc, name, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fsetxattr (call_frame_t *frame, xlator_t *this,
+                    fd_t *fd, dict_t *dict,
+                    int32_t flags, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_fsetxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr,
+                    fd, dict, flags, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fgetxattr (call_frame_t *frame, xlator_t *this,
+                    fd_t *fd, const char *name, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
 
+        STACK_WIND (frame, io_stats_fgetxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr,
+                    fd, name, xdata);
+        return 0;
+}
+
+
+int
+io_stats_fremovexattr (call_frame_t *frame, xlator_t *this,
+                       fd_t *fd, const char *name, dict_t *xdata)
+{
+        START_FOP_LATENCY (frame);
+
+        STACK_WIND (frame, io_stats_fremovexattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fremovexattr,
+                    fd, name, xdata);
         return 0;
 }
 
 
 int
 io_stats_opendir (call_frame_t *frame, xlator_t *this,
-                  loc_t *loc, fd_t *fd)
+                  loc_t *loc, fd_t *fd, dict_t *xdata)
 {
-        BUMP_FOP (OPENDIR);
 
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_opendir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->opendir,
-                    loc, fd);
+                    loc, fd, xdata);
         return 0;
 }
 
 int
 io_stats_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                   off_t offset)
+                   off_t offset, dict_t *dict)
 {
-        BUMP_FOP (READDIRP);
-
+        frame->local = fd->inode;
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_readdirp_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readdirp,
-                    fd, size, offset);
-
+                    fd, size, offset, dict);
         return 0;
 }
 
 
 int
 io_stats_readdir (call_frame_t *frame, xlator_t *this,
-                  fd_t *fd, size_t size, off_t offset)
+                  fd_t *fd, size_t size, off_t offset, dict_t *xdata)
 {
-        BUMP_FOP (READDIR);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_readdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readdir,
-                    fd, size, offset);
-
+                    fd, size, offset, xdata);
         return 0;
 }
 
 
 int
 io_stats_fsyncdir (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, int32_t datasync)
+                   fd_t *fd, int32_t datasync, dict_t *xdata)
 {
-        BUMP_FOP (FSYNCDIR);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_fsyncdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsyncdir,
-                    fd, datasync);
+                    fd, datasync, xdata);
         return 0;
 }
 
 
 int
 io_stats_access (call_frame_t *frame, xlator_t *this,
-                 loc_t *loc, int32_t mask)
+                 loc_t *loc, int32_t mask, dict_t *xdata)
 {
-        BUMP_FOP (ACCESS);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_access_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->access,
-                    loc, mask);
+                    loc, mask, xdata);
         return 0;
 }
 
 
 int
 io_stats_ftruncate (call_frame_t *frame, xlator_t *this,
-                    fd_t *fd, off_t offset)
+                    fd_t *fd, off_t offset, dict_t *xdata)
 {
-        BUMP_FOP (FTRUNCATE);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_ftruncate_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->ftruncate,
-                    fd, offset);
-
+                    fd, offset, xdata);
         return 0;
 }
 
 
 int
 io_stats_fsetattr (call_frame_t *frame, xlator_t *this,
-                   fd_t *fd, struct iatt *stbuf, int32_t valid)
+                   fd_t *fd, struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        BUMP_FOP (FSETATTR);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_setattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsetattr,
-                    fd, stbuf, valid);
+                    fd, stbuf, valid, xdata);
         return 0;
 }
 
 
 int
 io_stats_fstat (call_frame_t *frame, xlator_t *this,
-                fd_t *fd)
+                fd_t *fd, dict_t *xdata)
 {
-        BUMP_FOP (FSTAT);
-
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_fstat_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fstat,
-                    fd);
+                    fd, xdata);
         return 0;
 }
 
 
 int
-io_stats_lk (call_frame_t *frame, xlator_t *this,
-             fd_t *fd, int32_t cmd, struct gf_flock *lock)
+io_stats_fallocate(call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t mode,
+		   off_t offset, size_t len, dict_t *xdata)
+{
+	START_FOP_LATENCY(frame);
+
+	STACK_WIND(frame, io_stats_fallocate_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fallocate, fd, mode, offset, len,
+		   xdata);
+
+	return 0;
+}
+
+
+int
+io_stats_discard(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+		 size_t len, dict_t *xdata)
+{
+	START_FOP_LATENCY(frame);
+
+	STACK_WIND(frame, io_stats_discard_cbk, FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->discard, fd, offset, len, xdata);
+
+	return 0;
+}
+
+int
+io_stats_zerofill(call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 off_t len, dict_t *xdata)
 {
-        BUMP_FOP (LK);
+        START_FOP_LATENCY(frame);
+
+        STACK_WIND(frame, io_stats_zerofill_cbk, FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->zerofill, fd, offset, len, xdata);
+
+        return 0;
+}
+
 
+int
+io_stats_lk (call_frame_t *frame, xlator_t *this,
+             fd_t *fd, int32_t cmd, struct gf_flock *lock, dict_t *xdata)
+{
         START_FOP_LATENCY (frame);
 
         STACK_WIND (frame, io_stats_lk_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->lk,
-                    fd, cmd, lock);
+                    fd, cmd, lock, xdata);
         return 0;
 }
 
@@ -1511,15 +2508,23 @@ int
 io_stats_release (xlator_t *this, fd_t *fd)
 {
         struct ios_fd  *iosfd = NULL;
+        struct ios_conf *conf = NULL;
 
         BUMP_FOP (RELEASE);
 
+        conf = this->private;
+
+        LOCK (&conf->lock);
+        {
+                conf->cumulative.nr_opens--;
+        }
+        UNLOCK (&conf->lock);
+
         ios_fd_ctx_get (fd, this, &iosfd);
         if (iosfd) {
                 io_stats_dump_fd (this, iosfd);
 
-                if (iosfd->filename)
-                        GF_FREE (iosfd->filename);
+                GF_FREE (iosfd->filename);
                 GF_FREE (iosfd);
         }
 
@@ -1540,62 +2545,180 @@ int
 io_stats_forget (xlator_t *this, inode_t *inode)
 {
         BUMP_FOP (FORGET);
+        ios_stats_cleanup (this, inode);
+        return 0;
+}
+
+static int
+ios_init_top_stats (struct ios_conf *conf)
+{
+        int     i = 0;
+
+        GF_ASSERT (conf);
+
+        for (i = 0; i <IOS_STATS_TYPE_MAX; i++) {
+                conf->list[i].iosstats = GF_CALLOC (1,
+                                         sizeof(*conf->list[i].iosstats),
+                                         gf_io_stats_mt_ios_stat);
+
+                if (!conf->list[i].iosstats)
+                        return -1;
+
+                INIT_LIST_HEAD(&conf->list[i].iosstats->list);
+                LOCK_INIT (&conf->list[i].lock);
+        }
+
+        for (i = 0; i < IOS_STATS_THRU_MAX; i ++) {
+                conf->thru_list[i].iosstats = GF_CALLOC (1,
+                                 sizeof (*conf->thru_list[i].iosstats),
+                                 gf_io_stats_mt_ios_stat);
+
+                if (!conf->thru_list[i].iosstats)
+                        return -1;
+
+                INIT_LIST_HEAD(&conf->thru_list[i].iosstats->list);
+                LOCK_INIT (&conf->thru_list[i].lock);
+        }
 
         return 0;
 }
 
+static void
+ios_destroy_top_stats (struct ios_conf *conf)
+{
+        int                     i = 0;
+        struct ios_stat_head    *list_head = NULL;
+        struct ios_stat_list    *entry     = NULL;
+        struct ios_stat_list    *tmp       = NULL;
+        struct ios_stat_list    *list      = NULL;
+        struct ios_stat         *stat      = NULL;
+
+        GF_ASSERT (conf);
+
+        LOCK (&conf->lock);
+
+        conf->cumulative.nr_opens = 0;
+        conf->cumulative.max_nr_opens = 0;
+        conf->cumulative.max_openfd_time.tv_sec = 0;
+        conf->cumulative.max_openfd_time.tv_usec = 0;
+
+        for (i = 0; i < IOS_STATS_TYPE_MAX; i++) {
+                list_head = &conf->list[i];
+                if (!list_head)
+                        continue;
+                list_for_each_entry_safe (entry, tmp,
+                                          &list_head->iosstats->list, list) {
+                        list = entry;
+                        stat = list->iosstat;
+                        ios_stat_unref (stat);
+                        list_del (&list->list);
+                        GF_FREE (list);
+                        list_head->members--;
+                }
+        }
+
+        for (i = 0; i < IOS_STATS_THRU_MAX; i++) {
+                list_head = &conf->thru_list[i];
+                if (!list_head)
+                        continue;
+                list_for_each_entry_safe (entry, tmp,
+                                          &list_head->iosstats->list, list) {
+                        list = entry;
+                        stat = list->iosstat;
+                        ios_stat_unref (stat);
+                        list_del (&list->list);
+                        GF_FREE (list);
+                        list_head->members--;
+                }
+        }
+
+        UNLOCK (&conf->lock);
+
+        return;
+}
+
+static int
+io_stats_clear (struct ios_conf *conf)
+{
+        struct timeval      now;
+        int                 ret = -1;
+
+        GF_ASSERT (conf);
+
+        if (!gettimeofday (&now, NULL))
+        {
+            LOCK (&conf->lock);
+            {
+                    ios_global_stats_clear (&conf->cumulative, &now);
+                    ios_global_stats_clear (&conf->incremental, &now);
+                    conf->increment = 0;
+            }
+            UNLOCK (&conf->lock);
+            ret = 0;
+        }
+
+        return ret;
+}
+
 int
 reconfigure (xlator_t *this, dict_t *options)
 {
         struct ios_conf    *conf = NULL;
-        char               *str = NULL;
-        int                 ret = 0;
+        int                 ret = -1;
+        char               *sys_log_str = NULL;
+        char               *log_format_str = NULL;
+        char               *logger_str    = NULL;
+        int                 sys_log_level = -1;
         char               *log_str = NULL;
-        glusterfs_ctx_t     *ctx = NULL;
+        int                 log_level = -1;
+        int                 log_format = -1;
+        int                 logger = -1;
 
         if (!this || !this->private)
-                return -1;
+                goto out;
 
         conf = this->private;
 
-        ret = dict_get_str (options, "dump-fd-stats", &str);
-        if (ret == 0) {
-                ret = gf_string2boolean (str, &conf->dump_fd_stats);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "'dump-fd-stats' takes only boolean arguments");
-                        return -1;
-                }
+        GF_OPTION_RECONF ("dump-fd-stats", conf->dump_fd_stats, options, bool,
+                          out);
 
-                if (conf->dump_fd_stats) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"enabling dump-fd-stats");
-                }
+        GF_OPTION_RECONF ("count-fop-hits", conf->count_fop_hits, options, bool,
+                          out);
+
+        GF_OPTION_RECONF ("latency-measurement", conf->measure_latency,
+                          options, bool, out);
+
+        GF_OPTION_RECONF ("sys-log-level", sys_log_str, options, str, out);
+        if (sys_log_str) {
+                sys_log_level = glusterd_check_log_level (sys_log_str);
+                set_sys_log_level (sys_log_level);
         }
 
-        ret = dict_get_str_boolean (options, "latency-measurement", 0);
-        if (ret != -1) {
-                if (conf->measure_latency != ret) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "changing latency measurement from %d to %d",
-                                conf->measure_latency, ret);
-                }
-                conf->measure_latency = ret;
+        GF_OPTION_RECONF ("log-level", log_str, options, str, out);
+        if (log_str) {
+                log_level = glusterd_check_log_level (log_str);
+                gf_log_set_loglevel (log_level);
         }
-        ctx = glusterfs_ctx_get ();
-        if (!ctx)
-                return -1;
 
-        ret = dict_get_str (options, "log-level", &log_str);
-        if (!ret) {
-                if (!is_gf_log_command(this, "trusted.glusterfs*set-log-level", log_str)) {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                               "changing log-level to %s", log_str);
-                }
+        GF_OPTION_RECONF ("logger", logger_str, options, str, out);
+        if (logger_str) {
+                logger = gf_check_logger (logger_str);
+                gf_log_set_logger (logger);
         }
-        return 0;
+
+        GF_OPTION_RECONF ("log-format", log_format_str, options, str, out);
+        if (log_format_str) {
+                log_format = gf_check_log_format (log_format_str);
+                gf_log_set_logformat (log_format);
+        }
+
+        ret = 0;
+out:
+        gf_log (this->name, GF_LOG_DEBUG, "reconfigure returning %d", ret);
+        return ret;
 }
 
+
 int32_t
 mem_acct_init (xlator_t *this)
 {
@@ -1605,7 +2728,7 @@ mem_acct_init (xlator_t *this)
                 return ret;
 
         ret = xlator_mem_acct_init (this, gf_io_stats_mt_end + 1);
-        
+
         if (ret != 0) {
                 gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
                         " failed");
@@ -1618,27 +2741,34 @@ mem_acct_init (xlator_t *this)
 int
 init (xlator_t *this)
 {
-        dict_t             *options = NULL;
         struct ios_conf    *conf = NULL;
-        char               *str = NULL;
-        int                 ret = 0;
+        char               *sys_log_str = NULL;
+        char               *logger_str  = NULL;
+        char               *log_format_str  = NULL;
+        int                 logger = -1;
+        int                 log_format = -1;
+        int                 sys_log_level = -1;
+        char               *log_str = NULL;
+        int                 log_level = -1;
+        int                 ret = -1;
 
         if (!this)
                 return -1;
 
-        if (!this->children || this->children->next) {
+        if (!this->children) {
                 gf_log (this->name, GF_LOG_ERROR,
-                        "io_stats translator requires one subvolume");
+                        "io_stats translator requires atleast one subvolume");
                 return -1;
         }
 
         if (!this->parents) {
-                gf_log (this->name, GF_LOG_WARNING,
+                /* This is very much valid as io-stats currently is loaded
+                 * on top of volumes on both client and server, hence this is
+                 * not an warning message */
+                gf_log (this->name, GF_LOG_DEBUG,
                         "dangling volume. check volfile ");
         }
 
-        options = this->options;
-
         conf = GF_CALLOC (1, sizeof(*conf), gf_io_stats_mt_ios_conf);
 
         if (!conf) {
@@ -1652,32 +2782,46 @@ init (xlator_t *this)
         gettimeofday (&conf->cumulative.started_at, NULL);
         gettimeofday (&conf->incremental.started_at, NULL);
 
-        ret = dict_get_str (options, "dump-fd-stats", &str);
-        if (ret == 0) {
-                ret = gf_string2boolean (str, &conf->dump_fd_stats);
-                if (ret == -1) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "'dump-fd-stats' takes only boolean arguments");
-                        return -1;
-                }
+        ret = ios_init_top_stats (conf);
+        if (ret)
+                return -1;
 
-                if (conf->dump_fd_stats) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"enabling dump-fd-stats");
-                }
+        GF_OPTION_INIT ("dump-fd-stats", conf->dump_fd_stats, bool, out);
+
+        GF_OPTION_INIT ("count-fop-hits", conf->count_fop_hits, bool, out);
+
+        GF_OPTION_INIT ("latency-measurement", conf->measure_latency,
+                          bool, out);
+
+        GF_OPTION_INIT ("sys-log-level", sys_log_str, str, out);
+        if (sys_log_str) {
+                sys_log_level = glusterd_check_log_level (sys_log_str);
+                set_sys_log_level (sys_log_level);
         }
 
-        ret = dict_get_str_boolean (options, "latency-measurement", 0);
-        if (ret != -1) {
-                conf->measure_latency = ret;
-                if (conf->measure_latency)
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "enabling latency measurement");
+        GF_OPTION_INIT ("log-level", log_str, str, out);
+        if (log_str) {
+                log_level = glusterd_check_log_level (log_str);
+                gf_log_set_loglevel (log_level);
         }
 
-        this->private = conf;
+        GF_OPTION_INIT ("logger", logger_str, str, out);
+        if (logger_str) {
+                logger = gf_check_logger (logger_str);
+                gf_log_set_logger (logger);
+        }
 
-        return 0;
+        GF_OPTION_INIT ("log-format", log_format_str, str, out);
+        if (log_format_str) {
+                log_format = gf_check_log_format (log_format_str);
+                gf_log_set_logformat (log_format);
+        }
+
+
+        this->private = conf;
+        ret = 0;
+out:
+        return ret;
 }
 
 
@@ -1695,13 +2839,131 @@ fini (xlator_t *this)
                 return;
         this->private = NULL;
 
+        ios_destroy_top_stats (conf);
+
         GF_FREE(conf);
 
-        gf_log (this->name, GF_LOG_NORMAL,
+        gf_log (this->name, GF_LOG_INFO,
                 "io-stats translator unloaded");
         return;
 }
 
+int
+notify (xlator_t *this, int32_t event, void *data, ...)
+{
+        int          ret = 0;
+        struct ios_dump_args args = {0};
+        dict_t       *output = NULL;
+        dict_t       *dict = NULL;
+        int32_t       op = 0;
+        int32_t       list_cnt = 0;
+        double        throughput = 0;
+        double        time = 0;
+        gf_boolean_t  is_peek = _gf_false;
+        va_list ap;
+
+        dict = data;
+        va_start (ap, data);
+        output = va_arg (ap, dict_t*);
+        va_end (ap);
+        switch (event) {
+        case GF_EVENT_TRANSLATOR_INFO:
+                ret = dict_get_str_boolean (dict, "clear-stats", _gf_false);
+                if (ret) {
+                        ret = dict_set_int32 (output, "top-op", op);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set top-op in dict");
+                                goto out;
+                        }
+                        ios_destroy_top_stats (this->private);
+                        ret = ios_init_top_stats (this->private);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to reset top stats");
+                        ret = dict_set_int32 (output, "stats-cleared",
+                                              ret ? 0 : 1);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set stats-cleared"
+                                        " in dict");
+                        goto out;
+                }
+
+                ret = dict_get_int32 (dict, "top-op", &op);
+                if (!ret) {
+                        ret = dict_get_int32 (dict, "list-cnt", &list_cnt);
+                        if (op > IOS_STATS_TYPE_NONE &&
+                            op < IOS_STATS_TYPE_MAX)
+                                ret = io_stats_dump_stats_to_dict (this, output,
+                                                             op, list_cnt);
+                        if (op == IOS_STATS_TYPE_READ_THROUGHPUT ||
+                                op == IOS_STATS_TYPE_WRITE_THROUGHPUT) {
+                                ret = dict_get_double (dict, "throughput",
+                                                        &throughput);
+                                if (!ret) {
+                                        ret = dict_get_double (dict, "time",
+                                                                &time);
+                                        if (ret)
+                                                goto out;
+                                        ret = dict_set_double (output,
+                                                "throughput", throughput);
+                                        if (ret)
+                                                goto out;
+                                        ret = dict_set_double (output, "time",
+                                                        time);
+                                        if (ret)
+                                                goto out;
+                                }
+                                ret = 0;
+
+                        }
+                } else {
+                        ret = dict_get_int32 (dict, "info-op", &op);
+                        if (ret || op < GF_CLI_INFO_ALL ||
+                            GF_CLI_INFO_CLEAR < op)
+                            op = GF_CLI_INFO_ALL;
+
+                        ret = dict_set_int32 (output, "info-op", op);
+                        if (ret) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Failed to set info-op in dict");
+                                goto out;
+                        }
+
+                        if (GF_CLI_INFO_CLEAR == op) {
+                                ret = io_stats_clear (this->private);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Failed to clear info stats");
+
+                                ret = dict_set_int32 (output, "stats-cleared",
+                                              ret ? 0 : 1);
+                                if (ret)
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Failed to set stats-cleared"
+                                                " in dict");
+                        }
+                        else {
+                                ret = dict_get_str_boolean (dict, "peek",
+                                                            _gf_false);
+                                if (-1 != ret)
+                                        is_peek = ret;
+
+                                (void) ios_dump_args_init (&args,
+                                                IOS_DUMP_TYPE_DICT, output);
+                                ret = io_stats_dump (this, &args, op, is_peek);
+                        }
+                }
+                break;
+        default:
+                default_notify (this, event, data);
+                break;
+
+        }
+out:
+        return ret;
+}
 
 struct xlator_fops fops = {
         .stat        = io_stats_stat,
@@ -1723,6 +2985,9 @@ struct xlator_fops fops = {
         .setxattr    = io_stats_setxattr,
         .getxattr    = io_stats_getxattr,
         .removexattr = io_stats_removexattr,
+        .fsetxattr    = io_stats_fsetxattr,
+        .fgetxattr    = io_stats_fgetxattr,
+        .fremovexattr = io_stats_fremovexattr,
         .opendir     = io_stats_opendir,
         .readdir     = io_stats_readdir,
         .readdirp    = io_stats_readdirp,
@@ -1740,6 +3005,9 @@ struct xlator_fops fops = {
         .fxattrop    = io_stats_fxattrop,
         .setattr     = io_stats_setattr,
         .fsetattr    = io_stats_fsetattr,
+	.fallocate   = io_stats_fallocate,
+	.discard     = io_stats_discard,
+        .zerofill    = io_stats_zerofill,
 };
 
 struct xlator_cbks cbks = {
@@ -1751,12 +3019,81 @@ struct xlator_cbks cbks = {
 struct volume_options options[] = {
         { .key  = {"dump-fd-stats"},
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "If on stats related to file-operations would be "
+                         "tracked inside GlusterFS data-structures."
         },
         { .key  = { "latency-measurement" },
           .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "If on stats related to the latency of each operation "
+                         "would be tracked inside GlusterFS data-structures. "
+        },
+        { .key  = {"count-fop-hits"},
+          .type = GF_OPTION_TYPE_BOOL,
         },
         { .key = {"log-level"},
           .type = GF_OPTION_TYPE_STR,
+          .value = { "DEBUG", "WARNING", "ERROR", "INFO",
+                     "CRITICAL", "NONE", "TRACE"}
+        },
+
+        /* These are synthetic entries to assist validation of CLI's  *
+         *  volume set  command                                       */
+        { .key = {"client-log-level"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "INFO",
+          .description = "Changes the log-level of the clients",
+          .value = { "DEBUG", "WARNING", "ERROR", "INFO",
+                     "CRITICAL", "NONE", "TRACE"}
+        },
+        { .key = {"sys-log-level"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "CRITICAL",
+          .description = "Gluster's syslog log-level",
+          .value = { "WARNING", "ERROR", "INFO", "CRITICAL"}
+        },
+        { .key = {"brick-log-level"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = "INFO",
+          .description = "Changes the log-level of the bricks",
+          .value = { "DEBUG", "WARNING", "ERROR", "INFO",
+                     "CRITICAL", "NONE", "TRACE"}
+        },
+        { .key = {"logger"},
+          .type = GF_OPTION_TYPE_STR,
+          .value = { GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}
+        },
+        { .key = {"client-logger"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = GF_LOGGER_GLUSTER_LOG,
+          .description = "Changes the logging sub-system to log to, for the "
+                         "clients",
+          .value = { GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}
+        },
+        { .key = {"brick-logger"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = GF_LOGGER_GLUSTER_LOG,
+          .description = "Changes the logging sub-system to log to, for the "
+                         "bricks",
+          .value = { GF_LOGGER_GLUSTER_LOG, GF_LOGGER_SYSLOG}
+        },
+        { .key = {"log-format"},
+          .type = GF_OPTION_TYPE_STR,
+          .value = { GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}
+        },
+        { .key = {"client-log-format"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = GF_LOG_FORMAT_WITH_MSG_ID,
+          .description = "Changes log format for the clients",
+          .value = { GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}
+        },
+        { .key = {"brick-log-format"},
+          .type = GF_OPTION_TYPE_STR,
+          .default_value = GF_LOG_FORMAT_WITH_MSG_ID,
+          .description = "Changes the log format for the bricks",
+          .value = { GF_LOG_FORMAT_NO_MSG_ID, GF_LOG_FORMAT_WITH_MSG_ID}
         },
         { .key  = {NULL} },
+
 };
diff --git a/xlators/debug/trace/src/Makefile.am b/xlators/debug/trace/src/Makefile.am
index 0f1679a04..7b2597b4d 100644
--- a/xlators/debug/trace/src/Makefile.am
+++ b/xlators/debug/trace/src/Makefile.am
@@ -2,13 +2,15 @@
 xlator_LTLIBRARIES = trace.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/debug
 
-trace_la_LDFLAGS = -module -avoidversion
+trace_la_LDFLAGS = -module -avoid-version
 
 trace_la_SOURCES = trace.c
 trace_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+noinst_HEADERS = trace.h trace-mem-types.h
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/debug/trace/src/trace-mem-types.h b/xlators/debug/trace/src/trace-mem-types.h
new file mode 100644
index 000000000..9fa7d97c2
--- /dev/null
+++ b/xlators/debug/trace/src/trace-mem-types.h
@@ -0,0 +1,21 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef __TRACE_MEM_TYPES_H__
+#define __TRACE_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_trace_mem_types_ {
+        gf_trace_mt_trace_conf_t = gf_common_mt_end + 1,
+        gf_trace_mt_end
+};
+#endif
diff --git a/xlators/debug/trace/src/trace.c b/xlators/debug/trace/src/trace.c
index bc7a976dd..1efd50e65 100644
--- a/xlators/debug/trace/src/trace.c
+++ b/xlators/debug/trace/src/trace.c
@@ -1,26 +1,15 @@
 /*
-  Copyright (c) 2006-2009 Gluster, Inc. <http://www.gluster.com>
+  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
 */
 
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
+#include "trace.h"
+#include "trace-mem-types.h"
 
 /**
  * xlators/debug/trace :
@@ -29,1209 +18,1483 @@
  *    Very helpful translator for debugging.
  */
 
-#include <time.h>
-#include <errno.h>
-#include "glusterfs.h"
-#include "xlator.h"
-#include "common-utils.h"
-
-#define ERR_EINVAL_NORETURN(cond)                               \
-        do                                                      \
-        {                                                       \
-                if ((cond))                                     \
-                {                                               \
-                        gf_log ("ERROR", GF_LOG_ERROR,          \
-                                "%s: %s: (%s) is true",         \
-                                __FILE__, __FUNCTION__, #cond); \
-                }                                               \
-        } while (0)
-
-
-typedef struct trace_private {
-        int32_t debug_flag;
-} trace_private_t;
-
-
-struct {
-        char *name;
-        int enabled;
-} trace_fop_names[GF_FOP_MAXVALUE];
-
-int trace_log_level = GF_LOG_NORMAL;
-
-static char *
-trace_stat_to_str (struct iatt *stbuf)
+static inline void
+trace_stat_to_str(struct iatt *buf, char *str)
 {
-        char    *statstr           = NULL;
         char     atime_buf[256]    = {0,};
         char     mtime_buf[256]    = {0,};
         char     ctime_buf[256]    = {0,};
-        int      asprint_ret_value = 0;
         uint64_t ia_time           = 0;
 
-        ia_time = stbuf->ia_atime;
+        if (!buf)
+                return;
+
+        ia_time = buf->ia_atime;
         strftime (atime_buf, 256, "[%b %d %H:%M:%S]",
                   localtime ((time_t *)&ia_time));
-
-        ia_time = stbuf->ia_mtime;
+        ia_time = buf->ia_mtime;
         strftime (mtime_buf, 256, "[%b %d %H:%M:%S]",
                   localtime ((time_t *)&ia_time));
 
-        ia_time = stbuf->ia_ctime;
+        ia_time = buf->ia_ctime;
         strftime (ctime_buf, 256, "[%b %d %H:%M:%S]",
                   localtime ((time_t *)&ia_time));
 
-        asprint_ret_value = gf_asprintf (&statstr,
-                                      "ia_ino=%"PRIu64
-                                      ", st_mode=%o, ia_nlink=%"GF_PRI_NLINK", "
-                                      "ia_uid=%d, ia_gid=%d, ia_size=%"PRId64", ia_blocks=%"PRId64
-                                      ", ia_atime=%s, ia_mtime=%s, ia_ctime=%s",
-                                      stbuf->ia_ino,
-                                      st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type),
-                                      stbuf->ia_nlink, stbuf->ia_uid,
-                                      stbuf->ia_gid, stbuf->ia_size,
-                                      stbuf->ia_blocks, atime_buf,
-                                      mtime_buf, ctime_buf);
+        snprintf (str, sizeof (str),
+                  "gfid=%s ino=%"PRIu64", mode=%o, "
+                  "nlink=%"GF_PRI_NLINK", uid=%u, "
+                  "gid=%u, size=%"PRIu64", "
+                  "blocks=%"PRIu64", atime=%s, "
+                  "mtime=%s, ctime=%s",
+                  uuid_utoa (buf->ia_gfid),
+                  buf->ia_ino,
+                  st_mode_from_ia (buf->ia_prot, buf->ia_type),
+                  buf->ia_nlink, buf->ia_uid,
+                  buf->ia_gid, buf->ia_size,
+                  buf->ia_blocks, atime_buf,
+                  mtime_buf, ctime_buf);
+}
 
-        if (asprint_ret_value < 0)
-                statstr = NULL;
 
-        return statstr;
-}
+int
+dump_history_trace (circular_buffer_t *cb, void *data)
+{
+        char    *string      = NULL;
+        struct  tm      *tm  = NULL;
+        char    timestr[256] = {0,};
+
+        string = (char *)cb->data;
+        tm    = localtime (&cb->tv.tv_sec);
+
+        /* Since we are continuing with adding entries to the buffer even when
+           gettimeofday () fails, it's safe to check tm and then dump the time
+           at which the entry was added to the buffer */
+        if (tm) {
+                strftime (timestr, 256, "%Y-%m-%d %H:%M:%S", tm);
+                snprintf (timestr + strlen (timestr), 256 - strlen (timestr),
+                          ".%"GF_PRI_SUSECONDS, cb->tv.tv_usec);
+                gf_proc_dump_write ("TIME", "%s", timestr);
+        }
+
+        gf_proc_dump_write ("FOP", "%s\n", string);
 
+        return 0;
+}
 
 int
 trace_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, fd_t *fd,
                   inode_t *inode, struct iatt *buf,
-                  struct iatt *preparent, struct iatt *postparent)
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
 {
-        char  *statstr = NULL;
-        char  *preparentstr = NULL;
-        char  *postparentstr = NULL;
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_CREATE].enabled) {
+                char  string[4096] = {0,};
                 if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, fd=%p, ino=%"PRIu64" "
-                                "*stbuf {%s}, *preparent {%s}, *postparent = "
-                                "{%s})",
-                                frame->root->unique, op_ret, fd, inode->ino,
-                                statstr, preparentstr, postparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d, fd=%p"
+                                  "*stbuf {%s}, *preparent {%s}, "
+                                  "*postparent = {%s})",
+                                  frame->root->unique,
+                                  uuid_utoa (inode->gfid), op_ret, fd,
+                                  statstr, preparentstr, postparentstr);
+
+                        /* for 'release' log */
+                        fd_ctx_set (fd, this, 0);
                 } else {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (create, frame, op_ret, op_errno, fd, inode, buf,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, fd_t *fd)
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
+        trace_conf_t      *conf = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_OPEN].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (op_ret=%d, op_errno=%d, *fd=%p)",
-                        frame->root->unique, op_ret, op_errno, fd);
+                char     string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d, "
+                          "*fd=%p", frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno,
+                          fd);
+
+                LOG_ELEMENT (conf, string);
         }
 
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+out:
+        /* for 'release' log */
+        if (op_ret >= 0)
+                fd_ctx_set (fd, this, 0);
+
+        TRACE_STACK_UNWIND (open, frame, op_ret, op_errno, fd, xdata);
         return 0;
 }
 
-
 int
 trace_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                dict_t *xdata)
 {
-        uint64_t ia_time = 0;
-        char     atime_buf[256];
-        char     mtime_buf[256];
-        char     ctime_buf[256];
+        char          statstr[4096] = {0, };
+        trace_conf_t  *conf         = NULL;
 
-        if (trace_fop_names[GF_FOP_STAT].enabled) {
-                if (op_ret >= 0) {
-                        ia_time = buf->ia_atime;
-                        strftime (atime_buf, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
-
-                        ia_time = buf->ia_mtime;
-                        strftime (mtime_buf, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
-
-                        ia_time = buf->ia_ctime;
-                        strftime (ctime_buf, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
+        conf = this->private;
 
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, buf {"
-                                "ia_ino=%"PRIu64", st_mode=%o, ia_nlink=%"GF_PRI_NLINK", "
-                                "ia_uid=%d, ia_gid=%d, ia_rdev=%"PRIu64", ia_size=%"PRId64
-                                ", ia_blksize=%"GF_PRI_BLKSIZE", ia_blocks=%"PRId64", "
-                                "ia_atime=%s, ia_mtime=%s, ia_ctime=%s})",
-                                frame->root->unique, op_ret, buf->ia_ino,
-                                st_mode_from_ia (buf->ia_prot, buf->ia_type),
-                                buf->ia_nlink, buf->ia_uid, buf->ia_gid,
-                                buf->ia_rdev, buf->ia_size, buf->ia_blksize,
-                                buf->ia_blocks, atime_buf, mtime_buf, ctime_buf);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_STAT].enabled) {
+                char string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d buf=%s",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  statstr);
                 } else {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf);
+out:
+        TRACE_STACK_UNWIND (stat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
-
 int
 trace_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, struct iovec *vector,
-                 int32_t count, struct iatt *buf, struct iobref *iobref)
+                 int32_t count, struct iatt *buf, struct iobref *iobref,
+                 dict_t *xdata)
 {
-        uint64_t ia_time = 0;
-        char     atime_buf[256];
-        char     mtime_buf[256];
-        char     ctime_buf[256];
+        char          statstr[4096] = {0, };
+        trace_conf_t  *conf         = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READ].enabled) {
+                char  string[4096] = {0,};
                 if (op_ret >= 0) {
-                        ia_time = buf->ia_atime;
-                        strftime (atime_buf, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
-
-                        ia_time = buf->ia_mtime;
-                        strftime (mtime_buf, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
-
-                        ia_time = buf->ia_ctime;
-                        strftime (ctime_buf, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
-
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d, *buf {"
-                                "ia_ino=%"PRIu64", st_mode=%o, ia_nlink=%"GF_PRI_NLINK", "
-                                "ia_uid=%d, ia_gid=%d, ia_rdev=%"PRIu64", "
-                                "ia_size=%"PRId64", ia_blksize=%"GF_PRI_BLKSIZE", "
-                                "ia_blocks=%"PRId64", ia_atime=%s, ia_mtime=%s, ia_ctime=%s})",
-                                frame->root->unique, op_ret, op_errno, buf->ia_ino,
-                                st_mode_from_ia (buf->ia_prot, buf->ia_type),
-                                buf->ia_nlink, buf->ia_uid, buf->ia_gid,
-                                buf->ia_rdev, buf->ia_size, buf->ia_blksize, buf->ia_blocks,
-                                atime_buf, mtime_buf, ctime_buf);
+                        trace_stat_to_str (buf, statstr);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d buf=%s",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  statstr);
                 } else {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
-                             buf, iobref);
+out:
+        TRACE_STACK_UNWIND (readv, frame, op_ret, op_errno, vector, count,
+                            buf, iobref, xdata);
         return 0;
 }
 
-
 int
 trace_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno,
-                  struct iatt *prebuf, struct iatt *postbuf)
+                  struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        char  *preopstr = NULL;
-        char  *postopstr = NULL;
+        char         preopstr[4096]  = {0, };
+        char         postopstr[4096] = {0, };
+        trace_conf_t *conf           = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_WRITE].enabled) {
+                char  string[4096] = {0,};
                 if (op_ret >= 0) {
-                        preopstr = trace_stat_to_str (prebuf);
-                        postopstr = trace_stat_to_str (postbuf);
-
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, ino = %"PRIu64
-                                ", *prebuf = {%s}, *postbuf = {%s})",
-                                frame->root->unique, op_ret, postbuf->ia_ino,
-                                preopstr, postopstr);
-
-                        if (preopstr)
-                                GF_FREE (preopstr);
-
-                        if (postopstr)
-                                GF_FREE (postopstr);
+                        trace_stat_to_str (prebuf, preopstr);
+                        trace_stat_to_str (postbuf, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s})",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
                 } else {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+out:
+        TRACE_STACK_UNWIND (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
         return 0;
 }
 
-
-
 int
 trace_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, gf_dirent_t *buf)
+                   int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
+                   dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READDIR].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64" :(op_ret=%d, op_errno=%d)",
-                        frame->root->unique, op_ret, op_errno);
-        }
+                char    string[4096]  = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique, uuid_utoa (frame->local),
+                          op_ret, op_errno);
 
-        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, buf);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (readdir, frame, op_ret, op_errno, buf, xdata);
 
         return 0;
 }
 
-
 int
 trace_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, gf_dirent_t *buf)
+                    int32_t op_ret, int32_t op_errno, gf_dirent_t *buf,
+                    dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READDIRP].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64" :(op_ret=%d, op_errno=%d)",
-                        frame->root->unique, op_ret, op_errno);
-        }
+                char        string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64" : gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique, uuid_utoa (frame->local),
+                          op_ret, op_errno);
 
-        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, buf);
+                LOG_ELEMENT (conf, string);
+        }
 
+out:
+        TRACE_STACK_UNWIND (readdirp, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
-
 int
 trace_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
-                 struct iatt *prebuf, struct iatt *postbuf)
+                 struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        char  *preopstr = NULL;
-        char  *postopstr = NULL;
-
-        if (trace_fop_names[GF_FOP_FSYNC].enabled) {
-                if (op_ret >= 0) {
-                        preopstr = trace_stat_to_str (prebuf);
-                        postopstr = trace_stat_to_str (postbuf);
-
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, ino = %"PRIu64
-                                ", *prebuf = {%s}, *postbuf = {%s}",
-                                frame->root->unique, op_ret, postbuf->ia_ino,
-                                preopstr, postopstr);
+        char          preopstr[4096]  = {0, };
+        char          postopstr[4096] = {0, };
+        trace_conf_t  *conf           = NULL;
 
-                        if (preopstr)
-                                GF_FREE (preopstr);
+        conf = this->private;
 
-                        if (postopstr)
-                                GF_FREE (postopstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FSYNC].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (prebuf, preopstr);
+                        trace_stat_to_str (postbuf, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s}",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
                 } else {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
+
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        STACK_UNWIND_STRICT (fsync, frame, op_ret, op_errno, prebuf, postbuf);
+out:
+        TRACE_STACK_UNWIND (fsync, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
 
         return 0;
 }
 
-
 int
 trace_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno,
-                   struct iatt *statpre, struct iatt *statpost)
+                   struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
 {
-        uint64_t ia_time         = 0;
-        char     atime_pre[256]  = {0,};
-        char     mtime_pre[256]  = {0,};
-        char     ctime_pre[256]  = {0,};
-        char     atime_post[256] = {0,};
-        char     mtime_post[256] = {0,};
-        char     ctime_post[256] = {0,};
-
-        if (trace_fop_names[GF_FOP_SETATTR].enabled) {
-                if (op_ret >= 0) {
-                        ia_time = statpre->ia_atime;
-                        strftime (atime_pre, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
+        char          preopstr[4096]  = {0, };
+        char          postopstr[4096] = {0, };
+        trace_conf_t  *conf           = NULL;
 
-                        ia_time = statpre->ia_mtime;
-                        strftime (mtime_pre, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
-
-                        ia_time = statpre->ia_ctime;
-                        strftime (ctime_pre, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
+        conf = this->private;
 
-                        ia_time = statpost->ia_atime;
-                        strftime (atime_post, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
-
-                        ia_time = statpost->ia_mtime;
-                        strftime (mtime_post, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
-
-                        ia_time = statpost->ia_ctime;
-                        strftime (ctime_post, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
-
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, *statpre "
-                                "{ia_ino=%"PRIu64", st_mode=%o, ia_uid=%d, "
-                                "ia_gid=%d, ia_atime=%s, ia_mtime=%s, "
-                                "ia_ctime=%s}, *statpost {ia_ino=%"PRIu64", "
-                                "st_mode=%o, ia_uid=%d, ia_gid=%d, ia_atime=%s,"
-                                " ia_mtime=%s, ia_ctime=%s})",
-                                frame->root->unique, op_ret, statpre->ia_ino,
-                                st_mode_from_ia (statpre->ia_prot, statpre->ia_type),
-                                statpre->ia_uid,
-                                statpre->ia_gid, atime_pre, mtime_pre,
-                                ctime_pre, statpost->ia_ino,
-                                st_mode_from_ia (statpost->ia_prot, statpost->ia_type),
-                                statpost->ia_uid, statpost->ia_gid, atime_post,
-                                mtime_post, ctime_post);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_SETATTR].enabled) {
+                char  string[4096]  = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (statpre, preopstr);
+                        trace_stat_to_str (statpost, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s})",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
                 } else {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno, statpre, statpost);
+out:
+        TRACE_STACK_UNWIND (setattr, frame, op_ret, op_errno, statpre,
+                            statpost, xdata);
         return 0;
 }
 
-
 int
 trace_fsetattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
-                    struct iatt *statpre, struct iatt *statpost)
+                    struct iatt *statpre, struct iatt *statpost, dict_t *xdata)
 {
-        uint64_t ia_time         = 0;
-        char     atime_pre[256]  = {0,};
-        char     mtime_pre[256]  = {0,};
-        char     ctime_pre[256]  = {0,};
-        char     atime_post[256] = {0,};
-        char     mtime_post[256] = {0,};
-        char     ctime_post[256] = {0,};
+        char          preopstr[4096]  = {0, };
+        char          postopstr[4096] = {0, };
+        trace_conf_t  *conf           = NULL;
 
-        if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
-                if (op_ret >= 0) {
-                        ia_time = statpre->ia_atime;
-                        strftime (atime_pre, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
-
-                        ia_time = statpre->ia_mtime; 
-                        strftime (mtime_pre, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
-
-                        ia_time = statpre->ia_ctime;
-                        strftime (ctime_pre, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
-
-                        ia_time = statpost->ia_atime;
-                        strftime (atime_post, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
-
-                        ia_time = statpost->ia_mtime;
-                        strftime (mtime_post, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
+        conf = this->private;
 
-                        ia_time = statpost->ia_ctime;
-                        strftime (ctime_post, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
-
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, *statpre "
-                                "{ia_ino=%"PRIu64", st_mode=%o, ia_uid=%d, "
-                                "ia_gid=%d, ia_atime=%s, ia_mtime=%s, "
-                                "ia_ctime=%s}, *statpost {ia_ino=%"PRIu64", "
-                                "st_mode=%o, ia_uid=%d, ia_gid=%d, ia_atime=%s,"
-                                " ia_mtime=%s, ia_ctime=%s})",
-                                frame->root->unique, op_ret, statpre->ia_ino,
-                                st_mode_from_ia (statpre->ia_prot, statpre->ia_type),
-                                statpre->ia_uid,
-                                statpre->ia_gid, atime_pre, mtime_pre,
-                                ctime_pre, statpost->ia_ino,
-                                st_mode_from_ia (statpost->ia_prot, statpost->ia_type),
-                                statpost->ia_uid, statpost->ia_gid, atime_post,
-                                mtime_post, ctime_post);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (statpre, preopstr);
+                        trace_stat_to_str (statpost, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s})",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
                 } else {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, op_errno=%d)",
+                                  frame->root->unique, uuid_utoa (frame->local),
+                                  op_ret, op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        STACK_UNWIND_STRICT (fsetattr, frame, op_ret, op_errno,
-                             statpre, statpost);
+out:
+        TRACE_STACK_UNWIND (fsetattr, frame, op_ret, op_errno,
+                            statpre, statpost, xdata);
         return 0;
 }
 
-
 int
 trace_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno,
-                  struct iatt *preparent, struct iatt *postparent)
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
 {
-        char *preparentstr = NULL;
-        char *postparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_UNLINK].enabled) {
-                if (op_ret >= 0) {
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, *preparent = {%s}, "
-                                "*postparent = {%s})",
-                                frame->root->unique, op_ret, preparentstr,
-                                postparentstr);
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
+        conf = this->private;
 
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_UNLINK].enabled) {
+                char string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  " *preparent = {%s}, "
+                                  "*postparent = {%s})",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, preparentstr,
+                                  postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (unlink, frame, op_ret, op_errno,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
                   struct iatt *preoldparent, struct iatt *postoldparent,
-                  struct iatt *prenewparent, struct iatt *postnewparent)
+                  struct iatt *prenewparent, struct iatt *postnewparent,
+                  dict_t *xdata)
 {
-        char  *statstr = NULL;
-        char  *preoldparentstr = NULL;
-        char  *postoldparentstr = NULL;
-        char  *prenewparentstr = NULL;
-        char  *postnewparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_RENAME].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preoldparentstr = trace_stat_to_str (preoldparent);
-                        postoldparentstr = trace_stat_to_str (postoldparent);
-
-                        prenewparentstr = trace_stat_to_str (prenewparent);
-                        postnewparentstr = trace_stat_to_str (postnewparent);
-
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, *stbuf = {%s}, "
-                                "*preoldparent = {%s}, *postoldparent = {%s}"
-                                " *prenewparent = {%s}, *postnewparent = {%s})",
-                                frame->root->unique, op_ret, statstr,
-                                preoldparentstr, postoldparentstr,
-                                prenewparentstr, postnewparentstr);
-
-                        if (preoldparentstr)
-                                GF_FREE (preoldparentstr);
+        char           statstr[4096]          = {0, };
+        char           preoldparentstr[4096]  = {0, };
+        char           postoldparentstr[4096] = {0, };
+        char           prenewparentstr[4096]  = {0, };
+        char           postnewparentstr[4096] = {0, };
+        trace_conf_t   *conf                  = NULL;
 
-                        if (postoldparentstr)
-                                GF_FREE (postoldparentstr);
+        conf = this->private;
 
-                        if (prenewparentstr)
-                                GF_FREE (prenewparentstr);
-
-                        if (postnewparentstr)
-                                GF_FREE (postnewparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_RENAME].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preoldparent, preoldparentstr);
+                        trace_stat_to_str (postoldparent, postoldparentstr);
+                        trace_stat_to_str (prenewparent, prenewparentstr);
+                        trace_stat_to_str (postnewparent, postnewparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*stbuf = {%s}, *preoldparent = {%s},"
+                                  " *postoldparent = {%s}"
+                                  " *prenewparent = {%s}, "
+                                  "*postnewparent = {%s})",
+                                  frame->root->unique, op_ret, statstr,
+                                  preoldparentstr, postoldparentstr,
+                                  prenewparentstr, postnewparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, op_errno);
+
                 }
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (op_ret=%d, op_errno=%d, buf {ia_ino=%"PRIu64"})",
-                        frame->root->unique, op_ret, op_errno,
-                        (buf? buf->ia_ino : 0));
+                LOG_ELEMENT (conf, string);
         }
-
-        STACK_UNWIND_STRICT (rename, frame, op_ret, op_errno, buf,
-                             preoldparent, postoldparent,
-                             prenewparent, postnewparent);
+out:
+        TRACE_STACK_UNWIND (rename, frame, op_ret, op_errno, buf,
+                            preoldparent, postoldparent,
+                            prenewparent, postnewparent, xdata);
         return 0;
 }
 
-
 int
 trace_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
-                    const char *buf, struct iatt *stbuf)
+                    const char *buf, struct iatt *stbuf, dict_t *xdata)
 {
-        char *statstr = NULL;
+        char          statstr[4096] = {0, };
+        trace_conf_t  *conf         = NULL;
 
-        if (trace_fop_names[GF_FOP_READLINK].enabled) {
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_READLINK].enabled) {
+                char string[4096] = {0,};
                 if (op_ret == 0) {
-                        statstr = trace_stat_to_str (stbuf);
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d, buf=%s, "
-                                "stbuf = { %s })",
-                                frame->root->unique, op_ret, op_errno, buf,
-                                statstr);
-                } else
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d",
-                                frame->root->unique, op_ret, op_errno);
+                        trace_stat_to_str (stbuf, statstr);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, op_errno=%d,"
+                                  "buf=%s, stbuf = { %s })",
+                                  frame->root->unique, op_ret, op_errno,
+                                  buf, statstr);
+                } else {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
+                }
 
-                if (statstr)
-                        GF_FREE (statstr);
+                LOG_ELEMENT (conf, string);
         }
-
-        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, buf, stbuf);
+out:
+        TRACE_STACK_UNWIND (readlink, frame, op_ret, op_errno, buf, stbuf,
+                            xdata);
         return 0;
 }
 
-
 int
 trace_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                   int32_t op_ret, int32_t op_errno,
                   inode_t *inode, struct iatt *buf,
-                  dict_t *xattr, struct iatt *postparent)
+                  dict_t *xdata, struct iatt *postparent)
 {
-        char  *statstr = NULL;
-        char  *postparentstr = NULL;
+        char          statstr[4096]       = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, ino=%"PRIu64", "
-                                "*buf {%s}, *postparent {%s}",
-                                frame->root->unique, op_ret, inode->ino,
-                                statstr, postparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (postparent, postparentstr);
+                        /* print buf->ia_gfid instead of inode->gfid,
+                         * since if the inode is not yet linked to the
+                         * inode table (fresh lookup) then null gfid
+                         * will be printed.
+                         */
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d "
+                                  "*buf {%s}, *postparent {%s}",
+                                  frame->root->unique,
+                                  uuid_utoa (buf->ia_gfid),
+                                  op_ret, statstr, postparentstr);
+
+                        /* For 'forget' */
+                        inode_ctx_put (inode, this, 0);
                 } else {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
-                             xattr, postparent);
+out:
+        TRACE_STACK_UNWIND (lookup, frame, op_ret, op_errno, inode, buf,
+                            xdata, postparent);
         return 0;
 }
 
-
 int
 trace_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                    int32_t op_ret, int32_t op_errno,
                    inode_t *inode, struct iatt *buf,
-                   struct iatt *preparent, struct iatt *postparent)
+                   struct iatt *preparent, struct iatt *postparent,
+                   dict_t *xdata)
 {
-        char  *statstr = NULL;
-        char  *preparentstr = NULL;
-        char  *postparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, ino=%"PRIu64", "
-                                "*stbuf = {%s}, *preparent = {%s}, "
-                                "*postparent = {%s})",
-                                frame->root->unique, op_ret, inode->ino,
-                                statstr, preparentstr, postparentstr);
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-                        if (statstr)
-                                GF_FREE (statstr);
-
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
-
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d "
+                                  "*stbuf = {%s}, *preparent = {%s}, "
+                                  "*postparent = {%s})",
+                                  frame->root->unique,
+                                  uuid_utoa (inode->gfid),
+                                  op_ret, statstr, preparentstr,
+                                  postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": op_ret=%d, op_errno=%d",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (symlink, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_mknod_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
                  inode_t *inode, struct iatt *buf,
-                 struct iatt *preparent, struct iatt *postparent)
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        char *statstr = NULL;
-        char *preparentstr = NULL;
-        char *postparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_MKNOD].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, ino=%"PRIu64", "
-                                "*stbuf = {%s}, *preparent = {%s}, "
-                                "*postparent = {%s})",
-                                frame->root->unique, op_ret, inode->ino,
-                                statstr, preparentstr, postparentstr);
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-                        if (statstr)
-                                GF_FREE (statstr);
+        conf = this->private;
 
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
-
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        char string[4096]  = {0,};
+        if (trace_fop_names[GF_FOP_MKNOD].enabled) {
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d "
+                                  "*stbuf = {%s}, *preparent = {%s}, "
+                                  "*postparent = {%s})",
+                                  frame->root->unique,
+                                  uuid_utoa (inode->gfid),
+                                  op_ret, statstr, preparentstr,
+                                  postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        STACK_UNWIND_STRICT (mknod, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (mknod, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
                  inode_t *inode, struct iatt *buf,
-                 struct iatt *preparent, struct iatt *postparent)
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        char  *statstr = NULL;
-        char  *preparentstr = NULL;
-        char  *postparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_MKDIR].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, ino = %"PRIu64
-                                ", *stbuf = {%s}, *prebuf = {%s}, "
-                                "*postbuf = {%s} )",
-                                frame->root->unique, op_ret, buf->ia_ino,
-                                statstr, preparentstr, postparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
+        conf = this->private;
 
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_MKDIR].enabled) {
+                char  string[4096]  = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s (op_ret=%d "
+                                  ", *stbuf = {%s}, *prebuf = {%s}, "
+                                  "*postbuf = {%s} )",
+                                  frame->root->unique,
+                                  uuid_utoa (inode->gfid),
+                                  op_ret, statstr, preparentstr,
+                                  postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, op_errno=%d)",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (mkdir, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                 int32_t op_ret, int32_t op_errno,
                 inode_t *inode, struct iatt *buf,
-                struct iatt *preparent, struct iatt *postparent)
+                struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        char  *statstr = NULL;
-        char  *preparentstr = NULL;
-        char  *postparentstr = NULL;
+        char          statstr[4096]       = {0, };
+        char          preparentstr[4096]  = {0, };
+        char          postparentstr[4096] = {0, };
+        trace_conf_t  *conf               = NULL;
 
-        if (trace_fop_names[GF_FOP_LINK].enabled) {
-                if (op_ret >= 0) {
-                        statstr = trace_stat_to_str (buf);
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
+        conf = this->private;
 
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, ino = %"PRIu64
-                                ", *stbuf = {%s}, *prebuf = {%s}, "
-                                "*postbuf = {%s})",
-                                frame->root->unique, op_ret, buf->ia_ino,
-                                statstr, preparentstr, postparentstr);
-
-                        if (statstr)
-                                GF_FREE (statstr);
-
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
-
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        char  string[4096]  = {0,};
+        if (trace_fop_names[GF_FOP_LINK].enabled) {
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*stbuf = {%s},  *prebuf = {%s},"
+                                  " *postbuf = {%s})",
+                                  frame->root->unique, op_ret,
+                                  statstr, preparentstr, postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (link, frame, op_ret, op_errno, inode, buf,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno)
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        char      string[4096] = {0,};
         if (trace_fop_names[GF_FOP_FLUSH].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (op_ret=%d, op_errno=%d)",
-                        frame->root->unique, op_ret, op_errno);
-        }
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique, uuid_utoa (frame->local),
+                          op_ret, op_errno);
 
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (flush, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, fd_t *fd)
+                   int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        char    string[4096] = {0,};
         if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (op_ret=%d, op_errno=%d, fd=%p)",
-                        frame->root->unique, op_ret, op_errno, fd);
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
+                          " fd=%p",
+                          frame->root->unique, uuid_utoa (frame->local),
+                          op_ret, op_errno, fd);
+
+                LOG_ELEMENT (conf, string);
         }
+out:
+        /* for 'releasedir' log */
+        if (op_ret >= 0)
+                fd_ctx_set (fd, this, 0);
 
-        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd);
+        TRACE_STACK_UNWIND (opendir, frame, op_ret, op_errno, fd, xdata);
         return 0;
 }
 
-
 int
 trace_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno,
-                 struct iatt *preparent, struct iatt *postparent)
+                 struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
-        char  *preparentstr = NULL;
-        char  *postparentstr = NULL;
-
-        if (trace_fop_names[GF_FOP_RMDIR].enabled) {
-                if (op_ret >= 0) {
-                        preparentstr = trace_stat_to_str (preparent);
-                        postparentstr = trace_stat_to_str (postparent);
-
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s}",
-                                frame->root->unique, op_ret, preparentstr,
-                                postparentstr);
+        char           preparentstr[4096]  = {0, };
+        char           postparentstr[4096] = {0, };
+        trace_conf_t   *conf               = NULL;
 
-                        if (preparentstr)
-                                GF_FREE (preparentstr);
+        conf = this->private;
 
-                        if (postparentstr)
-                                GF_FREE (postparentstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_RMDIR].enabled) {
+                char  string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (preparent, preparentstr);
+                        trace_stat_to_str (postparent, postparentstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "*prebuf={%s},  *postbuf={%s}",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, preparentstr, postparentstr);
                 } else {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
-                             preparent, postparent);
+out:
+        TRACE_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
+                            preparent, postparent, xdata);
         return 0;
 }
 
-
 int
 trace_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                     int32_t op_ret, int32_t op_errno,
-                    struct iatt *prebuf, struct iatt *postbuf)
+                    struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        char  *preopstr = NULL;
-        char  *postopstr = NULL;
-
-        if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
-                if (op_ret >= 0) {
-                        preopstr = trace_stat_to_str (prebuf);
-                        postopstr = trace_stat_to_str (postbuf);
-
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s} )",
-                                frame->root->unique, op_ret, preopstr,
-                                postopstr);
+        char           preopstr[4096]  = {0, };
+        char           postopstr[4096] = {0, };
+        trace_conf_t   *conf           = NULL;
 
-                        if (preopstr)
-                                GF_FREE (preopstr);
+        conf = this->private;
 
-                        if (postopstr)
-                                GF_FREE (postopstr);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
+                char   string[4096] = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (prebuf, preopstr);
+                        trace_stat_to_str (postbuf, postopstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s} )",
+                                  frame->root->unique, op_ret,
+                                  preopstr, postopstr);
                 } else {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, prebuf, postbuf);
+out:
+        TRACE_STACK_UNWIND (truncate, frame, op_ret, op_errno, prebuf,
+                            postbuf, xdata);
         return 0;
 }
 
-
 int
 trace_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno, struct statvfs *buf)
+                  int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+                  dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_STATFS].enabled) {
-                if (op_ret >= 0) {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": ({f_bsize=%lu, f_frsize=%lu, f_blocks=%"GF_PRI_FSBLK
-                                ", f_bfree=%"GF_PRI_FSBLK", f_bavail=%"GF_PRI_FSBLK", "
-                                "f_files=%"GF_PRI_FSBLK", f_ffree=%"GF_PRI_FSBLK", f_favail=%"
-                                GF_PRI_FSBLK", f_fsid=%lu, f_flag=%lu, f_namemax=%lu}) => ret=%d",
-                                frame->root->unique, buf->f_bsize, buf->f_frsize, buf->f_blocks,
-                                buf->f_bfree, buf->f_bavail, buf->f_files, buf->f_ffree,
-                                buf->f_favail, buf->f_fsid, buf->f_flag, buf->f_namemax, op_ret);
+                char   string[4096] = {0,};
+                if (op_ret == 0) {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": ({f_bsize=%lu, "
+                                  "f_frsize=%lu, "
+                                  "f_blocks=%"GF_PRI_FSBLK
+                                  ", f_bfree=%"GF_PRI_FSBLK", "
+                                  "f_bavail=%"GF_PRI_FSBLK", "
+                                  "f_files=%"GF_PRI_FSBLK", "
+                                  "f_ffree=%"GF_PRI_FSBLK", "
+                                  "f_favail=%"GF_PRI_FSBLK", "
+                                  "f_fsid=%lu, f_flag=%lu, "
+                                  "f_namemax=%lu}) => ret=%d",
+                                  frame->root->unique, buf->f_bsize,
+                                  buf->f_frsize, buf->f_blocks,
+                                  buf->f_bfree, buf->f_bavail,
+                                  buf->f_files, buf->f_ffree,
+                                  buf->f_favail, buf->f_fsid,
+                                  buf->f_flag, buf->f_namemax, op_ret);
                 } else {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": (op_ret=%d, "
+                                  "op_errno=%d)",
+                                  frame->root->unique, op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf);
+out:
+        TRACE_STACK_UNWIND (statfs, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
-
 int
 trace_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t     *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (op_ret=%d, op_errno=%d)",
-                        frame->root->unique, op_ret, op_errno);
-        }
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret,
+                          op_errno);
 
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *dict)
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (op_ret=%d, op_errno=%d, dict=%p)",
-                        frame->root->unique, op_ret, op_errno, dict);
+                char       string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
+                          " dict=%p", frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno,
+                          dict);
+
+                LOG_ELEMENT (conf, string);
         }
+out:
+        TRACE_STACK_UNWIND (getxattr, frame, op_ret, op_errno, dict, xdata);
 
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
+        return 0;
+}
+
+int
+trace_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
 
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
+        if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
+                char    string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
+int
+trace_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
+{
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
+                char      string[4096]  = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d,"
+                          " dict=%p", frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno,
+                          dict);
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fgetxattr, frame, op_ret, op_errno, dict, xdata);
+
+        return 0;
+}
 
 int
 trace_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno)
+                       int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (op_ret=%d, op_errno=%d)",
-                        frame->root->unique, op_ret, op_errno);
-        }
+                char       string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
 
         return 0;
 }
 
-
 int
 trace_fsyncdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t     *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (op_ret=%d, op_errno=%d)",
-                        frame->root->unique, op_ret, op_errno);
-        }
+                char       string[4096]   =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        STACK_UNWIND_STRICT (fsyncdir, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fsyncdir, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_access_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                  int32_t op_ret, int32_t op_errno)
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_ACCESS].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (op_ret=%d, op_errno=%d)",
-                        frame->root->unique, op_ret, op_errno);
-        }
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, "
+                          "op_errno=%d)", frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        STACK_UNWIND_STRICT (access, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (access, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
-
 int
 trace_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                      int32_t op_ret, int32_t op_errno,
-                     struct iatt *prebuf, struct iatt *postbuf)
+                     struct iatt *prebuf, struct iatt *postbuf, dict_t *xdata)
 {
-        char  *prebufstr = NULL;
-        char  *postbufstr = NULL;
-
-        if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
-                if (op_ret >= 0) {
-                        prebufstr = trace_stat_to_str (prebuf);
-                        postbufstr = trace_stat_to_str (postbuf);
-
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, *prebuf = {%s}, "
-                                "*postbuf = {%s} )",
-                                frame->root->unique, op_ret,
-                                prebufstr, postbufstr);
+        char          prebufstr[4096]  = {0, };
+        char          postbufstr[4096] = {0, };
+        trace_conf_t  *conf            = NULL;
 
-                        if (prebufstr)
-                                GF_FREE (prebufstr);
-
-                        if (postbufstr)
-                                GF_FREE (postbufstr);
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
+                char  string[4096]  = {0,};
+                if (op_ret == 0) {
+                        trace_stat_to_str (prebuf, prebufstr);
+                        trace_stat_to_str (postbuf, postbufstr);
+
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": op_ret=%d, "
+                                  "*prebuf = {%s}, *postbuf = {%s} )",
+                                  frame->root->unique, op_ret,
+                                  prebufstr, postbufstr);
                 } else {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        STACK_UNWIND_STRICT (ftruncate, frame, op_ret, op_errno, prebuf, postbuf);
+out:
+        TRACE_STACK_UNWIND (ftruncate, frame, op_ret, op_errno, prebuf, postbuf,
+                            xdata);
         return 0;
 }
 
-
 int
 trace_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                 int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
 {
-        uint64_t ia_time        = 0;
-        char     atime_buf[256] = {0, };
-        char     mtime_buf[256] = {0, };
-        char     ctime_buf[256] = {0, };
+        char          statstr[4096] = {0, };
+        trace_conf_t  *conf         = NULL;
 
-        if (trace_fop_names[GF_FOP_FSTAT].enabled) {
-                if (op_ret >= 0) {
-                        ia_time = buf->ia_atime;
-                        strftime (atime_buf, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
-
-                        ia_time = buf->ia_mtime;
-                        strftime (mtime_buf, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
+        conf = this->private;
 
-                        ia_time = buf->ia_ctime;
-                        strftime (ctime_buf, 256, "[%b %d %H:%M:%S]",
-                                  localtime ((time_t *)&ia_time));
-
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, *buf {"
-                                "ia_ino=%"PRIu64", st_mode=%o, ia_nlink=%"GF_PRI_NLINK", "
-                                "ia_uid=%d, ia_gid=%d, ia_rdev=%"PRIu64", ia_size=%"PRId64", "
-                                "ia_blksize=%"GF_PRI_BLKSIZE", ia_blocks=%"PRId64", ia_atime=%s, "
-                                "ia_mtime=%s, ia_ctime=%s})",
-                                frame->root->unique, op_ret, buf->ia_ino,
-                                st_mode_from_ia (buf->ia_prot, buf->ia_type),
-                                buf->ia_nlink, buf->ia_uid, buf->ia_gid,
-                                buf->ia_rdev, buf->ia_size, buf->ia_blksize,
-                                buf->ia_blocks, atime_buf, mtime_buf, ctime_buf);
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FSTAT].enabled) {
+                char string[4096]  = {0.};
+                if (op_ret == 0) {
+                        trace_stat_to_str (buf, statstr);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d "
+                                  "buf=%s", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  statstr);
                 } else {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
+                LOG_ELEMENT (conf, string);
         }
-
-        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf);
+out:
+        TRACE_STACK_UNWIND (fstat, frame, op_ret, op_errno, buf, xdata);
         return 0;
 }
 
-
 int
 trace_lk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno, struct gf_flock *lock)
+              int32_t op_ret, int32_t op_errno, struct gf_flock *lock,
+              dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_LK].enabled) {
-                if (op_ret >= 0) {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, {l_type=%d, l_whence=%d, "
-                                "l_start=%"PRId64", l_len=%"PRId64", l_pid=%u})",
-                                frame->root->unique, op_ret, lock->l_type, lock->l_whence,
-                                lock->l_start, lock->l_len, lock->l_pid);
+                char      string[4096] = {0,};
+                if (op_ret == 0) {
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "{l_type=%d, l_whence=%d, "
+                                  "l_start=%"PRId64", "
+                                  "l_len=%"PRId64", l_pid=%u})",
+                                  frame->root->unique,
+                                  uuid_utoa (frame->local),
+                                  op_ret, lock->l_type, lock->l_whence,
+                                  lock->l_start, lock->l_len,
+                                  lock->l_pid);
                 } else {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (op_ret=%d, op_errno=%d)",
-                                frame->root->unique, op_ret, op_errno);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s op_ret=%d, "
+                                  "op_errno=%d)", frame->root->unique,
+                                  uuid_utoa (frame->local), op_ret,
+                                  op_errno);
                 }
-        }
 
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, lock);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (lk, frame, op_ret, op_errno, lock, xdata);
         return 0;
 }
 
-
-
 int
 trace_entrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": op_ret=%d, op_errno=%d",
-                        frame->root->unique, op_ret, op_errno);
-        }
+                char   string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (entrylk, frame, op_ret, op_errno, xdata);
         return 0;
 }
 
+int
+trace_fentrylk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fentrylk, frame, op_ret, op_errno, xdata);
+        return 0;
+}
 
 int
 trace_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, dict_t *dict)
+                   int32_t op_ret, int32_t op_errno, dict_t *dict,
+                   dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_XATTROP].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (op_ret=%d, op_errno=%d)",
-                        frame->root->unique, op_ret, op_errno);
-        }
+                char    string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        STACK_UNWIND_STRICT (xattrop, frame, op_ret, op_errno, dict);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (xattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
-
 int
 trace_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, dict_t *dict)
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (op_ret=%d, op_errno=%d)",
-                        frame->root->unique, op_ret, op_errno);
-        }
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
 
-        STACK_UNWIND_STRICT (fxattrop, frame, op_ret, op_errno, dict);
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, dict, xdata);
         return 0;
 }
 
-
 int
 trace_inodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno)
+                   int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_INODELK].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": op_ret=%d, op_errno=%d",
-                        frame->root->unique, op_ret, op_errno);
+                char       string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local),op_ret, op_errno);
+
+                LOG_ELEMENT (conf, string);
         }
+out:
+        TRACE_STACK_UNWIND (inodelk, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int
+trace_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FINODELK].enabled) {
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d, op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
+        TRACE_STACK_UNWIND (finodelk, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int
+trace_rchecksum_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno,
+                     uint32_t weak_checksum, uint8_t *strong_checksum,
+                     dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
+        if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s op_ret=%d op_errno=%d",
+                          frame->root->unique,
+                          uuid_utoa (frame->local), op_ret, op_errno);
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        TRACE_STACK_UNWIND (rchecksum, frame, op_ret, op_errno, weak_checksum,
+                            strong_checksum, xdata);
 
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno);
         return 0;
 }
 
+/* *_cbk section over <----------> fop section start */
 
 int
 trace_entrylk (call_frame_t *frame, xlator_t *this,
                const char *volume, loc_t *loc, const char *basename,
-               entrylk_cmd cmd, entrylk_type type)
+               entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_ENTRYLK].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": volume=%s, (loc= {path=%s, ino=%"PRIu64"} basename=%s, cmd=%s, type=%s)",
-                        frame->root->unique, volume, loc->path, loc->inode->ino, basename,
-                        ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" : "ENTRYLK_UNLOCK"),
-                        ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" : "ENTRYLK_WRLCK"));
+                char     string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s volume=%s, (path=%s "
+                          "basename=%s, cmd=%s, type=%s)",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid),
+                          volume, loc->path, basename,
+                          ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" :
+                           "ENTRYLK_UNLOCK"),
+                          ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" :
+                           "ENTRYLK_WRLCK"));
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_entrylk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->entrylk,
-                    volume, loc, basename, cmd, type);
+                    volume, loc, basename, cmd, type, xdata);
         return 0;
 }
 
-
 int
 trace_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-               loc_t *loc, int32_t cmd, struct gf_flock *flock)
+               loc_t *loc, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        char *cmd_str = NULL;
-        char *type_str = NULL;
+        char         *cmd_str  = NULL;
+        char         *type_str = NULL;
+        trace_conf_t *conf     = NULL;
 
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_INODELK].enabled) {
+                char string[4096]  = {0,};
                 switch (cmd) {
 #if F_GETLK != F_GETLK64
                 case F_GETLK64:
@@ -1274,45 +1537,43 @@ trace_inodelk (call_frame_t *frame, xlator_t *this, const char *volume,
                         break;
                 }
 
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": volume=%s, (loc {path=%s, ino=%"PRIu64"}, "
-                        "cmd=%s, type=%s, start=%llu, len=%llu, pid=%llu)",
-                        frame->root->unique, volume, loc->path, loc->inode->ino,
-                        cmd_str, type_str, (unsigned long long) flock->l_start,
-                        (unsigned long long) flock->l_len,
-                        (unsigned long long) flock->l_pid);
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s volume=%s, (path=%s "
+                          "cmd=%s, type=%s, start=%llu, len=%llu, "
+                          "pid=%llu)", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), volume,
+                          loc->path, cmd_str, type_str,
+                          (unsigned long long)flock->l_start,
+                          (unsigned long long) flock->l_len,
+                          (unsigned long long) flock->l_pid);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_inodelk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->inodelk,
-                    volume, loc, cmd, flock);
+                    volume, loc, cmd, flock, xdata);
         return 0;
 }
 
-
-int
-trace_finodelk_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno)
-{
-        if (trace_fop_names[GF_FOP_FINODELK].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": op_ret=%d, op_errno=%d",
-                        frame->root->unique, op_ret, op_errno);
-        }
-
-        STACK_UNWIND_STRICT (finodelk, frame, op_ret, op_errno);
-        return 0;
-}
-
-
 int
 trace_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
-                fd_t *fd, int32_t cmd, struct gf_flock *flock)
+                fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        char *cmd_str = NULL, *type_str = NULL;
+        char          *cmd_str  = NULL;
+        char          *type_str = NULL;
+        trace_conf_t  *conf     = NULL;
 
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FINODELK].enabled) {
+                char  string[4096] = {0,};
                 switch (cmd) {
 #if F_GETLK != F_GETLK64
                 case F_GETLK64:
@@ -1355,275 +1616,456 @@ trace_finodelk (call_frame_t *frame, xlator_t *this, const char *volume,
                         break;
                 }
 
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": volume=%s, (fd =%p, ino=%"PRIu64"}, "
-                        "cmd=%s, type=%s, start=%llu, len=%llu, pid=%llu)",
-                        frame->root->unique, volume, fd, fd->inode->ino,
-                        cmd_str, type_str, (unsigned long long) flock->l_start,
-                        (unsigned long long) flock->l_len,
-                        (unsigned long long) flock->l_pid);
-        }
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s volume=%s, (fd =%p "
+                          "cmd=%s, type=%s, start=%llu, len=%llu, "
+                          "pid=%llu)", frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), volume, fd,
+                          cmd_str, type_str,
+                          (unsigned long long) flock->l_start,
+                          (unsigned long long) flock->l_len,
+                          (unsigned long long) flock->l_pid);
 
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+out:
         STACK_WIND (frame, trace_finodelk_cbk,
                     FIRST_CHILD (this),
                     FIRST_CHILD (this)->fops->finodelk,
-                    volume, fd, cmd, flock);
+                    volume, fd, cmd, flock, xdata);
         return 0;
 }
 
-
 int
 trace_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               gf_xattrop_flags_t flags, dict_t *dict)
+               gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_XATTROP].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (path=%s, ino=%"PRIu64" flags=%d)",
-                        frame->root->unique, loc->path, loc->inode->ino, flags);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s (path=%s flags=%d)",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          flags);
 
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_xattrop_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->xattrop,
-                    loc, flags, dict);
+                    loc, flags, dict, xdata);
 
         return 0;
 }
 
-
 int
 trace_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                gf_xattrop_flags_t flags, dict_t *dict)
+                gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FXATTROP].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (fd=%p, flags=%d)",
-                        frame->root->unique, fd, flags);
+                char    string[4096]  = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, flags=%d",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, flags);
+
+                frame->local = fd->inode->gfid;
 
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fxattrop_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fxattrop,
-                    fd, flags, dict);
+                    fd, flags, dict, xdata);
 
         return 0;
 }
 
-
 int
 trace_lookup (call_frame_t *frame, xlator_t *this,
-              loc_t *loc, dict_t *xattr_req)
+              loc_t *loc, dict_t *xdata)
 {
+        trace_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
+                char string[4096] = {0,};
                 /* TODO: print all the keys mentioned in xattr_req */
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (loc {path=%s, ino=%"PRIu64"})",
-                        frame->root->unique, loc->path,
-                        loc->inode->ino);
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_lookup_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->lookup,
-                    loc, xattr_req);
+                    loc, xdata);
 
         return 0;
 }
 
-
 int
-trace_stat (call_frame_t *frame, xlator_t *this, loc_t *loc)
+trace_stat (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
         if (trace_fop_names[GF_FOP_STAT].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (loc {path=%s, ino=%"PRIu64"})",
-                        frame->root->unique, loc->path, loc->inode->ino);
+                char  string[4096]  = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_stat_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->stat,
-                    loc);
+                    loc, xdata);
 
         return 0;
 }
 
-
 int
-trace_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
+trace_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size,
+                dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READLINK].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (loc {path=%s, ino=%"PRIu64"}, size=%"GF_PRI_SIZET")",
-                        frame->root->unique, loc->path, loc->inode->ino, size);
+                char      string[4096] = {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s, "
+                          "size=%"GF_PRI_SIZET")", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          size);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_readlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readlink,
-                    loc, size);
+                    loc, size, xdata);
 
         return 0;
 }
 
-
 int
 trace_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc,
-             mode_t mode, dev_t dev, dict_t *params)
+             mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
         if (trace_fop_names[GF_FOP_MKNOD].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (loc {path=%s, ino=%"PRIu64"}, mode=%d, dev=%"GF_PRI_DEV")",
-                        frame->root->unique, loc->path, loc->inode->ino, mode, dev);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s mode=%d "
+                          "umask=0%o, dev=%"GF_PRI_DEV")",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          mode, umask, dev);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_mknod_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->mknod,
-                    loc, mode, dev, params);
+                    loc, mode, dev, umask, xdata);
 
         return 0;
 }
 
-
 int
 trace_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-             dict_t *params)
+             mode_t umask, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_MKDIR].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (path=%s, ino=%"PRIu64", mode=%d)",
-                        frame->root->unique, loc->path,
-                        ((loc->inode)? loc->inode->ino : 0), mode);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s mode=%d"
+                          " umask=0%o", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          mode, umask);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_mkdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->mkdir,
-                    loc, mode, params);
+                    loc, mode, umask, xdata);
         return 0;
 }
 
-
 int
-trace_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
+trace_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+              dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_UNLINK].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (loc {path=%s, ino=%"PRIu64"})",
-                        frame->root->unique, loc->path, loc->inode->ino);
-        }
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s flag=%d",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          xflag);
+
+                frame->local = loc->inode->gfid;
 
+                LOG_ELEMENT (conf, string);
+        }
+out:
         STACK_WIND (frame, trace_unlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->unlink,
-                    loc);
+                    loc, xflag, xdata);
         return 0;
 }
 
-
 int
-trace_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
+trace_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+             dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_RMDIR].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (loc {path=%s, ino=%"PRIu64"}, flags=%d)",
-                        frame->root->unique, loc->path, loc->inode->ino, flags);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s flags=%d",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          flags);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_rmdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->rmdir,
-                    loc, flags);
+                    loc, flags, xdata);
 
         return 0;
 }
 
-
 int
 trace_symlink (call_frame_t *frame, xlator_t *this, const char *linkpath,
-               loc_t *loc, dict_t *params)
+               loc_t *loc, mode_t umask, dict_t *xdata)
 {
+        trace_conf_t     *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_SYMLINK].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (linkpath=%s, loc {path=%s, ino=%"PRIu64"})",
-                        frame->root->unique, linkpath, loc->path,
-                        ((loc->inode)? loc->inode->ino : 0));
+                char     string[4096]   =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s linkpath=%s, path=%s"
+                          " umask=0%o", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), linkpath,
+                          loc->path, umask);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_symlink_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->symlink,
-                    linkpath, loc, params);
+                    linkpath, loc, umask, xdata);
 
         return 0;
 }
 
-
 int
-trace_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
+trace_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+              dict_t *xdata)
 {
+        char         oldgfid[50] = {0,};
+        char         newgfid[50] = {0,};
+        trace_conf_t *conf       = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_RENAME].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (oldloc {path=%s, ino=%"PRIu64"}, "
-                        "newloc{path=%s, ino=%"PRIu64"})",
-                        frame->root->unique, oldloc->path, oldloc->ino,
-                        newloc->path, newloc->ino);
+                char string[4096] = {0,};
+                if (newloc->inode)
+                        uuid_utoa_r (newloc->inode->gfid, newgfid);
+                else
+                        strcpy (newgfid, "0");
+
+                uuid_utoa_r (oldloc->inode->gfid, oldgfid);
+
+                snprintf (string, sizeof (string),
+                          "%"PRId64": oldgfid=%s oldpath=%s --> "
+                          "newgfid=%s newpath=%s",
+                          frame->root->unique, oldgfid,
+                          oldloc->path, newgfid, newloc->path);
+
+                frame->local = oldloc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_rename_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->rename,
-                    oldloc, newloc);
+                    oldloc, newloc, xdata);
 
         return 0;
 }
 
-
 int
-trace_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
+trace_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+            dict_t *xdata)
 {
+        char         oldgfid[50] = {0,};
+        char         newgfid[50] = {0,};
+        trace_conf_t *conf       = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_LINK].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (oldloc {path=%s, ino=%"PRIu64"}, "
-                        "newloc {path=%s, ino=%"PRIu64"})",
-                        frame->root->unique, oldloc->path, oldloc->inode->ino,
-                        newloc->path, newloc->inode->ino);
+                char string[4096]  = {0,};
+                if (newloc->inode)
+                        uuid_utoa_r (newloc->inode->gfid, newgfid);
+                else
+                        strcpy (newgfid, "0");
+
+                uuid_utoa_r (oldloc->inode->gfid, oldgfid);
+
+                snprintf (string, sizeof (string),
+                          "%"PRId64": oldgfid=%s oldpath=%s --> "
+                          "newgfid=%s newpath=%s", frame->root->unique,
+                          oldgfid, oldloc->path, newgfid,
+                          newloc->path);
+
+                frame->local = oldloc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_link_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->link,
-                    oldloc, newloc);
+                    oldloc, newloc, xdata);
         return 0;
 }
 
-
 int
 trace_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-               struct iatt *stbuf, int32_t valid)
+               struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        uint64_t ia_time          = 0;
-        char     actime_str[256]  = {0,};
-        char     modtime_str[256] = {0,};
+        uint64_t     ia_time          = 0;
+        char         actime_str[256]  = {0,};
+        char         modtime_str[256] = {0,};
+        trace_conf_t *conf            = NULL;
+
+        conf = this->private;
 
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_SETATTR].enabled) {
+                char     string[4096]  =  {0,};
                 if (valid & GF_SET_ATTR_MODE) {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (loc {path=%s, ino=%"PRIu64"},"
-                                " mode=%o)", frame->root->unique, loc->path,
-                                loc->inode->ino,
-                                st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type));
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s path=%s mode=%o)",
+                                  frame->root->unique,
+                                  uuid_utoa (loc->inode->gfid),
+                                  loc->path,
+                                  st_mode_from_ia (stbuf->ia_prot,
+                                                   stbuf->ia_type));
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0 , sizeof (string));
                 }
 
                 if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (loc {path=%s, ino=%"PRIu64"},"
-                                " uid=%o, gid=%o)",
-                                frame->root->unique, loc->path, loc->inode->ino,
-                                stbuf->ia_uid, stbuf->ia_gid);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s path=%s uid=%o,"
+                                  " gid=%o", frame->root->unique,
+                                  uuid_utoa (loc->inode->gfid),
+                                  loc->path, stbuf->ia_uid,
+                                  stbuf->ia_gid);
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0 , sizeof (string));
                 }
 
                 if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
@@ -1635,44 +2077,64 @@ trace_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
                         strftime (modtime_str, 256, "[%b %d %H:%M:%S]",
                                   localtime ((time_t *)&ia_time));
 
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (loc {path=%s, ino=%"PRIu64"}, "
-                                "*stbuf=%p {ia_atime=%s, ia_mtime=%s})",
-                                frame->root->unique, loc->path, loc->inode->ino,
-                                stbuf, actime_str, modtime_str);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s path=%s "
+                                  "ia_atime=%s, ia_mtime=%s",
+                                  frame->root->unique,
+                                  uuid_utoa (loc->inode->gfid),
+                                  loc->path, actime_str, modtime_str);
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0 , sizeof (string));
                 }
+                frame->local = loc->inode->gfid;
         }
 
+out:
         STACK_WIND (frame, trace_setattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setattr,
-                    loc, stbuf, valid);
+                    loc, stbuf, valid, xdata);
 
         return 0;
 }
 
-
 int
 trace_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                struct iatt *stbuf, int32_t valid)
+                struct iatt *stbuf, int32_t valid, dict_t *xdata)
 {
-        uint64_t ia_time          = 0;
-        char     actime_str[256]  = {0,};
-        char     modtime_str[256] = {0,};
+        uint64_t       ia_time          = 0;
+        char           actime_str[256]  = {0,};
+        char           modtime_str[256] = {0,};
+        trace_conf_t   *conf            = NULL;
 
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSETATTR].enabled) {
+                char     string[4096]  =  {0,};
                 if (valid & GF_SET_ATTR_MODE) {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (*fd=%p, mode=%o)",
-                                frame->root->unique, fd,
-                                st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type));
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s fd=%p, mode=%o",
+                                  frame->root->unique,
+                                  uuid_utoa (fd->inode->gfid), fd,
+                                  st_mode_from_ia (stbuf->ia_prot,
+                                                   stbuf->ia_type));
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0, sizeof (string));
                 }
 
                 if (valid & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (*fd=%p, uid=%o, gid=%o)",
-                                frame->root->unique, fd,
-                                stbuf->ia_uid, stbuf->ia_gid);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s fd=%p, uid=%o, "
+                                  "gid=%o", frame->root->unique,
+                                  uuid_utoa (fd->inode->gfid),
+                                  fd, stbuf->ia_uid, stbuf->ia_gid);
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0, sizeof (string));
                 }
 
                 if (valid & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
@@ -1684,372 +2146,816 @@ trace_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
                         strftime (modtime_str, 256, "[%b %d %H:%M:%S]",
                                   localtime ((time_t *)&ia_time));
 
-                        gf_log (this->name, GF_LOG_NORMAL,
-                                "%"PRId64": (*fd=%p"
-                                "*stbuf=%p {ia_atime=%s, ia_mtime=%s})",
-                                frame->root->unique, fd, stbuf, actime_str,
-                                modtime_str);
+                        snprintf (string, sizeof (string),
+                                  "%"PRId64": gfid=%s fd=%p "
+                                  "ia_atime=%s, ia_mtime=%s",
+                                  frame->root->unique,
+                                  uuid_utoa (fd->inode->gfid),
+                                  fd, actime_str, modtime_str);
+
+                        LOG_ELEMENT (conf, string);
+                        memset (string, 0, sizeof (string));
                 }
+                frame->local = fd->inode->gfid;
         }
 
+out:
         STACK_WIND (frame, trace_fsetattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsetattr,
-                    fd, stbuf, valid);
+                    fd, stbuf, valid, xdata);
 
         return 0;
 }
 
-
 int
 trace_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                off_t offset)
+                off_t offset, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_TRUNCATE].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (loc {path=%s, ino=%"PRIu64"}, offset=%"PRId64")",
-                        frame->root->unique, loc->path, loc->inode->ino, offset);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s, "
+                          "offset=%"PRId64"", frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          offset);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_truncate_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->truncate,
-                    loc, offset);
+                    loc, offset, xdata);
 
         return 0;
 }
 
-
 int
 trace_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
-            int32_t flags, fd_t *fd, int32_t wbflags)
+            int32_t flags, fd_t *fd, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_OPEN].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (loc {path=%s, ino=%"PRIu64"}, flags=%d, "
-                        "fd=%p, wbflags=%d)",
-                        frame->root->unique, loc->path, loc->inode->ino, flags,
-                        fd, wbflags);
+                char      string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s flags=%d fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          flags, fd);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_open_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->open,
-                    loc, flags, fd, wbflags);
+                    loc, flags, fd, xdata);
         return 0;
 }
 
-
 int
 trace_create (call_frame_t *frame, xlator_t *this, loc_t *loc,
-              int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
+              int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+              dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_CREATE].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (loc {path=%s, ino=%"PRIu64"}, flags=0%o mode=0%o)",
-                        frame->root->unique, loc->path, loc->inode->ino, flags, mode);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s, fd=%p, "
+                          "flags=0%o mode=0%o umask=0%o",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          fd, flags, mode, umask);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_create_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, xdata);
         return 0;
 }
 
-
 int
 trace_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
-             size_t size, off_t offset)
+             size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READ].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (*fd=%p, size=%"GF_PRI_SIZET", offset=%"PRId64")",
-                        frame->root->unique, fd, size, offset);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, size=%"
+                          GF_PRI_SIZET"offset=%"PRId64" flags=0%x)",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, size,
+                          offset, flags);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_readv_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readv,
-                    fd, size, offset);
+                    fd, size, offset, flags, xdata);
         return 0;
 }
 
-
 int
 trace_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
               struct iovec *vector, int32_t count,
-              off_t offset, struct iobref *iobref)
+              off_t offset, uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_WRITE].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (*fd=%p, *vector=%p, count=%d, offset=%"PRId64")",
-                        frame->root->unique, fd, vector, count, offset);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, count=%d, "
+                          " offset=%"PRId64" flags=0%x)",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, count,
+                          offset, flags);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_writev_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->writev,
-                    fd, vector, count, offset, iobref);
+                    fd, vector, count, offset, flags, iobref, xdata);
         return 0;
 }
 
-
 int
-trace_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc)
+trace_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_STATFS].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (loc {path=%s, ino=%"PRIu64"})",
-                        frame->root->unique, loc->path,
-                        ((loc->inode)? loc->inode->ino : 0));
+                char  string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s",
+                          frame->root->unique, (loc->inode)?
+                          uuid_utoa (loc->inode->gfid):"0", loc->path);
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_statfs_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->statfs,
-                    loc);
+                    loc, xdata);
         return 0;
 }
 
-
 int
-trace_flush (call_frame_t *frame, xlator_t *this, fd_t *fd)
+trace_flush (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FLUSH].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (*fd=%p)",
-                        frame->root->unique, fd);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_flush_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->flush,
-                    fd);
+                    fd, xdata);
         return 0;
 }
 
-
 int
-trace_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags)
+trace_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd, int32_t flags,
+             dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSYNC].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (flags=%d, *fd=%p)",
-                        frame->root->unique, flags, fd);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s flags=%d fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), flags, fd);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fsync_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsync,
-                    fd, flags);
+                    fd, flags, xdata);
         return 0;
 }
 
-
 int
 trace_setxattr (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, dict_t *dict, int32_t flags)
+                loc_t *loc, dict_t *dict, int32_t flags, dict_t *xdata)
 {
+        trace_conf_t    *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_SETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (loc {path=%s, ino=%"PRIu64"}, dict=%p, flags=%d)",
-                        frame->root->unique, loc->path,
-                        ((loc->inode)? loc->inode->ino : 0), dict, flags);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s flags=%d",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          flags);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_setxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setxattr,
-                    loc, dict, flags);
+                    loc, dict, flags, xdata);
         return 0;
 }
 
-
 int
 trace_getxattr (call_frame_t *frame, xlator_t *this,
-                loc_t *loc, const char *name)
+                loc_t *loc, const char *name, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_GETXATTR].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (loc {path=%s, ino=%"PRIu64"}), name=%s",
-                        frame->root->unique, loc->path,
-                        ((loc->inode)? loc->inode->ino : 0), name);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s name=%s",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          name);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_getxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->getxattr,
-                    loc, name);
+                    loc, name, xdata);
         return 0;
 }
 
-
 int
 trace_removexattr (call_frame_t *frame, xlator_t *this,
-                   loc_t *loc, const char *name)
+                   loc_t *loc, const char *name, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_REMOVEXATTR].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (loc {path=%s, ino=%"PRIu64"}, name=%s)",
-                        frame->root->unique, loc->path,
-                        ((loc->inode)? loc->inode->ino : 0), name);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s name=%s",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path,
+                          name);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_removexattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->removexattr,
-                    loc, name);
+                    loc, name, xdata);
 
         return 0;
 }
 
-
 int
-trace_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
+trace_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd,
+               dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64":( loc {path=%s, ino=%"PRIu64"}, fd=%p)",
-                        frame->root->unique, loc->path, loc->inode->ino, fd);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid), loc->path, fd);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_opendir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->opendir,
-                    loc, fd);
+                    loc, fd, xdata);
         return 0;
 }
 
 int
 trace_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                off_t offset)
+                off_t offset, dict_t *dict)
 {
+        trace_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READDIRP].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (fd=%p, size=%"GF_PRI_SIZET", offset=%"PRId64")",
-                        frame->root->unique, fd, size, offset);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET
+                          ", offset=%"PRId64" dict=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, size,
+                          offset, dict);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_readdirp_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readdirp,
-                    fd, size, offset);
+                    fd, size, offset, dict);
 
         return 0;
 }
 
-
 int
 trace_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
-               size_t size, off_t offset)
+               size_t size, off_t offset, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_READDIR].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (fd=%p, size=%"GF_PRI_SIZET", offset=%"PRId64")",
-                        frame->root->unique, fd, size, offset);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, size=%"GF_PRI_SIZET
+                          ", offset=%"PRId64,
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, size,
+                          offset);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_readdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->readdir,
-                    fd, size, offset);
+                    fd, size, offset, xdata);
 
         return 0;
 }
 
-
 int
 trace_fsyncdir (call_frame_t *frame, xlator_t *this,
-                fd_t *fd, int32_t datasync)
+                fd_t *fd, int32_t datasync, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSYNCDIR].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (datasync=%d, *fd=%p)",
-                        frame->root->unique, datasync, fd);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s datasync=%d fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), datasync, fd);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fsyncdir_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsyncdir,
-                    fd, datasync);
+                    fd, datasync, xdata);
         return 0;
 }
 
-
 int
-trace_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
+trace_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask,
+              dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_ACCESS].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (*loc {path=%s, ino=%"PRIu64"}, mask=0%o)",
-                        frame->root->unique, loc->path,
-                        ((loc->inode)? loc->inode->ino : 0), mask);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s path=%s mask=0%o",
+                          frame->root->unique,
+                          uuid_utoa (loc->inode->gfid),
+                          loc->path, mask);
+
+                frame->local = loc->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_access_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->access,
-                    loc, mask);
+                    loc, mask, xdata);
+        return 0;
+}
+
+int32_t
+trace_rchecksum (call_frame_t *frame, xlator_t *this, fd_t *fd, off_t offset,
+                 int32_t len, dict_t *xdata)
+{
+
+        trace_conf_t *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_RCHECKSUM].enabled) {
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s offset=%"PRId64
+                          "len=%u fd=%p", frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), offset, len, fd);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_rchecksum_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->rchecksum,
+                    fd, offset, len, xdata);
+
         return 0;
+
 }
 
+int32_t
+trace_fentrylk (call_frame_t *frame, xlator_t *this, const char *volume,
+                fd_t *fd, const char *basename, entrylk_cmd cmd,
+                entrylk_type type, dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FENTRYLK].enabled) {
+                char      string[4096]   =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s volume=%s, (fd=%p "
+                          "basename=%s, cmd=%s, type=%s)",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), volume, fd,
+                          basename,
+                          ((cmd == ENTRYLK_LOCK) ? "ENTRYLK_LOCK" :
+                           "ENTRYLK_UNLOCK"),
+                          ((type == ENTRYLK_RDLCK) ? "ENTRYLK_RDLCK" :
+                           "ENTRYLK_WRLCK"));
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_fentrylk_cbk,
+                    FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->fentrylk,
+                    volume, fd, basename, cmd, type, xdata);
+        return 0;
+
+}
+
+int32_t
+trace_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+                goto out;
+        if (trace_fop_names[GF_FOP_FGETXATTR].enabled) {
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p name=%s",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, name);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_fgetxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr,
+                    fd, name, xdata);
+        return 0;
+}
+
+int32_t
+trace_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_FSETXATTR].enabled) {
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p flags=%d",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, flags);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        STACK_WIND (frame, trace_fsetxattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr,
+                    fd, dict, flags, xdata);
+        return 0;
+}
 
 int
 trace_ftruncate (call_frame_t *frame, xlator_t *this,
-                 fd_t *fd, off_t offset)
+                 fd_t *fd, off_t offset, dict_t *xdata)
 {
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FTRUNCATE].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (offset=%"PRId64", *fd=%p)",
-                        frame->root->unique, offset, fd);
+                char    string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s offset=%"PRId64" fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), offset, fd);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_ftruncate_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->ftruncate,
-                    fd, offset);
+                    fd, offset, xdata);
 
         return 0;
 }
 
-
 int
-trace_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd)
+trace_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_FSTAT].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (*fd=%p)",
-                        frame->root->unique, fd);
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_fstat_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fstat,
-                    fd);
+                    fd, xdata);
         return 0;
 }
 
-
 int
 trace_lk (call_frame_t *frame, xlator_t *this, fd_t *fd,
-          int32_t cmd, struct gf_flock *lock)
+          int32_t cmd, struct gf_flock *lock, dict_t *xdata)
 {
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
         if (trace_fop_names[GF_FOP_LK].enabled) {
-                gf_log (this->name, GF_LOG_NORMAL,
-                        "%"PRId64": (*fd=%p, cmd=%d, lock {l_type=%d, l_whence=%d, "
-                        "l_start=%"PRId64", l_len=%"PRId64", l_pid=%u})",
-                        frame->root->unique, fd, cmd, lock->l_type, lock->l_whence,
-                        lock->l_start, lock->l_len, lock->l_pid);
+                char     string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "%"PRId64": gfid=%s fd=%p, cmd=%d, "
+                          "lock {l_type=%d, "
+                          "l_whence=%d, l_start=%"PRId64", "
+                          "l_len=%"PRId64", l_pid=%u})",
+                          frame->root->unique,
+                          uuid_utoa (fd->inode->gfid), fd, cmd,
+                          lock->l_type, lock->l_whence,
+                          lock->l_start, lock->l_len, lock->l_pid);
+
+                frame->local = fd->inode->gfid;
+
+                LOG_ELEMENT (conf, string);
         }
 
+out:
         STACK_WIND (frame, trace_lk_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->lk,
-                    fd, cmd, lock);
+                    fd, cmd, lock, xdata);
+        return 0;
+}
+
+int32_t
+trace_forget (xlator_t *this, inode_t *inode)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+        /* If user want to understand when a lookup happens,
+           he should know about 'forget' too */
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_LOOKUP].enabled) {
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "gfid=%s", uuid_utoa (inode->gfid));
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        return 0;
+}
+
+int32_t
+trace_releasedir (xlator_t *this, fd_t *fd)
+{
+        trace_conf_t  *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_OPENDIR].enabled) {
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "gfid=%s fd=%p",
+                          uuid_utoa (fd->inode->gfid), fd);
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
         return 0;
 }
 
+int32_t
+trace_release (xlator_t *this, fd_t *fd)
+{
+        trace_conf_t   *conf = NULL;
+
+        conf = this->private;
+
+        if (!conf->log_file && !conf->log_history)
+		goto out;
+        if (trace_fop_names[GF_FOP_OPEN].enabled ||
+            trace_fop_names[GF_FOP_CREATE].enabled) {
+                char   string[4096]  =  {0,};
+                snprintf (string, sizeof (string),
+                          "gfid=%s fd=%p",
+                          uuid_utoa (fd->inode->gfid), fd);
+
+                LOG_ELEMENT (conf, string);
+        }
+
+out:
+        return 0;
+}
 
 
 void
@@ -2061,7 +2967,6 @@ enable_all_calls (int enabled)
                 trace_fop_names[i].enabled = enabled;
 }
 
-
 void
 enable_call (const char *name, int enabled)
 {
@@ -2089,6 +2994,105 @@ process_call_list (const char *list, int include)
         }
 }
 
+int32_t
+trace_dump_history (xlator_t *this)
+{
+        int ret = -1;
+        char key_prefix[GF_DUMP_MAX_BUF_LEN] = {0,};
+        trace_conf_t *conf = NULL;
+
+        GF_VALIDATE_OR_GOTO ("trace", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, this->history, out);
+
+        conf = this->private;
+        // Is it ok to return silently if log-history option his off?
+        if (conf && conf->log_history == _gf_true) {
+                gf_proc_dump_build_key (key_prefix, "xlator.debug.trace",
+                                        "history");
+                gf_proc_dump_add_section (key_prefix);
+                eh_dump (this->history, NULL, dump_history_trace);
+        }
+        ret = 0;
+
+out:
+        return ret;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_trace_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
+                        " failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        int32_t       ret            = -1;
+        trace_conf_t    *conf           = NULL;
+        char            *includes = NULL, *excludes = NULL;
+
+        GF_VALIDATE_OR_GOTO ("quick-read", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, this->private, out);
+        GF_VALIDATE_OR_GOTO (this->name, options, out);
+
+        conf = this->private;
+
+        includes = data_to_str (dict_get (options, "include-ops"));
+        excludes = data_to_str (dict_get (options, "exclude-ops"));
+
+        {
+                int i;
+                for (i = 0; i < GF_FOP_MAXVALUE; i++) {
+                        if (gf_fop_list[i])
+                                strncpy (trace_fop_names[i].name,
+                                         gf_fop_list[i],
+                                         strlen (gf_fop_list[i]));
+                        else
+                                strncpy (trace_fop_names[i].name, ":O",
+                                         strlen (":O"));
+                        trace_fop_names[i].enabled = 1;
+                }
+        }
+
+        if (includes && excludes) {
+                gf_log (this->name,
+                        GF_LOG_ERROR,
+                        "must specify only one of 'include-ops' and "
+                        "'exclude-ops'");
+                goto out;
+        }
+
+        if (includes)
+                process_call_list (includes, 1);
+        if (excludes)
+                process_call_list (excludes, 0);
+
+        /* Should resizing of the event-history be allowed in reconfigure?
+         * for which a new event_history might have to be allocated and the
+         * older history has to be freed.
+         */
+        GF_OPTION_RECONF ("log-file", conf->log_file, options, bool, out);
+
+        GF_OPTION_RECONF ("log-history", conf->log_history, options, bool, out);
+
+        ret = 0;
+
+out:
+        return ret;
+}
 
 int32_t
 init (xlator_t *this)
@@ -2096,6 +3100,10 @@ init (xlator_t *this)
         dict_t *options = NULL;
         char *includes = NULL, *excludes = NULL;
         char *forced_loglevel = NULL;
+        eh_t *history = NULL;
+        int  ret = -1;
+        size_t  history_size = TRACE_DEFAULT_HISTORY_SIZE;
+        trace_conf_t    *conf = NULL;
 
         if (!this)
                 return -1;
@@ -2110,6 +3118,12 @@ init (xlator_t *this)
                         "dangling volume. check volfile ");
         }
 
+        conf = GF_CALLOC (1, sizeof (trace_conf_t), gf_trace_mt_trace_conf_t);
+        if (!conf) {
+                gf_log (this->name, GF_LOG_ERROR, "cannot allocate "
+                        "xl->private");
+                return -1;
+        }
 
         options = this->options;
         includes = data_to_str (dict_get (options, "include-ops"));
@@ -2118,8 +3132,13 @@ init (xlator_t *this)
         {
                 int i;
                 for (i = 0; i < GF_FOP_MAXVALUE; i++) {
-                        trace_fop_names[i].name = (gf_fop_list[i] ?
-                                                   gf_fop_list[i] : ":O");
+                        if (gf_fop_list[i])
+                                strncpy (trace_fop_names[i].name,
+                                         gf_fop_list[i],
+                                         strlen (gf_fop_list[i]));
+                        else
+                                strncpy (trace_fop_names[i].name, ":O",
+                                         strlen (":O"));
                         trace_fop_names[i].enabled = 1;
                 }
         }
@@ -2127,40 +3146,78 @@ init (xlator_t *this)
         if (includes && excludes) {
                 gf_log (this->name,
                         GF_LOG_ERROR,
-                        "must specify only one of 'include-ops' and 'exclude-ops'");
+                        "must specify only one of 'include-ops' and "
+                        "'exclude-ops'");
                 return -1;
         }
+
         if (includes)
                 process_call_list (includes, 1);
         if (excludes)
                 process_call_list (excludes, 0);
 
+
+        GF_OPTION_INIT ("history-size", conf->history_size, size, out);
+
+        gf_log (this->name, GF_LOG_INFO, "history size %"GF_PRI_SIZET,
+                history_size);
+
+        GF_OPTION_INIT ("log-file", conf->log_file, bool, out);
+
+        gf_log (this->name, GF_LOG_INFO, "logging to file %s",
+                (conf->log_file == _gf_true)?"enabled":"disabled");
+
+        GF_OPTION_INIT ("log-history", conf->log_history, bool, out);
+
+        gf_log (this->name, GF_LOG_DEBUG, "logging to history %s",
+                (conf->log_history == _gf_true)?"enabled":"disabled");
+
+        history = eh_new (history_size, _gf_false, NULL);
+        if (!history) {
+                gf_log (this->name, GF_LOG_ERROR, "event history cannot be "
+                        "initialized");
+                return -1;
+        }
+
+        this->history = history;
+
+        conf->trace_log_level = GF_LOG_INFO;
+
         if (dict_get (options, "force-log-level")) {
                 forced_loglevel = data_to_str (dict_get (options,
-                                               "force-log-level"));
+                                                         "force-log-level"));
                 if (!forced_loglevel)
                         goto setloglevel;
 
-                if (strcmp (forced_loglevel, "NORMAL") == 0)
-                        trace_log_level = GF_LOG_NORMAL;
+                if (strcmp (forced_loglevel, "INFO") == 0)
+                        conf->trace_log_level = GF_LOG_INFO;
                 else if (strcmp (forced_loglevel, "TRACE") == 0)
-                        trace_log_level = GF_LOG_TRACE;
+                        conf->trace_log_level = GF_LOG_TRACE;
                 else if (strcmp (forced_loglevel, "ERROR") == 0)
-                        trace_log_level = GF_LOG_ERROR;
+                        conf->trace_log_level = GF_LOG_ERROR;
                 else if (strcmp (forced_loglevel, "DEBUG") == 0)
-                        trace_log_level = GF_LOG_DEBUG;
+                        conf->trace_log_level = GF_LOG_DEBUG;
                 else if (strcmp (forced_loglevel, "WARNING") == 0)
-                        trace_log_level = GF_LOG_WARNING;
+                        conf->trace_log_level = GF_LOG_WARNING;
                 else if (strcmp (forced_loglevel, "CRITICAL") == 0)
-                        trace_log_level = GF_LOG_CRITICAL;
+                        conf->trace_log_level = GF_LOG_CRITICAL;
                 else if (strcmp (forced_loglevel, "NONE") == 0)
-                        trace_log_level = GF_LOG_NONE;
+                        conf->trace_log_level = GF_LOG_NONE;
         }
 
 setloglevel:
-        gf_log_set_loglevel (trace_log_level);
+        gf_log_set_loglevel (conf->trace_log_level);
+        this->private = conf;
+        ret = 0;
+out:
+        if (ret == -1) {
+                if (history)
+                        GF_FREE (history);
+                if (conf)
+                        GF_FREE (conf);
+        }
 
-        return 0;
+        return ret;
 }
 
 void
@@ -2169,7 +3226,10 @@ fini (xlator_t *this)
         if (!this)
                 return;
 
-        gf_log (this->name, GF_LOG_NORMAL,
+        if (this->history)
+                eh_destroy (this->history);
+
+        gf_log (this->name, GF_LOG_INFO,
                 "trace translator unloaded");
         return;
 }
@@ -2193,6 +3253,8 @@ struct xlator_fops fops = {
         .fsync       = trace_fsync,
         .setxattr    = trace_setxattr,
         .getxattr    = trace_getxattr,
+        .fsetxattr   = trace_fsetxattr,
+        .fgetxattr   = trace_fgetxattr,
         .removexattr = trace_removexattr,
         .opendir     = trace_opendir,
         .readdir     = trace_readdir,
@@ -2206,15 +3268,19 @@ struct xlator_fops fops = {
         .inodelk     = trace_inodelk,
         .finodelk    = trace_finodelk,
         .entrylk     = trace_entrylk,
+        .fentrylk    = trace_fentrylk,
         .lookup      = trace_lookup,
+        .rchecksum   = trace_rchecksum,
         .xattrop     = trace_xattrop,
         .fxattrop    = trace_fxattrop,
         .setattr     = trace_setattr,
         .fsetattr    = trace_fsetattr,
 };
 
-
 struct xlator_cbks cbks = {
+        .release     = trace_release,
+        .releasedir  = trace_releasedir,
+        .forget      = trace_forget,
 };
 
 struct volume_options options[] = {
@@ -2226,5 +3292,21 @@ struct volume_options options[] = {
           .type = GF_OPTION_TYPE_STR
           /*.value = { ""} */
         },
+        { .key  = {"history-size"},
+          .type = GF_OPTION_TYPE_SIZET,
+          .default_value = "1024",
+        },
+        { .key  = {"log-file"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "no",
+        },
+        { .key  = {"log-history"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "no",
+        },
         { .key  = {NULL} },
 };
+
+struct xlator_dumpops dumpops = {
+        .history = trace_dump_history
+};
diff --git a/xlators/debug/trace/src/trace.h b/xlators/debug/trace/src/trace.h
new file mode 100644
index 000000000..62d1bc9c9
--- /dev/null
+++ b/xlators/debug/trace/src/trace.h
@@ -0,0 +1,61 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include <time.h>
+#include <errno.h>
+#include "glusterfs.h"
+#include "xlator.h"
+#include "common-utils.h"
+#include "event-history.h"
+#include "logging.h"
+#include "circ-buff.h"
+#include "statedump.h"
+#include "options.h"
+
+#define TRACE_DEFAULT_HISTORY_SIZE 1024
+
+typedef struct {
+        /* Since the longest fop name is fremovexattr i.e 12 characters, array size
+         * is kept 24, i.e double of the maximum.
+         */
+        char name[24];
+        int enabled;
+} trace_fop_name_t;
+
+trace_fop_name_t trace_fop_names[GF_FOP_MAXVALUE];
+
+typedef struct {
+        gf_boolean_t log_file;
+        gf_boolean_t log_history;
+        size_t       history_size;
+        int trace_log_level;
+} trace_conf_t;
+
+#define TRACE_STACK_UNWIND(op, frame, params ...)                       \
+        do {                                                            \
+                frame->local = NULL;                                    \
+                STACK_UNWIND_STRICT (op, frame, params);                \
+        } while (0);
+
+#define LOG_ELEMENT(_conf, _string)                                     \
+        do {                                                            \
+                if (_conf) {                                            \
+                        if ((_conf->log_history) == _gf_true)           \
+                                gf_log_eh ("%s", _string);              \
+                        if ((_conf->log_file) == _gf_true)              \
+                                gf_log (THIS->name, _conf->trace_log_level, \
+                                        "%s", _string);                 \
+                }                                                       \
+        } while (0);
diff --git a/xlators/encryption/Makefile.am b/xlators/encryption/Makefile.am
index 2cbde680f..36efc6698 100644
--- a/xlators/encryption/Makefile.am
+++ b/xlators/encryption/Makefile.am
@@ -1,3 +1,3 @@
-SUBDIRS = rot-13
+SUBDIRS = rot-13 crypt
 
 CLEANFILES = 
diff --git a/xlators/protocol/legacy/server/Makefile.am b/xlators/encryption/crypt/Makefile.am
index d471a3f92..d471a3f92 100644
--- a/xlators/protocol/legacy/server/Makefile.am
+++ b/xlators/encryption/crypt/Makefile.am
diff --git a/xlators/encryption/crypt/src/Makefile.am b/xlators/encryption/crypt/src/Makefile.am
new file mode 100644
index 000000000..b13f65043
--- /dev/null
+++ b/xlators/encryption/crypt/src/Makefile.am
@@ -0,0 +1,24 @@
+if ENABLE_CRYPT_XLATOR
+
+xlator_LTLIBRARIES = crypt.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/encryption
+
+crypt_la_LDFLAGS = -module -avoid-version -lssl -lcrypto
+
+crypt_la_SOURCES = keys.c data.c metadata.c atom.c crypt.c
+crypt_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = crypt-common.h crypt-mem-types.h crypt.h metadata.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
+
+else
+
+noinst_DIST = keys.c data.c metadata.c atom.c crypt.c
+noinst_HEADERS = crypt-common.h crypt-mem-types.h crypt.h metadata.h
+
+endif
+\ No newline at end of file
diff --git a/xlators/encryption/crypt/src/atom.c b/xlators/encryption/crypt/src/atom.c
new file mode 100644
index 000000000..1ec41495c
--- /dev/null
+++ b/xlators/encryption/crypt/src/atom.c
@@ -0,0 +1,962 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+
+/*
+ *                              Glossary
+ *
+ *
+ * cblock            (or cipher block). A logical unit in a file.
+ *                   cblock size is defined as the number of bits
+ *                   in an input (or output) block of the block
+ *                   cipher (*). Cipher block size is a property of
+ *                   cipher algorithm. E.g. cblock size is 64 bits
+ *                   for DES, 128 bits for AES, etc.
+ *
+ * atomic cipher     A cipher algorithm, which requires some chunks of
+ * algorithm         text to be padded at left and(or) right sides before
+ *                   cipher transaform.
+ *
+ *
+ * block (atom)      Minimal chunk of file's data, which doesn't require
+ *                   padding. We'll consider logical units in a file of
+ *                   block size (atom size).
+ *
+ * cipher algorithm  Atomic cipher algorithm, which requires the last
+ * with EOF issue    incomplete cblock in a file to be padded with some
+ *                   data (usually zeros).
+ *
+ *
+ * operation, which  reading/writing from offset, which is not aligned to
+ * forms a gap at    to atom size
+ * the beginning
+ *
+ *
+ * operation, which  reading/writing count bytes starting from offset off,
+ * forms a gap at    so that off+count is not aligned to atom_size
+ * the end
+ *
+ * head block        the first atom affected by an operation, which forms
+ *                   a gap at the beginning, or(and) at the end.
+ *                   Сomment. Head block has at least one gap (either at
+ *                   the beginning, or at the end)
+ *
+ *
+ * tail block        the last atom different from head, affected by an
+ *                   operation, which forms a gap at the end.
+ *                   Сomment: Tail block has exactly one gap (at the end).
+ *
+ *
+ * partial block     head or tail block
+ *
+ *
+ * full block        block without gaps.
+ *
+ *
+ * (*) Recommendation for Block Cipher Modes of Operation
+ *     Methods and Techniques
+ *     NIST Special Publication 800-38A Edition 2001
+ */
+
+/*
+ * atom->offset_at()
+ */
+static off_t offset_at_head(struct avec_config *conf)
+{
+	return conf->aligned_offset;
+}
+
+static off_t offset_at_hole_head(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_head(get_hole_conf(frame));
+}
+
+static off_t offset_at_data_head(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_head(get_data_conf(frame));
+}
+
+
+static off_t offset_at_tail(struct avec_config *conf,
+				    struct object_cipher_info *object)
+{
+	return conf->aligned_offset +
+		(conf->off_in_head ? get_atom_size(object) : 0) +
+		(conf->nr_full_blocks << get_atom_bits(object));
+}
+
+static off_t offset_at_hole_tail(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_tail(get_hole_conf(frame), object);
+}
+
+
+static off_t offset_at_data_tail(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_tail(get_data_conf(frame), object);
+}
+
+static off_t offset_at_full(struct avec_config *conf,
+				    struct object_cipher_info *object)
+{
+	return conf->aligned_offset +
+		(conf->off_in_head ? get_atom_size(object) : 0);
+}
+
+static off_t offset_at_data_full(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_full(get_data_conf(frame), object);
+}
+
+static off_t offset_at_hole_full(call_frame_t *frame,
+					 struct object_cipher_info *object)
+{
+	return offset_at_full(get_hole_conf(frame), object);
+}
+
+/*
+ * atom->io_size_nopad()
+ */
+
+static uint32_t io_size_nopad_head(struct avec_config *conf,
+				   struct object_cipher_info *object)
+{
+	uint32_t gap_at_beg;
+	uint32_t gap_at_end;
+
+	check_head_block(conf);
+
+	gap_at_beg = conf->off_in_head;
+
+	if (has_tail_block(conf) || has_full_blocks(conf) || conf->off_in_tail == 0 )
+		gap_at_end = 0;
+	else
+		gap_at_end = get_atom_size(object) - conf->off_in_tail;
+
+	return get_atom_size(object) - (gap_at_beg + gap_at_end);
+}
+
+static uint32_t io_size_nopad_tail(struct avec_config *conf,
+				   struct object_cipher_info *object)
+{
+	check_tail_block(conf);
+	return conf->off_in_tail;
+}
+
+static uint32_t io_size_nopad_full(struct avec_config *conf,
+				   struct object_cipher_info *object)
+{	
+	check_full_block(conf);
+	return get_atom_size(object);
+}
+
+static uint32_t io_size_nopad_data_head(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_head(get_data_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_hole_head(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_head(get_hole_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_data_tail(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_tail(get_data_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_hole_tail(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_tail(get_hole_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_data_full(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_full(get_data_conf(frame), object);
+}
+
+static uint32_t io_size_nopad_hole_full(call_frame_t *frame,
+					struct object_cipher_info *object)
+{
+	return io_size_nopad_full(get_hole_conf(frame), object);
+}
+
+static uint32_t offset_in_head(struct avec_config *conf)
+{
+	check_cursor_head(conf);
+
+	return conf->off_in_head;
+}
+
+static uint32_t offset_in_tail(call_frame_t *frame,
+			       struct object_cipher_info *object)
+{
+	return 0;
+}
+
+static uint32_t offset_in_full(struct avec_config *conf,
+			       struct object_cipher_info *object)
+{
+	check_cursor_full(conf);
+
+	if (has_head_block(conf))
+		return (conf->cursor - 1) << get_atom_bits(object);
+	else
+		return conf->cursor << get_atom_bits(object);
+}
+
+static uint32_t offset_in_data_head(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_head(get_data_conf(frame));
+}
+
+static uint32_t offset_in_hole_head(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_head(get_hole_conf(frame));
+}
+
+static uint32_t offset_in_data_full(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_full(get_data_conf(frame), object);
+}
+
+static uint32_t offset_in_hole_full(call_frame_t *frame,
+				    struct object_cipher_info *object)
+{
+	return offset_in_full(get_hole_conf(frame), object);
+}
+
+/*
+ * atom->rmw()
+ */
+/*
+ * Pre-conditions:
+ * @vec contains plain text of the latest
+ * version.
+ *
+ * Uptodate gaps of the @partial block with
+ * this plain text, encrypt the whole block
+ * and write the result to disk.
+ */
+static int32_t rmw_partial_block(call_frame_t *frame,
+				 void *cookie,
+				 xlator_t *this,
+				 int32_t op_ret,
+				 int32_t op_errno,
+				 struct iovec *vec,
+				 int32_t count,
+				 struct iatt *stbuf,
+				 struct iobref *iobref,
+				 struct rmw_atom *atom)
+{
+	size_t was_read = 0;
+	uint64_t file_size;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	struct iovec *partial = atom->get_iovec(frame, 0);
+	struct avec_config *conf = atom->get_config(frame);
+	end_writeback_handler_t end_writeback_partial_block;
+#if DEBUG_CRYPT
+	gf_boolean_t check_last_cblock = _gf_false;
+#endif
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto exit;
+
+	file_size = local->cur_file_size;
+	was_read = op_ret;
+
+	if (atom->locality == HEAD_ATOM && conf->off_in_head) {
+		/*
+		 * head atom with a non-uptodate gap
+		 * at the beginning
+		 *
+		 * fill the gap with plain text of the
+		 * latest version. Convert a part of hole
+		 * (if any) to zeros.
+		 */
+		int32_t i;
+		int32_t copied = 0;
+		int32_t to_gap; /* amount of data needed to uptodate
+				   the gap at the beginning */
+#if 0
+		int32_t hole = 0; /* The part of the hole which
+				   * got in the head block */
+#endif /* 0 */
+		to_gap = conf->off_in_head;
+
+		if (was_read < to_gap) {
+			if (file_size >
+			    offset_at_head(conf) + was_read) {
+				/*
+				 * It is impossible to uptodate
+				 * head block: too few bytes have
+				 * been read from disk, so that
+				 * partial write is impossible.
+				 *
+				 * It could happen because of many
+				 * reasons: IO errors, (meta)data
+				 * corruption in the local file system,
+				 * etc.
+				 */
+				gf_log(this->name, GF_LOG_WARNING,
+				     "Can not uptodate a gap at the beginning");
+				local->op_ret = -1;
+				local->op_errno = EIO;
+				goto exit;
+			}
+#if 0
+			hole = to_gap - was_read;
+#endif /* 0 */
+			to_gap = was_read;
+		}
+		/*
+		 * uptodate the gap at the beginning
+		 */
+		for (i = 0; i < count && copied < to_gap; i++) {
+			int32_t to_copy;
+
+			to_copy = vec[i].iov_len;
+			if (to_copy > to_gap - copied)
+				to_copy = to_gap - copied;
+
+			memcpy(partial->iov_base, vec[i].iov_base, to_copy);
+			copied += to_copy;
+		}
+#if 0
+		/*
+		 * If possible, convert part of the
+		 * hole, which got in the head block
+		 */
+		ret = TRY_LOCK(&local->hole_lock);
+		if (!ret) {
+			if (local->hole_handled)
+				/*
+				 * already converted by
+				 * crypt_writev_cbk()
+				 */
+				UNLOCK(&local->hole_lock);
+			else {
+				/*
+				 * convert the part of the hole
+				 * which got in the head block
+				 * to zeros.
+				 *
+				 * Update the orig_offset to make
+				 * sure writev_cbk() won't care
+				 * about this part of the hole.
+				 *
+				 */
+				memset(partial->iov_base + to_gap, 0, hole);
+
+				conf->orig_offset -= hole;
+				conf->orig_size += hole;
+				UNLOCK(&local->hole_lock);
+			}
+		}
+		else /*
+		      * conversion is being performed
+		      * by crypt_writev_cbk()
+		      */
+			;
+#endif /* 0 */
+	}
+	if (atom->locality == TAIL_ATOM ||
+	    (!has_tail_block(conf) && conf->off_in_tail)) {
+		/*
+		 * tail atom, or head atom with a non-uptodate
+		 * gap at the end.
+		 *
+		 * fill the gap at the end of the block
+		 * with plain text of the latest version.
+		 * Pad the result, (if needed)
+		 */
+		int32_t i;
+		int32_t to_gap;
+		int copied;
+		off_t off_in_tail;
+		int32_t to_copy;
+
+		off_in_tail = conf->off_in_tail;
+		to_gap = conf->gap_in_tail;
+
+		if (to_gap && was_read < off_in_tail + to_gap) {
+			/*
+			 * It is impossible to uptodate
+			 * the gap at the end: too few bytes
+			 * have been read from disk, so that
+			 * partial write is impossible.
+			 *
+			 * It could happen because of many
+			 * reasons: IO errors, (meta)data
+			 * corruption in the local file system,
+			 * etc.
+			 */
+			gf_log(this->name, GF_LOG_WARNING,
+			       "Can not uptodate a gap at the end");
+			local->op_ret = -1;
+			local->op_errno = EIO;
+			goto exit;
+		}
+		/*
+		 * uptodate the gap at the end
+		 */
+		copied = 0;
+		to_copy = to_gap;
+		for(i = count - 1; i >= 0 && to_copy > 0; i--) {
+			uint32_t from_vec, off_in_vec;
+
+			off_in_vec = 0;
+			from_vec = vec[i].iov_len;
+			if (from_vec > to_copy) {
+				off_in_vec = from_vec - to_copy;
+				from_vec = to_copy;
+			}
+			memcpy(partial->iov_base +
+			       off_in_tail + to_gap - copied - from_vec,
+			       vec[i].iov_base + off_in_vec,
+			       from_vec);
+
+			gf_log(this->name, GF_LOG_DEBUG,
+			       "uptodate %d bytes at tail. Offset at target(source): %d(%d)",
+			       (int)from_vec,
+			       (int)off_in_tail + to_gap - copied - from_vec,
+			       (int)off_in_vec);
+
+			copied += from_vec;
+			to_copy -= from_vec;
+		}
+		partial->iov_len = off_in_tail + to_gap;
+
+		if (object_alg_should_pad(object)) {
+			int32_t resid = 0;
+			resid = partial->iov_len & (object_alg_blksize(object) - 1);
+			if (resid) {
+				/*
+				 * append a new EOF padding
+				 */
+				local->eof_padding_size =
+					object_alg_blksize(object) - resid;
+
+				gf_log(this->name, GF_LOG_DEBUG,
+				       "set padding size %d",
+				       local->eof_padding_size);
+
+				memset(partial->iov_base + partial->iov_len,
+				       1,
+				       local->eof_padding_size);
+				partial->iov_len += local->eof_padding_size;
+#if DEBUG_CRYPT
+				gf_log(this->name, GF_LOG_DEBUG,
+				       "pad cblock with %d zeros:",
+				       local->eof_padding_size);
+				dump_cblock(this,
+					    (unsigned char *)partial->iov_base +
+					    partial->iov_len - object_alg_blksize(object));
+				check_last_cblock = _gf_true;
+#endif
+			}
+		}
+	}
+	/*
+	 * encrypt the whole block
+	 */
+	encrypt_aligned_iov(object,
+			    partial,
+			    1,
+			    atom->offset_at(frame, object));
+#if DEBUG_CRYPT
+	if (check_last_cblock == _gf_true) {
+		gf_log(this->name, GF_LOG_DEBUG,
+		       "encrypt last cblock with offset %llu",
+		       (unsigned long long)atom->offset_at(frame, object));
+		dump_cblock(this, (unsigned char *)partial->iov_base +
+			    partial->iov_len - object_alg_blksize(object));
+	}
+#endif
+	set_local_io_params_writev(frame, object, atom,
+				   atom->offset_at(frame, object),
+				   iovec_get_size(partial, 1));
+	/*
+	 * write the whole block to disk
+	 */
+	end_writeback_partial_block = dispatch_end_writeback(local->fop);
+	conf->cursor ++;
+	STACK_WIND(frame,
+		   end_writeback_partial_block,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->writev,
+		   local->fd,
+		   partial,
+		   1,
+		   atom->offset_at(frame, object),
+		   local->flags,
+		   local->iobref_data,
+		   local->xdata);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "submit partial block: %d bytes from %d offset",
+	       (int)iovec_get_size(partial, 1),
+	       (int)atom->offset_at(frame, object));
+ exit:
+	return 0;
+}
+
+/*
+ * Perform a (read-)modify-write sequence.
+ * This should be performed only after approval
+ * of upper server-side manager, i.e. the caller
+ * needs to make sure this is his turn to rmw.
+ */
+void submit_partial(call_frame_t *frame,
+		    xlator_t *this,
+		    fd_t *fd,
+		    atom_locality_type ltype)
+{
+	int32_t ret;
+	dict_t *dict;
+	struct rmw_atom *atom;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	atom = atom_by_types(local->active_setup, ltype);
+	/*
+	 * To perform the "read" component of the read-modify-write
+	 * sequence the crypt translator does stack_wind to itself.
+	 * 
+	 * Pass current file size to crypt_readv()
+	 */
+	dict = dict_new();
+	if (!dict) {
+		/*
+		 * FIXME: Handle the error
+		 */
+		gf_log("crypt", GF_LOG_WARNING, "Can not alloc dict");
+		return;
+	}
+	ret = dict_set(dict,
+		       FSIZE_XATTR_PREFIX,
+		       data_from_uint64(local->cur_file_size));
+	if (ret) {
+		/*
+		 * FIXME: Handle the error
+		 */
+		dict_unref(dict);
+		gf_log("crypt", GF_LOG_WARNING, "Can not set dict");
+		goto exit;
+	}
+	STACK_WIND(frame,
+		   atom->rmw,
+		   this,
+		   this->fops->readv, /* crypt_readv */
+		   fd,
+		   atom->count_to_uptodate(frame, object), /* count */
+		   atom->offset_at(frame, object), /* offset to read from */
+		   0,
+		   dict);
+ exit:
+	dict_unref(dict);
+}
+
+/*
+ * submit blocks of FULL_ATOM type
+ */
+void submit_full(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+	struct rmw_atom *atom = atom_by_types(local->active_setup, FULL_ATOM);
+	uint32_t count; /* total number of full blocks to submit */
+	uint32_t granularity; /* number of blocks to submit in one iteration */
+
+	uint64_t off_in_file; /* start offset in the file, bytes */
+	uint32_t off_in_atom; /* start offset in the atom, blocks */
+	uint32_t blocks_written = 0; /* blocks written for this submit */
+
+	struct avec_config *conf = atom->get_config(frame);
+	end_writeback_handler_t end_writeback_full_block;
+	/*
+	 * Write full blocks by groups of granularity size.
+	 */
+	end_writeback_full_block = dispatch_end_writeback(local->fop);
+
+	if (is_ordered_mode(frame)) {
+		uint32_t skip = has_head_block(conf) ? 1 : 0;
+		count = 1;
+		granularity = 1;
+		/*
+		 * calculate start offset using cursor value;
+		 * here we should take into accout head block,
+		 * which corresponds to cursor value 0.
+		 */
+		off_in_file = atom->offset_at(frame, object) +
+			((conf->cursor - skip) << get_atom_bits(object));
+		off_in_atom = conf->cursor - skip;
+	}
+	else {
+		/*
+		 * in parallel mode
+		 */
+		count = conf->nr_full_blocks;
+		granularity = MAX_IOVEC;
+		off_in_file = atom->offset_at(frame, object);
+		off_in_atom = 0;
+	}
+	while (count) {
+		uint32_t blocks_to_write = count;
+
+		if (blocks_to_write > granularity)
+			blocks_to_write = granularity;
+		if (conf->type == HOLE_ATOM)
+			/*
+			 * reset iovec before encryption
+			 */
+			memset(atom->get_iovec(frame, 0)->iov_base,
+			       0,
+			       get_atom_size(object));
+		/*
+		 * encrypt the group
+		 */
+		encrypt_aligned_iov(object,
+				    atom->get_iovec(frame,
+						    off_in_atom +
+						    blocks_written),
+				    blocks_to_write,
+				    off_in_file + (blocks_written <<
+						   get_atom_bits(object)));
+		
+		set_local_io_params_writev(frame, object, atom,
+		        off_in_file + (blocks_written << get_atom_bits(object)),
+			blocks_to_write <<  get_atom_bits(object));
+
+		conf->cursor += blocks_to_write;
+
+		STACK_WIND(frame,
+			   end_writeback_full_block,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->writev,
+			   local->fd,
+			   atom->get_iovec(frame, off_in_atom + blocks_written),
+			   blocks_to_write,
+			   off_in_file + (blocks_written << get_atom_bits(object)),
+			   local->flags,
+			   local->iobref_data ? local->iobref_data : local->iobref,
+			   local->xdata);
+
+		gf_log("crypt", GF_LOG_DEBUG, "submit %d full blocks from %d offset",
+		       blocks_to_write,
+		       (int)(off_in_file + (blocks_written << get_atom_bits(object))));
+
+		count -= blocks_to_write;
+		blocks_written += blocks_to_write;
+	}
+	return;
+}
+
+static int32_t rmw_data_head(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(DATA_ATOM, HEAD_ATOM));
+}
+
+static int32_t rmw_data_tail(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(DATA_ATOM, TAIL_ATOM));
+}
+
+static int32_t rmw_hole_head(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(HOLE_ATOM, HEAD_ATOM));
+}
+
+static int32_t rmw_hole_tail(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     struct iovec *vec,
+			     int32_t count,
+			     struct iatt *stbuf,
+			     struct iobref *iobref,
+			     dict_t *xdata)
+{
+	return rmw_partial_block(frame,
+				 cookie,
+				 this,
+				 op_ret,
+				 op_errno,
+				 vec,
+				 count,
+				 stbuf,
+				 iobref,
+				 atom_by_types(HOLE_ATOM, TAIL_ATOM));
+}
+
+/*
+ * atom->count_to_uptodate()
+ */
+static uint32_t count_to_uptodate_head(struct avec_config *conf,
+				       struct object_cipher_info *object)
+{
+	if (conf->acount == 1 && conf->off_in_tail)
+		return get_atom_size(object);
+	else
+		/* there is no need to read the whole head block */
+		return conf->off_in_head;
+}
+
+static uint32_t count_to_uptodate_tail(struct avec_config *conf,
+				       struct object_cipher_info *object)
+{
+	/* we need to read the whole tail block */
+	return get_atom_size(object);
+}
+
+static uint32_t count_to_uptodate_data_head(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_head(get_data_conf(frame), object);
+}
+
+static uint32_t count_to_uptodate_data_tail(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_tail(get_data_conf(frame), object);
+}
+
+static uint32_t count_to_uptodate_hole_head(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_head(get_hole_conf(frame), object);
+}
+
+static uint32_t count_to_uptodate_hole_tail(call_frame_t *frame,
+					    struct object_cipher_info *object)
+{
+	return count_to_uptodate_tail(get_hole_conf(frame), object);
+}
+
+/* atom->get_config() */
+
+static struct avec_config *get_config_data(call_frame_t *frame)
+{
+	return &((crypt_local_t *)frame->local)->data_conf;
+}
+
+static struct avec_config *get_config_hole(call_frame_t *frame)
+{
+	return &((crypt_local_t *)frame->local)->hole_conf;
+}
+
+/*
+ * atom->get_iovec()
+ */
+static struct iovec *get_iovec_hole_head(call_frame_t *frame,
+					 uint32_t count)
+{
+	struct avec_config *conf = get_hole_conf(frame);
+
+	return conf->avec;
+}
+
+static struct iovec *get_iovec_hole_full(call_frame_t *frame,
+					 uint32_t count)
+{
+	struct avec_config *conf = get_hole_conf(frame);
+
+	return conf->avec + (conf->off_in_head ? 1 : 0);
+}
+
+static inline struct iovec *get_iovec_hole_tail(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_hole_conf(frame);
+
+	return conf->avec + (conf->blocks_in_pool - 1);
+}
+
+static inline struct iovec *get_iovec_data_head(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->avec;
+}
+
+static inline struct iovec *get_iovec_data_full(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->avec + (conf->off_in_head ? 1 : 0) + count;
+}
+
+static inline struct iovec *get_iovec_data_tail(call_frame_t *frame,
+						uint32_t count)
+{
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->avec +
+		(conf->off_in_head ? 1 : 0) +
+		conf->nr_full_blocks;
+}
+
+static struct rmw_atom atoms[LAST_DATA_TYPE][LAST_LOCALITY_TYPE] = {
+	[DATA_ATOM][HEAD_ATOM] =
+	{ .locality = HEAD_ATOM,
+	  .rmw = rmw_data_head,
+	  .offset_at = offset_at_data_head,
+	  .offset_in = offset_in_data_head,
+	  .get_iovec = get_iovec_data_head,
+	  .io_size_nopad = io_size_nopad_data_head,
+	  .count_to_uptodate = count_to_uptodate_data_head,
+	  .get_config = get_config_data
+	},
+	[DATA_ATOM][TAIL_ATOM] =
+	{ .locality = TAIL_ATOM,
+	  .rmw = rmw_data_tail,
+	  .offset_at = offset_at_data_tail,
+	  .offset_in = offset_in_tail,
+	  .get_iovec = get_iovec_data_tail,
+	  .io_size_nopad = io_size_nopad_data_tail,
+	  .count_to_uptodate = count_to_uptodate_data_tail,
+	  .get_config = get_config_data
+	},
+	[DATA_ATOM][FULL_ATOM] =
+	{ .locality = FULL_ATOM,
+	  .offset_at = offset_at_data_full,
+	  .offset_in = offset_in_data_full,
+	  .get_iovec = get_iovec_data_full,
+	  .io_size_nopad = io_size_nopad_data_full,
+	  .get_config = get_config_data
+	},
+	[HOLE_ATOM][HEAD_ATOM] =
+	{ .locality = HEAD_ATOM,
+	  .rmw = rmw_hole_head,
+	  .offset_at = offset_at_hole_head,
+	  .offset_in = offset_in_hole_head,
+	  .get_iovec = get_iovec_hole_head,
+	  .io_size_nopad = io_size_nopad_hole_head,
+	  .count_to_uptodate = count_to_uptodate_hole_head,
+	  .get_config = get_config_hole
+	},
+	[HOLE_ATOM][TAIL_ATOM] =
+	{ .locality = TAIL_ATOM,
+	  .rmw = rmw_hole_tail,
+	  .offset_at = offset_at_hole_tail,
+	  .offset_in = offset_in_tail,
+	  .get_iovec = get_iovec_hole_tail,
+	  .io_size_nopad = io_size_nopad_hole_tail,
+	  .count_to_uptodate = count_to_uptodate_hole_tail,
+	  .get_config = get_config_hole
+	},
+	[HOLE_ATOM][FULL_ATOM] =
+	{ .locality = FULL_ATOM,
+	  .offset_at = offset_at_hole_full,
+	  .offset_in = offset_in_hole_full,
+	  .get_iovec = get_iovec_hole_full,
+	  .io_size_nopad = io_size_nopad_hole_full,
+	  .get_config = get_config_hole
+	}
+};
+
+struct rmw_atom *atom_by_types(atom_data_type data,
+			       atom_locality_type locality)
+{
+	return &atoms[data][locality];
+}
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/crypt-common.h b/xlators/encryption/crypt/src/crypt-common.h
new file mode 100644
index 000000000..7c212ad5d
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt-common.h
@@ -0,0 +1,141 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CRYPT_COMMON_H__
+#define __CRYPT_COMMON_H__
+
+#define INVAL_SUBVERSION_NUMBER (0xff)
+#define CRYPT_INVAL_OP   (GF_FOP_NULL)
+
+#define CRYPTO_FORMAT_PREFIX  "trusted.glusterfs.crypt.att.cfmt"
+#define FSIZE_XATTR_PREFIX    "trusted.glusterfs.crypt.att.size"
+#define SUBREQ_PREFIX         "trusted.glusterfs.crypt.msg.sreq"
+#define FSIZE_MSG_PREFIX      "trusted.glusterfs.crypt.msg.size"
+#define DE_MSG_PREFIX         "trusted.glusterfs.crypt.msg.dent"
+#define REQUEST_ID_PREFIX     "trusted.glusterfs.crypt.msg.rqid"
+#define MSGFLAGS_PREFIX       "trusted.glusterfs.crypt.msg.xfgs"
+
+
+/* messages for crypt_open() */
+#define MSGFLAGS_REQUEST_MTD_RLOCK   1 /* take read lock and don't unlock */
+#define MSGFLAGS_REQUEST_MTD_WLOCK   2 /* take write lock and don't unlock */
+
+#define AES_BLOCK_BITS (4) /* AES_BLOCK_SIZE == 1 << AES_BLOCK_BITS */
+
+#define noop   do {; } while (0)
+#define cassert(cond) ({ switch (-1) { case (cond): case 0: break; } })
+#define __round_mask(x, y) ((__typeof__(x))((y)-1))
+#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
+
+/*
+ * Format of file's metadata
+ */
+struct crypt_format {
+	uint8_t loader_id;  /* version of metadata loader */
+	uint8_t versioned[0]; /* file's metadata of specific version */
+} __attribute__((packed));
+
+typedef enum {
+	AES_CIPHER_ALG,
+	LAST_CIPHER_ALG
+} cipher_alg_t;
+
+typedef enum {
+	XTS_CIPHER_MODE,
+	LAST_CIPHER_MODE
+} cipher_mode_t;
+
+typedef enum {
+	MTD_LOADER_V1,
+	LAST_MTD_LOADER
+} mtd_loader_id;
+
+static inline void msgflags_set_mtd_rlock(uint32_t *flags)
+{
+	*flags |= MSGFLAGS_REQUEST_MTD_RLOCK;
+}
+
+static inline void msgflags_set_mtd_wlock(uint32_t *flags)
+{
+	*flags |= MSGFLAGS_REQUEST_MTD_WLOCK;
+}
+
+static inline gf_boolean_t msgflags_check_mtd_rlock(uint32_t *flags)
+{
+	return *flags & MSGFLAGS_REQUEST_MTD_RLOCK;
+}
+
+static inline gf_boolean_t msgflags_check_mtd_wlock(uint32_t *flags)
+{
+	return *flags & MSGFLAGS_REQUEST_MTD_WLOCK;
+}
+
+static inline gf_boolean_t msgflags_check_mtd_lock(uint32_t *flags)
+{
+	return  msgflags_check_mtd_rlock(flags) ||
+		msgflags_check_mtd_wlock(flags);
+}
+
+/*
+ * returns number of logical blocks occupied
+ * (maybe partially) by @count bytes
+ * at offset @start.
+ */
+static inline off_t logical_blocks_occupied(uint64_t start, off_t count,
+					    int blkbits)
+{
+	return ((start + count - 1) >> blkbits) - (start >> blkbits) + 1;
+}
+
+/*
+ * are two bytes (represented by offsets @off1
+ * and @off2 respectively) in the same logical
+ * block.
+ */
+static inline int in_same_lblock(uint64_t off1, uint64_t off2,
+				 int blkbits)
+{
+	return off1 >> blkbits == off2 >> blkbits;
+}
+
+static inline void dump_cblock(xlator_t *this, unsigned char *buf)
+{
+	gf_log(this->name, GF_LOG_DEBUG,
+	       "dump cblock: %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x %x",
+	       (buf)[0],
+	       (buf)[1],
+	       (buf)[2],
+	       (buf)[3],
+	       (buf)[4],
+	       (buf)[5],
+	       (buf)[6],
+	       (buf)[7],
+	       (buf)[8],
+	       (buf)[9],
+	       (buf)[10],
+	       (buf)[11],
+	       (buf)[12],
+	       (buf)[13],
+	       (buf)[14],
+	       (buf)[15]);
+}
+
+#endif /* __CRYPT_COMMON_H__ */
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/crypt-mem-types.h b/xlators/encryption/crypt/src/crypt-mem-types.h
new file mode 100644
index 000000000..2eab921fc
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt-mem-types.h
@@ -0,0 +1,44 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+
+#ifndef __CRYPT_MEM_TYPES_H__
+#define __CRYPT_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_crypt_mem_types_ {
+	gf_crypt_mt_priv = gf_common_mt_end + 1,
+	gf_crypt_mt_inode,
+	gf_crypt_mt_data,
+	gf_crypt_mt_mtd,
+	gf_crypt_mt_loc,
+	gf_crypt_mt_iatt,
+	gf_crypt_mt_key,
+	gf_crypt_mt_iovec,
+	gf_crypt_mt_char,
+	gf_crypt_mt_end,
+};
+
+#endif /* __CRYPT_MEM_TYPES_H__ */
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
+
+
+
diff --git a/xlators/encryption/crypt/src/crypt.c b/xlators/encryption/crypt/src/crypt.c
new file mode 100644
index 000000000..13b1bd962
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt.c
@@ -0,0 +1,4522 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+#include <ctype.h>
+#include <sys/uio.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "logging.h"
+#include "defaults.h"
+
+#include "crypt-common.h"
+#include "crypt.h"
+
+static void init_inode_info_head(struct crypt_inode_info *info, fd_t *fd);
+static int32_t init_inode_info_tail(struct crypt_inode_info *info,
+				    struct master_cipher_info *master);
+static int32_t prepare_for_submit_hole(call_frame_t *frame, xlator_t *this,
+				       uint64_t from, off_t size);
+static int32_t load_file_size(call_frame_t *frame, void *cookie,
+			      xlator_t *this, int32_t op_ret, int32_t op_errno,
+			      dict_t *dict, dict_t *xdata);
+static void do_ordered_submit(call_frame_t *frame, xlator_t *this,
+			      atom_data_type dtype);
+static void do_parallel_submit(call_frame_t *frame, xlator_t *this,
+			       atom_data_type dtype);
+static void put_one_call_open(call_frame_t *frame);
+static void put_one_call_readv(call_frame_t *frame, xlator_t *this);
+static void put_one_call_writev(call_frame_t *frame, xlator_t *this);
+static void put_one_call_ftruncate(call_frame_t *frame, xlator_t *this);
+static void free_avec(struct iovec *avec, char **pool, int blocks_in_pool);
+static void free_avec_data(crypt_local_t *local);
+static void free_avec_hole(crypt_local_t *local);
+
+static crypt_local_t *crypt_alloc_local(call_frame_t *frame, xlator_t *this,
+					glusterfs_fop_t fop)
+{
+	crypt_local_t *local = NULL;
+
+	local = mem_get0(this->local_pool);
+	if (!local) {
+		gf_log(this->name, GF_LOG_ERROR, "out of memory");
+		return NULL;
+	}
+	local->fop = fop;
+	LOCK_INIT(&local->hole_lock);
+	LOCK_INIT(&local->call_lock);
+	LOCK_INIT(&local->rw_count_lock);
+
+	frame->local = local;
+	return local;
+}
+
+struct crypt_inode_info *get_crypt_inode_info(inode_t *inode, xlator_t *this)
+{
+	int ret;
+	uint64_t value = 0;
+	struct crypt_inode_info *info;
+
+	ret = inode_ctx_get(inode, this, &value);
+	if (ret == -1) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"Can not get inode info");
+		return NULL;
+	}
+	info = (struct crypt_inode_info *)(long)value;
+	if (info == NULL) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"Can not obtain inode info");
+		return NULL;
+	}
+	return info;
+}
+
+static struct crypt_inode_info *local_get_inode_info(crypt_local_t *local,
+						      xlator_t *this)
+{
+	if (local->info)
+		return local->info;
+	local->info = get_crypt_inode_info(local->fd->inode, this);
+	return local->info;
+}
+
+static struct crypt_inode_info *alloc_inode_info(crypt_local_t *local,
+						 loc_t *loc)
+{
+	struct crypt_inode_info *info;
+
+	info = GF_CALLOC(1, sizeof(*info), gf_crypt_mt_inode);
+	if (!info) {
+		local->op_ret = -1;
+		local->op_errno = ENOMEM;
+		gf_log ("crypt", GF_LOG_WARNING,
+			"Can not allocate inode info");
+		return NULL;
+	}
+	memset(info, 0, sizeof(*info));
+#if DEBUG_CRYPT
+	info->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!info->loc) {
+		gf_log("crypt", GF_LOG_WARNING, "Can not allocate loc");
+		GF_FREE(info);
+		return NULL;
+	}
+	if (loc_copy(info->loc, loc)){
+		GF_FREE(info->loc);
+		GF_FREE(info);
+		return NULL;
+	}
+#endif /* DEBUG_CRYPT */
+
+	local->info = info;
+	return info;
+}
+
+static void free_inode_info(struct crypt_inode_info *info)
+{
+#if DEBUG_CRYPT
+	loc_wipe(info->loc);
+	GF_FREE(info->loc);
+#endif
+	memset(info, 0, sizeof(*info));
+	GF_FREE(info);
+}
+
+int crypt_forget (xlator_t *this, inode_t *inode)
+{
+        uint64_t ctx_addr = 0;
+        if (!inode_ctx_del (inode, this, &ctx_addr))
+                free_inode_info((struct crypt_inode_info *)(long)ctx_addr);
+        return 0;
+}
+
+#if DEBUG_CRYPT
+static void check_read(call_frame_t *frame, xlator_t *this, int32_t read,
+		       struct iovec *vec, int32_t count, struct iatt *stbuf)
+{
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = get_object_cinfo(local->info);
+	struct avec_config *conf = &local->data_conf;
+	uint32_t resid = stbuf->ia_size & (object_alg_blksize(object) - 1);
+
+	if (read <= 0)
+		return;
+	if (read != iovec_get_size(vec, count))
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"op_ret differs from amount of read bytes");
+
+	if (object_alg_should_pad(object) && (read & (object_alg_blksize(object) - 1)))
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"bad amount of read bytes (!= 0 mod(cblock size))");
+
+	if (conf->aligned_offset + read >
+	    stbuf->ia_size + (resid ? object_alg_blksize(object) - resid : 0))
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"bad amount of read bytes (too large))");
+
+}
+
+#define PT_BYTES_TO_DUMP (32)
+static void dump_plain_text(crypt_local_t *local, struct iovec *avec)
+{
+	int32_t to_dump;
+	char str[PT_BYTES_TO_DUMP + 1];
+
+	if (!avec)
+		return;
+	to_dump = avec->iov_len;
+	if (to_dump > PT_BYTES_TO_DUMP)
+		to_dump = PT_BYTES_TO_DUMP;
+	memcpy(str, avec->iov_base, to_dump);
+	memset(str + to_dump, '0', 1);
+	gf_log("crypt", GF_LOG_DEBUG, "Read file: %s", str);
+}
+
+static int32_t data_conf_invariant(struct avec_config *conf)
+{
+	return conf->acount ==
+		!!has_head_block(conf) +
+		!!has_tail_block(conf)+
+		conf->nr_full_blocks;
+}
+
+static int32_t hole_conf_invariant(struct avec_config *conf)
+{
+	return conf->blocks_in_pool ==
+		!!has_head_block(conf) +
+		!!has_tail_block(conf)+
+		!!has_full_blocks(conf);
+}
+
+static void crypt_check_conf(struct avec_config *conf)
+{
+	int32_t ret = 0;
+	const char *msg;
+
+	switch (conf->type) {
+	case DATA_ATOM:
+		msg = "data";
+		ret = data_conf_invariant(conf);
+		break;
+	case HOLE_ATOM:
+		msg = "hole";
+		ret = hole_conf_invariant(conf);
+		break;
+	default:
+		msg = "unknown";
+	}
+	if (!ret)
+		gf_log("crypt", GF_LOG_DEBUG, "bad %s conf", msg);
+}
+
+static void check_buf(call_frame_t *frame, xlator_t *this, struct iatt *buf)
+{
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+	uint64_t local_file_size;
+
+	switch(local->fop) {
+	case GF_FOP_FTRUNCATE:
+		return;
+	case GF_FOP_WRITE:
+		local_file_size = local->new_file_size;
+		break;
+	case GF_FOP_READ:
+		if (parent_is_crypt_xlator(frame, this))
+			return;
+		local_file_size = local->cur_file_size;
+		break;
+	default:
+		gf_log("crypt", GF_LOG_DEBUG, "bad file operation");
+		return;
+	}
+	if (buf->ia_size != round_up(local_file_size,
+					 object_alg_blksize(object)))
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "bad ia_size in buf (%llu), should be %llu",
+		       (unsigned long long)buf->ia_size,
+		       (unsigned long long)round_up(local_file_size,
+						    object_alg_blksize(object)));
+}
+
+#else
+#define check_read(frame, this, op_ret, vec, count, stbuf) noop
+#define dump_plain_text(local, avec) noop
+#define crypt_check_conf(conf) noop
+#define check_buf(frame, this, buf) noop
+#endif /* DEBUG_CRYPT */
+
+/*
+ * Pre-conditions:
+ * @vec represents a ciphertext of expanded size and
+ * aligned offset.
+ *
+ * Compound a temporal vector @avec with block-aligned
+ * components, decrypt and fix it up to represent a chunk
+ * of data corresponding to the original size and offset.
+ * Pass the result to the next translator.
+ */
+int32_t crypt_readv_cbk(call_frame_t *frame,
+			void *cookie,
+			xlator_t *this,
+			int32_t op_ret,
+			int32_t op_errno,
+			struct iovec *vec,
+			int32_t count,
+			struct iatt *stbuf,
+			struct iobref *iobref,
+			dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	struct iovec *avec;
+	uint32_t i;
+	uint32_t to_vec;
+	uint32_t to_user;
+
+	check_buf(frame, this, stbuf);
+	check_read(frame, this, op_ret, vec, count, stbuf);
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	local->iobref = iobref_ref(iobref);
+
+	local->buf = *stbuf;
+	local->buf.ia_size = local->cur_file_size;
+
+	if (op_ret <= 0 || count == 0 || vec[0].iov_len == 0)
+		goto put_one_call;
+
+	if (conf->orig_offset >= local->cur_file_size) {
+		local->op_ret = 0;
+		goto put_one_call;
+	}
+	/* 
+	 * correct config params with real file size
+	 * and actual amount of bytes read
+	 */
+	set_config_offsets(frame, this,
+			   conf->orig_offset, op_ret, DATA_ATOM, 0);
+
+	if (conf->orig_offset + conf->orig_size > local->cur_file_size)
+		conf->orig_size = local->cur_file_size - conf->orig_offset;
+	/*
+	 * calculate amount of data to be returned
+	 * to user.
+	 */
+	to_user = op_ret;
+	if (conf->aligned_offset + to_user <= conf->orig_offset) {
+		gf_log(this->name, GF_LOG_WARNING, "Incomplete read");
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		goto put_one_call;
+	}
+	to_user -= (conf->aligned_offset - conf->orig_offset);
+
+	if (to_user > conf->orig_size)
+		to_user = conf->orig_size;
+	local->rw_count = to_user;
+
+	op_errno = set_config_avec_data(this, local,
+					conf, object, vec, count);
+	if (op_errno) {
+		local->op_ret = -1;
+		local->op_errno = op_errno;
+		goto put_one_call;
+	}
+	avec = conf->avec;
+#if DEBUG_CRYPT
+	if (conf->off_in_tail != 0 &&
+	    conf->off_in_tail < object_alg_blksize(object) &&
+	    object_alg_should_pad(object))
+		gf_log(this->name, GF_LOG_DEBUG, "Bad offset in tail %d",
+		       conf->off_in_tail);
+	if (iovec_get_size(vec, count) != 0 &&
+	    in_same_lblock(conf->orig_offset + iovec_get_size(vec, count) - 1,
+			   local->cur_file_size - 1,
+			   object_alg_blkbits(object))) {
+		gf_log(this->name, GF_LOG_DEBUG, "Compound last cblock");
+		dump_cblock(this,
+			    (unsigned char *)(avec[conf->acount - 1].iov_base) +
+			    avec[conf->acount - 1].iov_len - object_alg_blksize(object));
+		dump_cblock(this,
+			    (unsigned char *)(vec[count - 1].iov_base) +
+			    vec[count - 1].iov_len - object_alg_blksize(object));
+	}
+#endif
+	decrypt_aligned_iov(object, avec,
+			    conf->acount, conf->aligned_offset);
+	/*
+	 * pass proper plain data to user
+	 */
+	avec[0].iov_base += (conf->aligned_offset - conf->orig_offset);
+	avec[0].iov_len  -= (conf->aligned_offset - conf->orig_offset);
+
+	to_vec = to_user;
+	for (i = 0; i < conf->acount; i++) {
+		if (avec[i].iov_len > to_vec)
+			avec[i].iov_len = to_vec;
+		to_vec -= avec[i].iov_len;
+	}
+ put_one_call:
+	put_one_call_readv(frame, this);
+	return 0;
+}
+
+static int32_t do_readv(call_frame_t *frame,
+			void *cookie,
+			xlator_t *this,
+			int32_t op_ret,
+			int32_t op_errno,
+			dict_t *dict,
+			dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;	
+	/*
+	 * extract regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_errno = EIO;
+		goto error;
+	}
+	local->cur_file_size = data_to_uint64(data);
+	
+	get_one_call(frame);
+	STACK_WIND(frame,
+		   crypt_readv_cbk,
+		   FIRST_CHILD (this),
+		   FIRST_CHILD (this)->fops->readv,
+		   local->fd,
+		   /*
+		    * FIXME: read amount can be reduced
+		    */
+		   local->data_conf.expanded_size,
+		   local->data_conf.aligned_offset,
+		   local->flags,
+		   local->xdata);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+
+	get_one_call(frame);
+	put_one_call_readv(frame, this);
+	return 0;
+}
+
+static int32_t crypt_readv_finodelk_cbk(call_frame_t *frame,
+					void *cookie,
+					xlator_t *this,
+					int32_t op_ret,
+					int32_t op_errno,
+					dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	/*
+	 * An access has been granted,
+	 * retrieve file size
+	 */
+	STACK_WIND(frame,
+		   do_readv,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return  0;
+ error:
+	fd_unref(local->fd);
+	if (local->xdata)
+		dict_unref(local->xdata);
+	STACK_UNWIND_STRICT(readv,
+			    frame,
+			    -1,
+			    op_errno,
+			    NULL,
+			    0,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t readv_trivial_completion(call_frame_t *frame,
+					void *cookie,
+					xlator_t *this,
+					int32_t op_ret,
+					int32_t op_errno,
+					struct iatt *buf,
+					dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "stat failed (%d)", op_errno);
+		goto error;
+	}
+	local->buf = *buf;
+	STACK_WIND(frame,
+		   load_file_size,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->getxattr,
+		   local->loc,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;	
+ error:
+	STACK_UNWIND_STRICT(readv, frame, op_ret, op_errno,
+			    NULL, 0, NULL, NULL, NULL);
+	return 0;
+}
+
+int32_t crypt_readv(call_frame_t *frame,
+		    xlator_t *this,
+		    fd_t *fd,
+		    size_t size,
+		    off_t offset,
+		    uint32_t flags, dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+	struct crypt_inode_info *info;
+	struct gf_flock lock = {0, };
+
+#if DEBUG_CRYPT
+	gf_log("crypt", GF_LOG_DEBUG, "reading %d bytes from offset %llu",
+	       (int)size, (long long)offset);
+	if (parent_is_crypt_xlator(frame, this))
+		gf_log("crypt", GF_LOG_DEBUG, "parent is crypt");
+#endif	
+	local = crypt_alloc_local(frame, this, GF_FOP_READ);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	if (size == 0)
+		goto trivial;
+
+	local->fd = fd_ref(fd);
+	local->flags = flags;
+
+	info = local_get_inode_info(local, this);
+	if (info == NULL) {
+		ret = EINVAL;
+		fd_unref(fd);
+		goto error;
+	}
+	if (!object_alg_atomic(&info->cinfo)) {
+		ret = EINVAL;
+		fd_unref(fd);
+		goto error;
+	}
+	set_config_offsets(frame, this, offset, size,
+			   DATA_ATOM, 0);
+	if (parent_is_crypt_xlator(frame, this)) {
+		data_t *data;
+		/*
+		 * We are called by crypt_writev (or cypt_ftruncate)
+		 * to perform the "read" component of the read-modify-write
+		 * (or read-prune-write) sequence for some atom;
+		 *
+		 * don't ask for access:
+		 * it has already been acquired
+		 *
+		 * Retrieve current file size
+		 */
+		if (!xdata) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size hasn't been passed");
+			ret = EIO;
+			goto error;
+		}
+		data = dict_get(xdata, FSIZE_XATTR_PREFIX);
+		if (!data) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size not found");
+			ret = EIO;
+			goto error;
+		}
+		local->old_file_size =
+			local->cur_file_size = data_to_uint64(data);
+
+		get_one_call(frame);
+		STACK_WIND(frame,
+			   crypt_readv_cbk,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->readv,
+			   local->fd,
+			   /*
+			    * FIXME: read amount can be reduced
+			    */
+			   local->data_conf.expanded_size,
+			   local->data_conf.aligned_offset,
+			   flags,
+			   NULL);
+		return 0;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_RDLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_readv_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ trivial:
+	STACK_WIND(frame,
+		   readv_trivial_completion,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   fd,
+		   NULL);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(readv,
+			    frame,
+			    -1,
+			    ret,
+			    NULL,
+			    0,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+void set_local_io_params_writev(call_frame_t *frame,
+				struct object_cipher_info *object,
+				struct rmw_atom *atom,
+				off_t io_offset,
+				uint32_t io_size)
+{
+	crypt_local_t *local = frame->local;
+
+	local->io_offset = io_offset;
+	local->io_size = io_size;
+
+	local->io_offset_nopad =
+		atom->offset_at(frame, object) + atom->offset_in(frame, object);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "set nopad offset to %llu",
+	       (unsigned long long)local->io_offset_nopad);
+
+	local->io_size_nopad = atom->io_size_nopad(frame, object);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "set nopad size to %llu",
+	       (unsigned long long)local->io_size_nopad);
+
+	local->update_disk_file_size = 0;
+	/*
+	 * NOTE: eof_padding_size is 0 for all full atoms;
+	 * For head and tail atoms it will be set up at rmw_partial block()
+	 */
+	local->new_file_size = local->cur_file_size;
+
+	if (local->io_offset_nopad + local->io_size_nopad > local->cur_file_size) {
+
+		local->new_file_size = local->io_offset_nopad + local->io_size_nopad;
+
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "set new file size to %llu",
+		       (unsigned long long)local->new_file_size);
+
+		local->update_disk_file_size = 1;
+	}
+}
+
+void set_local_io_params_ftruncate(call_frame_t *frame,
+				   struct object_cipher_info *object)
+{
+	uint32_t resid;
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+
+	resid = conf->orig_offset & (object_alg_blksize(object) - 1);
+	if (resid) {
+		local->eof_padding_size =
+			object_alg_blksize(object) - resid;
+		local->new_file_size = conf->aligned_offset;
+		local->update_disk_file_size = 0;
+		/*
+		 * file size will be updated
+		 * in the ->writev() stack,
+		 * when submitting file tail
+		 */
+	}
+	else {	
+		local->eof_padding_size = 0;
+		local->new_file_size = conf->orig_offset;
+		local->update_disk_file_size = 1;
+		/*
+		 * file size will be updated
+		 * in this ->ftruncate stack
+		 */
+	}
+}
+
+static inline void submit_head(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	submit_partial(frame, this, local->fd, HEAD_ATOM);
+}
+
+static inline void submit_tail(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	submit_partial(frame, this, local->fd, TAIL_ATOM);
+}
+
+static void submit_hole(call_frame_t *frame, xlator_t *this)
+{
+	/*
+	 * hole conversion always means
+	 * appended write and goes in ordered fashion
+	 */
+	do_ordered_submit(frame, this, HOLE_ATOM);
+}
+
+static void submit_data(call_frame_t *frame, xlator_t *this)
+{
+	if (is_ordered_mode(frame)) {
+		do_ordered_submit(frame, this, DATA_ATOM);
+		return;
+	}
+	gf_log("crypt", GF_LOG_WARNING, "Bad submit mode");
+	get_nr_calls(frame, nr_calls_data(frame));
+	do_parallel_submit(frame, this, DATA_ATOM);
+	return;
+}
+
+/*
+ * heplers called by writev_cbk, fruncate_cbk in ordered mode
+ */
+
+static inline int32_t should_submit_hole(crypt_local_t *local)
+{
+	struct avec_config *conf = &local->hole_conf;
+
+	return conf->avec != NULL;
+}
+
+static inline int32_t should_resume_submit_hole(crypt_local_t *local)
+{
+	struct avec_config *conf = &local->hole_conf;
+
+	if (local->fop == GF_FOP_WRITE && has_tail_block(conf))
+		/*
+		 * Don't submit a part of hole, which
+		 * fits into a data block:
+ 		 * this part of hole will be converted
+		 * as a gap filled by zeros in data head
+		 * block.
+		 */
+		return conf->cursor < conf->acount - 1;
+	else
+		return conf->cursor < conf->acount;
+}
+
+static inline int32_t should_resume_submit_data(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+
+	if (is_ordered_mode(frame))
+		return conf->cursor < conf->acount;
+	/*
+	 * parallel writes
+	 */
+	return 0;
+}
+
+static inline int32_t should_submit_data_after_hole(crypt_local_t *local)
+{
+	return local->data_conf.avec != NULL;
+}
+
+static void update_local_file_params(call_frame_t *frame,
+				     xlator_t *this,
+				     struct iatt *prebuf,
+				     struct iatt *postbuf)
+{
+	crypt_local_t *local = frame->local;
+
+	check_buf(frame, this, postbuf);
+
+	local->prebuf = *prebuf;
+	local->postbuf = *postbuf;
+
+	local->prebuf.ia_size  = local->cur_file_size;
+	local->postbuf.ia_size = local->new_file_size;
+
+	local->cur_file_size = local->new_file_size;	
+}
+
+static int32_t end_writeback_writev(call_frame_t *frame,
+				    void *cookie,
+				    xlator_t *this,
+				    int32_t op_ret,
+				    int32_t op_errno,
+				    struct iatt *prebuf,
+				    struct iatt *postbuf,
+				    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret <= 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "writev iteration failed");
+		goto put_one_call;
+	}
+	/*
+	 * op_ret includes paddings (atom's head, atom's tail and EOF)
+	 */
+	if (op_ret < local->io_size) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "Incomplete writev iteration");
+		goto put_one_call;
+	}
+	op_ret -= local->eof_padding_size;
+	local->op_ret = op_ret;
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+
+	if (data_write_in_progress(local)) {
+
+		LOCK(&local->rw_count_lock);
+		local->rw_count += op_ret;
+		UNLOCK(&local->rw_count_lock);
+
+		if (should_resume_submit_data(frame))
+			submit_data(frame, this);
+	}
+	else {
+		/*
+		 * hole conversion is going on;
+		 * don't take into account written zeros
+		 */
+		if (should_resume_submit_hole(local))
+			submit_hole(frame, this);
+
+		else if (should_submit_data_after_hole(local))
+			submit_data(frame, this);
+	}
+ put_one_call:
+	put_one_call_writev(frame, this);
+	return 0;
+}
+
+#define crypt_writev_cbk end_writeback_writev
+
+#define HOLE_WRITE_CHUNK_BITS 12
+#define HOLE_WRITE_CHUNK_SIZE (1 << HOLE_WRITE_CHUNK_BITS)
+
+/*
+ * Convert hole of size @size at offset @off to
+ * zeros and prepare respective iovecs for submit.
+ * The hole lock should be held.
+ *
+ * Pre-conditions:
+ * @local->file_size is set and valid.
+ */
+int32_t prepare_for_submit_hole(call_frame_t *frame, xlator_t *this,
+				uint64_t off, off_t size)
+{
+	int32_t ret;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	set_config_offsets(frame, this, off, size, HOLE_ATOM, 1);
+
+	ret = set_config_avec_hole(this, local,
+				   &local->hole_conf, object, local->fop);
+	crypt_check_conf(&local->hole_conf);
+
+	return ret;
+}
+
+/*
+ * prepare for submit @count bytes at offset @from
+ */
+int32_t prepare_for_submit_data(call_frame_t *frame, xlator_t *this,
+				off_t from, int32_t size, struct iovec *vec,
+				int32_t vec_count, int32_t setup_gap)
+{
+	uint32_t ret;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	set_config_offsets(frame, this, from, size,
+			   DATA_ATOM, setup_gap);
+
+	ret = set_config_avec_data(this, local,
+				   &local->data_conf, object, vec, vec_count);
+	crypt_check_conf(&local->data_conf);
+
+	return ret;
+}
+
+static void free_avec(struct iovec *avec,
+		      char **pool, int blocks_in_pool)
+{
+	if (!avec)
+		return;
+	GF_FREE(pool);
+	GF_FREE(avec);
+}
+
+static void free_avec_data(crypt_local_t *local)
+{
+	return free_avec(local->data_conf.avec,
+			 local->data_conf.pool,
+			 local->data_conf.blocks_in_pool);
+}
+
+static void free_avec_hole(crypt_local_t *local)
+{
+	return free_avec(local->hole_conf.avec,
+			 local->hole_conf.pool,
+			 local->hole_conf.blocks_in_pool);
+}
+
+
+static void do_parallel_submit(call_frame_t *frame, xlator_t *this,
+			       atom_data_type dtype)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf;
+
+	local->active_setup = dtype;
+	conf = conf_by_type(frame, dtype);
+
+	if (has_head_block(conf))
+		submit_head(frame, this);
+
+	if (has_full_blocks(conf))
+		submit_full(frame, this);
+
+	if (has_tail_block(conf))
+		submit_tail(frame, this);
+	return;
+}
+
+static void do_ordered_submit(call_frame_t *frame, xlator_t *this,
+			      atom_data_type dtype)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf;
+
+	local->active_setup = dtype;
+	conf = conf_by_type(frame, dtype);
+
+	if (should_submit_head_block(conf)) {
+		get_one_call_nolock(frame);
+		submit_head(frame, this);
+	}
+	else if (should_submit_full_block(conf)) {
+		get_one_call_nolock(frame);
+		submit_full(frame, this);
+	}
+	else if (should_submit_tail_block(conf)) {
+		get_one_call_nolock(frame);
+		submit_tail(frame, this);
+	}
+	else
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "nothing has been submitted in ordered mode");
+	return;
+}
+
+static int32_t do_writev(call_frame_t *frame,
+			 void *cookie,
+			 xlator_t *this,
+			 int32_t op_ret,
+			 int32_t op_errno,
+			 dict_t *dict,
+			 dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+	struct object_cipher_info *object = &local->info->cinfo;
+	/*
+	 * extract regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_ret = -1;
+		op_errno = EIO;
+		goto error;
+	}
+	local->old_file_size = local->cur_file_size = data_to_uint64(data);
+
+	set_gap_at_end(frame, object, &local->data_conf, DATA_ATOM);
+
+	if (local->cur_file_size < local->data_conf.orig_offset) {
+		/*
+		 * Set up hole config
+		 */
+		op_errno = prepare_for_submit_hole(frame,
+			       this,
+			       local->cur_file_size,
+			       local->data_conf.orig_offset - local->cur_file_size);
+		if (op_errno) {
+			local->op_ret = -1;
+			local->op_errno = op_errno;
+			goto error;
+		}
+	}
+	if (should_submit_hole(local))
+		submit_hole(frame, this);
+	else 
+		submit_data(frame, this);
+	return 0;
+ error:
+	get_one_call_nolock(frame);
+	put_one_call_writev(frame, this);
+	return 0;
+}
+
+static int32_t crypt_writev_finodelk_cbk(call_frame_t *frame,
+					 void *cookie,
+					 xlator_t *this,
+					 int32_t op_ret,
+					 int32_t op_errno,
+					 dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto error;
+	/*
+	 * An access has been granted,
+	 * retrieve file size first
+	 */
+	STACK_WIND(frame,
+		   do_writev,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;
+ error:
+	get_one_call_nolock(frame);
+	put_one_call_writev(frame, this);
+	return 0;	
+}
+
+static int32_t writev_trivial_completion(call_frame_t *frame,
+					 void *cookie,
+					 xlator_t *this,
+					 int32_t op_ret,
+					 int32_t op_errno,
+					 struct iatt *buf,
+					 dict_t *dict)
+{
+	crypt_local_t *local = frame->local;
+	
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	local->prebuf = *buf;
+	local->postbuf = *buf;
+
+	local->prebuf.ia_size = local->cur_file_size;
+	local->postbuf.ia_size = local->cur_file_size;
+
+	get_one_call(frame);
+	put_one_call_writev(frame, this);
+	return 0;
+}
+
+int crypt_writev(call_frame_t *frame,
+		 xlator_t *this,
+		 fd_t *fd,
+		 struct iovec *vec,
+		 int32_t count,
+		 off_t offset,
+		 uint32_t flags,
+		 struct iobref *iobref,
+		 dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+	struct crypt_inode_info *info;
+	struct gf_flock lock = {0, };
+#if DEBUG_CRYPT
+	gf_log ("crypt", GF_LOG_DEBUG, "writing %d bytes from offset %llu",
+		(int)iovec_get_size(vec, count), (long long)offset);
+#endif
+	local = crypt_alloc_local(frame, this, GF_FOP_WRITE);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+
+	if (iobref)
+		local->iobref = iobref_ref(iobref);
+	/*
+	 * to update real file size on the server
+	 */
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->flags = flags;
+
+	info = local_get_inode_info(local, this);
+	if (info == NULL) {
+		ret = EINVAL;
+		goto error;
+	}
+	if (!object_alg_atomic(&info->cinfo)) {
+		ret = EINVAL;
+		goto error;
+	}
+	if (iovec_get_size(vec, count) == 0)
+		goto trivial;
+
+	ret = prepare_for_submit_data(frame, this, offset,
+				      iovec_get_size(vec, count),
+				      vec, count, 0 /* don't setup gup
+						       in tail: we don't
+						       know file size yet */);
+	if (ret)
+		goto error;
+
+	if (parent_is_crypt_xlator(frame, this)) {
+		data_t *data;
+		/*
+		 * we are called by shinking crypt_ftruncate(),
+		 * which doesn't perform hole conversion;
+		 *
+		 * don't ask for access:
+		 * it has already been acquired
+		 */
+
+		/*
+		 * extract file size
+		 */
+		if (!xdata) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size hasn't been passed");
+			ret = EIO;
+			goto error;
+		}
+		data = dict_get(xdata, FSIZE_XATTR_PREFIX);
+		if (!data) {
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size not found");
+			ret = EIO;
+			goto error;
+		}
+		local->old_file_size =
+			local->cur_file_size = data_to_uint64(data);
+
+		submit_data(frame, this);
+		return 0;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	/*
+	 * lock the file and retrieve its size
+	 */
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_writev_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ trivial:
+	STACK_WIND(frame,
+		   writev_trivial_completion,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   fd,
+		   NULL);
+	return 0;
+ error:
+	if (local && local->fd)
+		fd_unref(fd);
+	if (local && local->iobref)
+		iobref_unref(iobref);
+	if (local && local->xdata)
+		dict_unref(xdata);
+	if (local && local->xattr)
+		dict_unref(local->xattr);
+	if (local && local->info)
+		free_inode_info(local->info);
+
+	STACK_UNWIND_STRICT(writev, frame, -1, ret, NULL, NULL, NULL);
+	return 0;
+}
+
+int32_t prepare_for_prune(call_frame_t *frame, xlator_t *this, uint64_t offset)
+{
+	set_config_offsets(frame, this,
+			   offset,
+			   0, /* count */
+			   DATA_ATOM,
+			   0 /* since we prune, there is no
+				gap in tail to uptodate */);
+	return 0;
+}
+
+/*
+ * Finish the read-prune-modify sequence
+ *
+ * Can be invoked as
+ * 1) ->ftruncate_cbk() for cblock-aligned, or trivial prune
+ * 2) ->writev_cbk() for non-cblock-aligned prune
+ */
+
+static int32_t prune_complete(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      struct iatt *prebuf,
+			      struct iatt *postbuf,
+			      dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * This is called as ->ftruncate_cbk()
+ *
+ * Perform the "write" component of the
+ * read-prune-write sequence.
+ *
+ * submuit the rest of the file
+ */
+static int32_t prune_submit_file_tail(call_frame_t *frame,
+				      void *cookie,
+				      xlator_t *this,
+				      int32_t op_ret,
+				      int32_t op_errno,
+				      struct iatt *prebuf,
+				      struct iatt *postbuf,
+				      dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+	dict_t *dict;
+
+	if (op_ret < 0)
+		goto put_one_call;
+
+	if (local->xdata) {
+		dict_unref(local->xdata);
+		local->xdata = NULL;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	dict = dict_new();
+	if (!dict) {
+		op_errno = ENOMEM;
+		goto error;
+	}
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+	local->new_file_size = conf->orig_offset;
+
+	/*
+	 * The rest of the file is a partial block and, hence,
+	 * should be written via RMW sequence, so the crypt xlator
+	 * does STACK_WIND to itself.
+	 *
+	 * Pass current file size to crypt_writev()
+	 */
+	op_errno = dict_set(dict,
+			    FSIZE_XATTR_PREFIX,
+			    data_from_uint64(local->cur_file_size));
+	if (op_errno) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "can not set key to update file size");
+		dict_unref(dict);
+		goto error;
+	}
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "passing current file size (%llu) to crypt_writev",
+	       (unsigned long long)local->cur_file_size);
+	/*
+	 * Padding will be filled with
+	 * zeros by rmw_partial_block()
+	 */
+	STACK_WIND(frame,
+		   prune_complete,
+		   this,
+		   this->fops->writev, /* crypt_writev */
+		   local->fd,
+		   &local->vec,
+		   1,
+		   conf->aligned_offset, /* offset to write from */
+		   0,
+		   local->iobref,
+		   dict);
+
+	dict_unref(dict);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+ put_one_call:
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * This is called as a callback of ->writev() invoked in behalf
+ * of ftruncate(): it can be
+ * 1) ordered writes issued by hole conversion in the case of
+ *    expanded truncate, or
+ * 2) an rmw partial data block issued by non-cblock-aligned
+ * prune.
+ */
+int32_t end_writeback_ftruncate(call_frame_t *frame,
+				void *cookie,
+				xlator_t *this,
+				int32_t op_ret,
+				int32_t op_errno,
+				struct iatt *prebuf,
+				struct iatt *postbuf,
+				dict_t *xdata)
+{
+	crypt_local_t  *local = frame->local;
+	/*
+	 * if nothing has been written,
+	 * then it must be an error
+	 */
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto put_one_call;
+
+	update_local_file_params(frame, this, prebuf, postbuf);
+
+	if (data_write_in_progress(local))
+		/* case (2) */
+		goto put_one_call;
+	/* case (1) */
+	if (should_resume_submit_hole(local))
+		submit_hole(frame, this);
+	/*
+	 * case of hole, when we should't resume
+	 */
+ put_one_call:
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * Perform prune and write components of the
+ * read-prune-write sequence.
+ *
+ * Called as ->readv_cbk()
+ *
+ * Pre-conditions:
+ * @vec contains the latest atom of the file
+ * (plain text)
+ */
+static int32_t prune_write(call_frame_t *frame,
+			   void *cookie,
+			   xlator_t *this,
+			   int32_t op_ret,
+			   int32_t op_errno,
+			   struct iovec *vec,
+			   int32_t count,
+			   struct iatt *stbuf,
+			   struct iobref *iobref,
+			   dict_t *xdata)
+{
+	int32_t i;
+	size_t to_copy;
+	size_t copied = 0;
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	if (op_ret == -1)
+		goto put_one_call;
+
+	/*
+	 * At first, uptodate head block
+	 */
+	if (iovec_get_size(vec, count) < conf->off_in_head) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "Failed to uptodate head block for prune");
+		local->op_ret = -1;
+		local->op_errno = EIO;
+		goto put_one_call;
+	}
+	local->vec.iov_len = conf->off_in_head;
+	local->vec.iov_base = GF_CALLOC(1, local->vec.iov_len,
+					gf_crypt_mt_data);
+
+	if (local->vec.iov_base == NULL) {
+	        gf_log(this->name, GF_LOG_WARNING,
+                       "Failed to calloc head block for prune");
+		local->op_ret = -1;
+		local->op_errno = ENOMEM;
+	        goto put_one_call;
+	}
+	for (i = 0; i < count; i++) {
+		to_copy = vec[i].iov_len;
+		if (to_copy > local->vec.iov_len - copied)
+			to_copy = local->vec.iov_len - copied;
+
+		memcpy((char *)local->vec.iov_base + copied,
+		       vec[i].iov_base,
+		       to_copy);
+		copied += to_copy;
+		if (copied == local->vec.iov_len)
+			break;
+	}
+	/*
+	 * perform prune with aligned offset
+	 * (i.e. at this step we prune a bit
+	 * more then it is needed
+	 */
+	STACK_WIND(frame,
+		   prune_submit_file_tail,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->ftruncate,
+		   local->fd,
+		   conf->aligned_offset,
+		   local->xdata);
+	return 0;
+ put_one_call:
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * Perform a read-prune-write sequence
+ */
+int32_t read_prune_write(call_frame_t *frame, xlator_t *this)
+{
+	int32_t ret = 0;
+	dict_t *dict = NULL;
+	crypt_local_t  *local = frame->local;
+	struct avec_config *conf = &local->data_conf;
+	struct object_cipher_info *object = &local->info->cinfo;
+
+	set_local_io_params_ftruncate(frame, object);
+	get_one_call_nolock(frame);
+
+	if ((conf->orig_offset & (object_alg_blksize(object) - 1)) == 0) {
+		/*
+		 * cblock-aligned prune:
+		 * we don't need read and write components,
+		 * just cut file body
+		 */
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "prune without RMW (at offset %llu",
+		       (unsigned long long)conf->orig_offset);
+
+		STACK_WIND(frame,
+			   prune_complete,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->ftruncate,
+			   local->fd,
+			   conf->orig_offset,
+			   local->xdata);
+		return 0;
+	}
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "prune with RMW (at offset %llu",
+	       (unsigned long long)conf->orig_offset);	
+	/*
+	 * We are about to perform the "read" component of the
+	 * read-prune-write sequence. It means that we need to
+	 * read encrypted data from disk and decrypt it.
+	 * So, the crypt translator does STACK_WIND to itself.
+	 * 
+	 * Pass current file size to crypt_readv()
+	 
+	 */
+	dict = dict_new();
+	if (!dict) {
+		gf_log("crypt", GF_LOG_WARNING, "Can not alloc dict");
+		ret = ENOMEM;
+		goto exit;
+	}
+	ret = dict_set(dict,
+		       FSIZE_XATTR_PREFIX,
+		       data_from_uint64(local->cur_file_size));
+	if (ret) {
+		gf_log("crypt", GF_LOG_WARNING, "Can not set dict");
+		goto exit;
+	}
+	STACK_WIND(frame,
+		   prune_write,
+		   this,
+		   this->fops->readv, /* crypt_readv */
+		   local->fd,
+		   get_atom_size(object), /* bytes to read */
+		   conf->aligned_offset, /* offset to read from */
+		   0,
+		   dict);
+ exit:
+	if (dict)
+		dict_unref(dict);
+	return ret;
+}
+
+/*
+ * File prune is more complicated than expand.
+ * First we need to read the latest atom to not lose info
+ * needed for proper update. Also we need to make sure that
+ * every component of read-prune-write sequence leaves data
+ * consistent
+ *
+ * Non-cblock aligned prune is performed as read-prune-write
+ * sequence:
+ *
+ * 1) read the latest atom;
+ * 2) perform cblock-aligned prune
+ * 3) issue a write request for the end-of-file
+ */
+int32_t prune_file(call_frame_t *frame, xlator_t *this, uint64_t offset)
+{
+	int32_t ret;
+
+	ret = prepare_for_prune(frame, this, offset);
+	if (ret)
+		return ret;
+	return read_prune_write(frame, this);
+}
+
+int32_t expand_file(call_frame_t *frame, xlator_t *this,
+		    uint64_t offset)
+{
+	int32_t ret;
+	crypt_local_t *local = frame->local;
+
+	ret = prepare_for_submit_hole(frame, this,
+				      local->old_file_size,
+				      offset - local->old_file_size);
+	if (ret)
+		return ret;
+	submit_hole(frame, this);
+	return 0;
+}
+
+static int32_t ftruncate_trivial_completion(call_frame_t *frame,
+					    void *cookie,
+					    xlator_t *this,
+					    int32_t op_ret,
+					    int32_t op_errno,
+					    struct iatt *buf,
+					    dict_t *dict)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	local->prebuf = *buf;
+	local->postbuf = *buf;
+
+	local->prebuf.ia_size = local->cur_file_size;
+	local->postbuf.ia_size = local->cur_file_size;
+
+	get_one_call(frame);
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+static int32_t do_ftruncate(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    dict_t *dict,
+			    dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+
+	if (op_ret)
+		goto error;	
+	/*
+	 * extract regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_errno = EIO;
+		goto error;
+	}
+	local->old_file_size = local->cur_file_size = data_to_uint64(data);
+
+	if (local->data_conf.orig_offset == local->cur_file_size) {
+#if DEBUG_CRYPT
+		gf_log("crypt", GF_LOG_DEBUG,
+		       "trivial ftruncate (current file size %llu)",
+		       (unsigned long long)local->cur_file_size);
+#endif
+		goto trivial;
+	}
+	else if (local->data_conf.orig_offset < local->cur_file_size) {
+#if DEBUG_CRYPT
+		gf_log("crypt", GF_LOG_DEBUG, "prune from %llu to %llu",
+		       (unsigned long long)local->cur_file_size,
+		       (unsigned long long)local->data_conf.orig_offset);
+#endif
+		op_errno = prune_file(frame,
+				      this,
+				      local->data_conf.orig_offset);
+	}
+	else {
+#if DEBUG_CRYPT
+		gf_log("crypt", GF_LOG_DEBUG, "expand from %llu to %llu",
+		       (unsigned long long)local->cur_file_size,
+		       (unsigned long long)local->data_conf.orig_offset);
+#endif
+		op_errno = expand_file(frame,
+				       this,
+				       local->data_conf.orig_offset);
+	}
+	if (op_errno)
+		goto error;
+	return 0;
+ trivial:
+	STACK_WIND(frame,
+		   ftruncate_trivial_completion,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	/*
+	 * finish with ftruncate
+	 */
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+
+	get_one_call_nolock(frame);
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+static int32_t crypt_ftruncate_finodelk_cbk(call_frame_t *frame,
+					    void *cookie,
+					    xlator_t *this,
+					    int32_t op_ret,
+					    int32_t op_errno,
+					    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0)
+		goto error;
+	/*
+	 * An access has been granted,
+	 * retrieve file size first
+	 */
+	STACK_WIND(frame,
+		   do_ftruncate,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;
+ error:
+	get_one_call_nolock(frame);
+	put_one_call_ftruncate(frame, this);
+	return 0;
+}
+
+/*
+ * ftruncate is performed in 2 steps:
+ * . recieve file size;
+ * . expand or prune file.
+ */
+static int32_t crypt_ftruncate(call_frame_t *frame,
+			       xlator_t *this,
+			       fd_t *fd,
+			       off_t offset,
+			       dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+	struct crypt_inode_info *info;
+	struct gf_flock lock = {0, };
+
+	local = crypt_alloc_local(frame, this, GF_FOP_FTRUNCATE);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		ret = ENOMEM;
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+	info = local_get_inode_info(local, this);
+	if (info == NULL) {
+		ret = EINVAL;
+		goto error;
+	}
+	if (!object_alg_atomic(&info->cinfo)) {
+		ret = EINVAL;
+		goto error;
+	}
+	local->data_conf.orig_offset = offset;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_ftruncate_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ error:
+	if (local && local->fd)
+		fd_unref(fd);
+	if (local && local->xdata)
+		dict_unref(xdata);
+	if (local && local->xattr)
+		dict_unref(local->xattr);
+	if (local && local->info)
+		free_inode_info(local->info);
+
+	STACK_UNWIND_STRICT(ftruncate, frame, -1, ret, NULL, NULL, NULL);
+	return 0;
+}
+
+/* ->flush_cbk() */
+int32_t truncate_end(call_frame_t *frame,
+		     void *cookie,
+		     xlator_t *this,
+		     int32_t op_ret,
+		     int32_t op_errno,
+		     dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_UNWIND_STRICT(truncate,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local->xdata);
+	return 0;
+}
+
+/* ftruncate_cbk() */
+int32_t truncate_flush(call_frame_t *frame,
+		       void *cookie,
+		       xlator_t *this,
+		       int32_t op_ret,
+		       int32_t op_errno,
+		       struct iatt *prebuf,
+		       struct iatt *postbuf,
+		       dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *fd = local->fd;
+	local->prebuf = *prebuf;
+	local->postbuf = *postbuf;
+
+	STACK_WIND(frame,
+		   truncate_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   fd,
+		   NULL);
+	fd_unref(fd);
+	return 0;
+}
+
+/*
+ * is called as ->open_cbk()
+ */
+static int32_t truncate_begin(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      fd_t *fd,
+			      dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0) {
+		fd_unref(fd);
+		STACK_UNWIND_STRICT(truncate,
+				    frame,
+				    op_ret,
+				    op_errno, NULL, NULL, NULL);
+		return 0;
+	}
+	/*
+	 * crypt_truncate() is implemented via crypt_ftruncate(),
+	 * so the crypt xlator does STACK_WIND to itself here
+	 */
+	STACK_WIND(frame,
+		   truncate_flush,
+		   this,
+		   this->fops->ftruncate, /* crypt_ftruncate */
+		   fd,
+		   local->offset,
+		   NULL);
+	return 0;
+}
+
+/*
+ * crypt_truncate() is implemented via crypt_ftruncate() as a
+ * sequence crypt_open() - crypt_ftruncate() - truncate_flush()
+ */
+int32_t crypt_truncate(call_frame_t *frame,
+		       xlator_t *this,
+		       loc_t *loc,
+		       off_t offset,
+		       dict_t *xdata)
+{
+	fd_t *fd;
+	crypt_local_t *local;
+
+#if DEBUG_CRYPT
+	gf_log(this->name, GF_LOG_DEBUG,
+	       "truncate file %s at offset %llu",
+	       loc->path, (unsigned long long)offset);
+#endif
+	local = crypt_alloc_local(frame, this, GF_FOP_TRUNCATE);
+	if (!local)
+		goto error;
+
+	fd = fd_create(loc->inode, frame->root->pid);
+	if (!fd) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create fd");
+		goto error;
+	}
+	local->fd = fd;
+	local->offset = offset;
+	local->xdata = xdata;
+	STACK_WIND(frame,
+		   truncate_begin,
+		   this,
+		   this->fops->open, /* crypt_open() */
+		   loc,
+		   O_RDWR,
+		   fd,
+		   NULL);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(truncate, frame, -1, EINVAL, NULL, NULL, NULL);
+	return 0;
+}
+
+end_writeback_handler_t dispatch_end_writeback(glusterfs_fop_t fop)
+{
+	switch (fop) {
+	case GF_FOP_WRITE:
+		return end_writeback_writev;
+	case GF_FOP_FTRUNCATE:
+		return end_writeback_ftruncate;
+	default:
+		gf_log("crypt", GF_LOG_WARNING, "Bad wb operation %d", fop);
+		return NULL;
+	}
+}
+
+/*
+ * true, if the caller needs metadata string
+ */
+static int32_t is_custom_mtd(dict_t *xdata)
+{
+	data_t *data;
+	uint32_t flags;
+
+	if (!xdata)
+		return 0;
+
+	data = dict_get(xdata, MSGFLAGS_PREFIX);
+	if (!data)
+		return 0;
+	if (data->len != sizeof(uint32_t)) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "Bad msgflags size (%d)", data->len);
+		return -1;
+	}
+	flags = *((uint32_t *)data->data);
+	return msgflags_check_mtd_lock(&flags);
+}
+
+static int32_t crypt_open_done(call_frame_t *frame,
+			       void *cookie,
+			       xlator_t *this,
+			       int32_t op_ret,
+			       int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+	if (op_ret < 0)
+		gf_log(this->name, GF_LOG_WARNING, "mtd unlock failed (%d)",
+		       op_errno);
+	put_one_call_open(frame);
+	return 0;
+}
+
+static void crypt_open_tail(call_frame_t *frame, xlator_t *this)
+{
+	struct gf_flock  lock  = {0, };
+	crypt_local_t *local = frame->local;
+
+        lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   crypt_open_done,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+}
+
+/*
+ * load private inode info at open time
+ * called as ->fgetxattr_cbk()
+ */
+static int load_mtd_open(call_frame_t *frame,
+			 void *cookie,
+			 xlator_t *this,
+			 int32_t op_ret,
+			 int32_t op_errno,
+			 dict_t *dict,
+			 dict_t *xdata)
+{
+	int32_t ret;
+	gf_boolean_t upload_info;
+	data_t *mtd;
+	uint64_t value = 0;
+	struct crypt_inode_info *info;
+	crypt_local_t *local = frame->local;
+	crypt_private_t *priv = this->private;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (local->fd->inode->ia_type == IA_IFLNK)
+		goto exit;
+	if (op_ret < 0)
+		goto exit;
+	/*
+	 * first, check for cached info
+	 */
+	ret = inode_ctx_get(local->fd->inode, this, &value);
+	if (ret != -1) {
+		info = (struct crypt_inode_info *)(long)value;
+		if (info == NULL) {
+			gf_log(this->name, GF_LOG_WARNING,
+			       "Inode info expected, but not found");
+			local->op_ret = -1;
+			local->op_errno = EIO;
+			goto exit;
+		}
+		/*
+		 * info has been found in the cache
+		 */
+		upload_info = _gf_false;
+	}
+	else {
+		/*
+		 * info hasn't been found in the cache.
+		 */
+		info = alloc_inode_info(local, local->loc);
+		if (!info) {
+			local->op_ret = -1;
+			local->op_errno = ENOMEM;
+			goto exit;
+		}
+		init_inode_info_head(info, local->fd);
+		upload_info = _gf_true;
+	}
+	/*
+	 * extract metadata
+	 */
+	mtd = dict_get(dict, CRYPTO_FORMAT_PREFIX);
+	if (!mtd) {
+		local->op_ret = -1;
+		local->op_errno = ENOENT;
+		gf_log (this->name, GF_LOG_WARNING,
+			"Format string wasn't found");
+		goto exit;
+	}
+	/*
+	 * authenticate metadata against the path
+	 */	
+	ret = open_format((unsigned char *)mtd->data,
+			  mtd->len,
+			  local->loc,
+			  info,
+			  get_master_cinfo(priv),
+			  local,
+			  upload_info);
+	if (ret) {
+		local->op_ret = -1;
+		local->op_errno = ret;
+		goto exit;
+	}
+	if (upload_info) {
+		ret = init_inode_info_tail(info, get_master_cinfo(priv));
+		if (ret) {
+			local->op_ret = -1;
+			local->op_errno = ret;
+			goto exit;	
+		}
+		ret = inode_ctx_put(local->fd->inode,
+				    this, (uint64_t)(long)info);
+		if (ret == -1) {
+			local->op_ret = -1;
+			local->op_errno = EIO;
+			goto exit;
+		}
+	}
+	if (local->custom_mtd) {
+		/*
+		 * pass the metadata string to the customer
+		 */
+		ret = dict_set_static_bin(local->xdata,
+					  CRYPTO_FORMAT_PREFIX,
+					  mtd->data,
+					  mtd->len);
+		if (ret) {
+			local->op_ret = -1;
+			local->op_errno = ret;
+			goto exit;
+		}
+	}
+ exit:
+	if (!local->custom_mtd)
+		crypt_open_tail(frame, this);
+	else
+		put_one_call_open(frame);
+	return 0;
+}
+
+static int32_t crypt_open_finodelk_cbk(call_frame_t *frame,
+				       void *cookie,
+				       xlator_t *this,
+				       int32_t op_ret,
+				       int32_t op_errno,
+				       dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING, "finodelk (LOCK) failed");
+		goto exit;
+	}
+	STACK_WIND(frame,
+		   load_mtd_open,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fgetxattr,
+		   local->fd,
+		   CRYPTO_FORMAT_PREFIX,
+		   NULL);
+	return 0;
+ exit:
+	put_one_call_open(frame);
+	return 0;
+}
+
+/*
+ * verify metadata against the specified pathname
+ */
+static int32_t crypt_open_cbk(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      fd_t *fd,
+			      dict_t *xdata)
+{
+	struct gf_flock lock = {0, };
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (local->fd->inode->ia_type == IA_IFLNK)
+		goto exit;
+	if (op_ret < 0)
+		goto exit;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	else if (local->custom_mtd){
+		local->xdata = dict_new();
+		if (!local->xdata) {
+			local->op_ret = -1;
+			local->op_errno = ENOMEM;
+			gf_log ("crypt", GF_LOG_ERROR,
+				"Can not get new dict for mtd string");
+			goto exit;
+		}
+	}
+	lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = local->custom_mtd ? F_WRLCK : F_RDLCK;
+        lock.l_whence = SEEK_SET;
+
+	STACK_WIND(frame,
+		   crypt_open_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ exit:
+	put_one_call_open(frame);
+	return 0;
+}
+
+static int32_t crypt_open(call_frame_t *frame,
+			  xlator_t *this,
+			  loc_t *loc,
+			  int32_t flags,
+			  fd_t *fd,
+			  dict_t *xdata)
+{
+	int32_t ret = ENOMEM;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_OPEN);
+	if (!local)
+		goto error;
+	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!local->loc) {
+		ret = ENOMEM;
+		goto error;
+	}
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, loc);
+	if (ret) {
+		GF_FREE(local->loc);
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+
+	ret = is_custom_mtd(xdata);
+	if (ret < 0) {
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+		ret = EINVAL;
+		goto error;
+	}
+	local->custom_mtd = ret;
+
+	if ((flags & O_ACCMODE) == O_WRONLY)
+		/*
+		 * we can't open O_WRONLY, because
+		 * we need to do read-modify-write
+		 */
+		flags = (flags & ~O_ACCMODE) | O_RDWR;
+	/*
+	 * Make sure that out translated offsets
+	 * and counts won't be ignored
+	 */
+	flags &= ~O_APPEND;
+	get_one_call_nolock(frame);
+	STACK_WIND(frame,
+		   crypt_open_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->open,
+		   loc,
+		   flags,
+		   fd,
+		   xdata);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(open,
+			    frame,
+			    -1,
+			    ret,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t init_inode_info_tail(struct crypt_inode_info *info,
+				    struct master_cipher_info *master)
+{
+	int32_t ret;
+	struct object_cipher_info *object = &info->cinfo;
+
+#if DEBUG_CRYPT
+	gf_log("crypt", GF_LOG_DEBUG, "Init inode info for object %s",
+	       uuid_utoa(info->oid));
+#endif
+	ret = data_cipher_algs[object->o_alg][object->o_mode].set_private(info,
+									master);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Set private info failed");
+		return ret;
+	}
+	return 0;
+}
+
+/*
+ * Init inode info at ->create() time
+ */
+static void init_inode_info_create(struct crypt_inode_info *info,
+				   struct master_cipher_info *master,
+				   data_t *data)
+{
+	struct object_cipher_info *object;
+
+	info->nr_minor = CRYPT_XLATOR_ID;
+	memcpy(info->oid, data->data, data->len);
+
+	object = &info->cinfo;
+
+	object->o_alg        = master->m_alg;
+	object->o_mode       = master->m_mode;
+	object->o_block_bits = master->m_block_bits;
+	object->o_dkey_size  = master->m_dkey_size;
+}
+
+static void init_inode_info_head(struct crypt_inode_info *info, fd_t *fd)
+{
+	memcpy(info->oid, fd->inode->gfid, sizeof(uuid_t));
+}
+
+static int32_t crypt_create_done(call_frame_t *frame,
+				 void *cookie,
+				 xlator_t *this,
+				 int32_t op_ret,
+				 int32_t op_errno, dict_t *xdata)
+{
+	crypt_private_t *priv = this->private;
+	crypt_local_t *local = frame->local;
+	struct crypt_inode_info *info = local->info;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	inode_t *local_inode = local->inode;
+
+	if (op_ret < 0) {
+		free_inode_info(info);
+		goto unwind;
+	}
+	op_errno = init_inode_info_tail(info, get_master_cinfo(priv));
+	if (op_errno) {
+		op_ret = -1;
+		free_inode_info(info);
+		goto unwind;
+	}
+	/*
+	 * FIXME: drop major subversion number
+	 */
+	op_ret = inode_ctx_put(local->fd->inode, this, (uint64_t)(long)info);
+	if (op_ret == -1) {
+		op_errno = EIO;
+		free_inode_info(info);
+		goto unwind;
+	}
+ unwind:
+	free_format(local);
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    local_fd,
+			    local_inode,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+	fd_unref(local_fd);
+	inode_unref(local_inode);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	return 0;
+}
+
+static int crypt_create_tail(call_frame_t *frame,
+			     void *cookie,
+			     xlator_t *this,
+			     int32_t op_ret,
+			     int32_t op_errno,
+			     dict_t *xdata)
+{
+	struct gf_flock  lock  = {0, };
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	inode_t *local_inode = local->inode;
+
+	dict_unref(local->xattr);
+
+	if (op_ret < 0)
+		goto error;
+
+        lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   crypt_create_done,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ error:
+	free_inode_info(local->info);
+	free_format(local);
+
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    local_fd,
+			    local_inode,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+
+	fd_unref(local_fd);
+	inode_unref(local_inode);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	return 0;
+}
+
+static int32_t crypt_create_finodelk_cbk(call_frame_t *frame,
+					 void *cookie,
+					 xlator_t *this,
+					 int32_t op_ret,
+					 int32_t op_errno,
+					 dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct crypt_inode_info *info = local->info;
+
+	if (op_ret < 0)
+		goto error;
+
+	STACK_WIND(frame,
+		   crypt_create_tail,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fsetxattr,
+		   local->fd,
+		   local->xattr, /* CRYPTO_FORMAT_PREFIX */
+		   0,
+		   NULL);
+	return 0;
+ error: 
+	free_inode_info(info);
+	free_format(local);
+	fd_unref(local->fd);
+	dict_unref(local->xattr);
+	if (local->xdata)
+		dict_unref(local->xdata);
+
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+/*
+ * Create and store crypt-specific format on disk;
+ * Populate cache with private inode info
+ */
+static int32_t crypt_create_cbk(call_frame_t *frame,
+				void *cookie,
+				xlator_t *this,
+				int32_t op_ret,
+				int32_t op_errno,
+				fd_t *fd,
+				inode_t *inode,
+				struct iatt *buf,
+				struct iatt *preparent,
+				struct iatt *postparent,
+				dict_t *xdata)
+{
+	struct gf_flock lock = {0, };
+	crypt_local_t *local = frame->local;
+	struct crypt_inode_info *info = local->info;
+
+	if (op_ret < 0)
+		goto error;
+	if (xdata)
+		local->xdata = dict_ref(xdata);	
+	local->inode = inode_ref(inode);
+	local->buf = *buf;
+	local->prebuf = *preparent;
+	local->postbuf = *postparent;
+
+        lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+        STACK_WIND(frame,
+		   crypt_create_finodelk_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);	
+	return 0;
+ error:
+	free_inode_info(info);
+	free_format(local);
+	fd_unref(local->fd);
+	dict_unref(local->xattr);
+
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    NULL, NULL, NULL,
+			    NULL, NULL, NULL);
+	return 0;
+}
+
+static int32_t crypt_create(call_frame_t *frame,
+			    xlator_t *this,
+			    loc_t *loc,
+			    int32_t flags,
+			    mode_t mode,
+			    mode_t umask,
+			    fd_t *fd,
+			    dict_t *xdata)
+{
+	int ret;
+	data_t *data;
+	crypt_local_t *local;
+	crypt_private_t *priv;
+	struct master_cipher_info *master;
+	struct crypt_inode_info *info;
+
+	priv = this->private;
+	master = get_master_cinfo(priv);
+
+	if (master_alg_atomic(master)) {
+		/*
+		 * We can't open O_WRONLY, because we
+		 * need to do read-modify-write.
+		 */
+		if ((flags & O_ACCMODE) == O_WRONLY)
+			flags = (flags & ~O_ACCMODE) | O_RDWR;
+		/*
+		 * Make sure that out translated offsets
+		 * and counts won't be ignored
+		 */
+		flags &= ~O_APPEND;
+	}
+	local = crypt_alloc_local(frame, this, GF_FOP_CREATE);
+	if (!local) {
+		ret = ENOMEM;
+		goto error;
+	}
+	data = dict_get(xdata, "gfid-req");
+	if (!data) {
+		ret = EINVAL;
+		gf_log("crypt", GF_LOG_WARNING, "gfid not found");
+		goto error;
+	}
+	if (data->len != sizeof(uuid_t)) {
+		ret = EINVAL;
+		gf_log("crypt", GF_LOG_WARNING,
+		       "bad gfid size (%d), should be %d",
+		       (int)data->len, (int)sizeof(uuid_t));
+		goto error;
+	}
+	info = alloc_inode_info(local, loc);
+	if (!info){
+		ret = ENOMEM;
+		goto error;
+	}
+	/*
+	 * NOTE:
+	 * format has to be created BEFORE
+	 * proceeding to the untrusted server
+	 */
+	ret = alloc_format_create(local);
+	if (ret) {
+		free_inode_info(info);
+		goto error;
+	}
+	init_inode_info_create(info, master, data);
+
+	ret = create_format(local->format,
+			    loc,
+			    info,
+			    master);
+	if (ret) {
+		free_inode_info(info);
+		goto error;		
+	}
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		free_inode_info(info);
+		free_format(local);	
+		goto error;
+	}
+	ret = dict_set_static_bin(local->xattr,
+				  CRYPTO_FORMAT_PREFIX,
+				  local->format,
+				  new_format_size());
+	if (ret) {
+		dict_unref(local->xattr);
+		free_inode_info(info);
+		free_format(local);
+		goto error;
+	}
+	ret = dict_set(local->xattr, FSIZE_XATTR_PREFIX, data_from_uint64(0));
+	if (ret) {
+		dict_unref(local->xattr);
+		free_inode_info(info);
+		free_format(local);
+		goto error;
+	}
+	local->fd = fd_ref(fd);
+
+	STACK_WIND(frame,
+		   crypt_create_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->create,
+		   loc,
+		   flags,
+		   mode,
+		   umask,
+		   fd,
+		   xdata);
+	return 0;
+ error:
+	gf_log("crypt", GF_LOG_WARNING, "can not create file");
+	STACK_UNWIND_STRICT(create,
+			    frame,
+			    -1,
+			    ret,
+			    NULL, NULL, NULL,
+			    NULL, NULL, NULL);
+	return 0;
+}
+
+/*
+ * FIXME: this should depends on the version of format string
+ */
+static int32_t filter_crypt_xattr(dict_t *dict,
+				  char *key, data_t *value, void *data)
+{
+        dict_del(dict, key);
+        return 0;
+}
+
+static int32_t crypt_fsetxattr(call_frame_t *frame,
+			       xlator_t *this,
+			       fd_t *fd,
+			       dict_t *dict,
+			       int32_t flags, dict_t *xdata)
+{
+	dict_foreach_fnmatch(dict, "trusted.glusterfs.crypt*",
+			     filter_crypt_xattr, NULL);
+	STACK_WIND(frame,
+		   default_fsetxattr_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fsetxattr,
+		   fd,
+		   dict,
+		   flags,
+		   xdata);
+	return 0;
+}
+
+/*
+ * TBD: verify file metadata before wind
+ */
+static int32_t crypt_setxattr(call_frame_t *frame,
+			      xlator_t *this,
+			      loc_t *loc,
+			      dict_t *dict,
+			      int32_t flags, dict_t *xdata)
+{
+	dict_foreach_fnmatch(dict, "trusted.glusterfs.crypt*",
+			     filter_crypt_xattr, NULL);
+	STACK_WIND(frame,
+		   default_setxattr_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->setxattr,
+		   loc,
+		   dict,
+		   flags,
+		   xdata);
+	return 0;
+}
+
+/*
+ * called as flush_cbk()
+ */
+static int32_t linkop_end(call_frame_t *frame,
+			  void *cookie,
+			  xlator_t *this,
+			  int32_t op_ret,
+			  int32_t op_errno,
+			  dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	linkop_unwind_handler_t unwind_fn;
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret < 0 &&
+	    op_errno == ENOENT &&
+	    local->loc->inode->ia_type == IA_IFLNK) {
+		local->op_ret = 0;
+		local->op_errno = 0;
+	}
+	unwind_fn(frame);
+	return 0;
+}
+
+/*
+ * unpin inode on the server
+ */
+static int32_t link_flush(call_frame_t *frame,
+			  void *cookie,
+			  xlator_t *this,
+			  int32_t op_ret,
+			  int32_t op_errno,
+			  inode_t *inode,
+			  struct iatt *buf,
+			  struct iatt *preparent,
+			  struct iatt *postparent, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	if (local->xdata) {
+		dict_unref(local->xdata);
+		local->xdata = NULL;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	local->inode = inode_ref(inode);
+	local->buf = *buf;
+	local->prebuf = *preparent;
+	local->postbuf = *postparent;
+
+	STACK_WIND(frame,
+		   linkop_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	link_unwind(frame);
+	return 0;
+}
+
+void link_unwind(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	dict_t *xdata;
+	dict_t *xattr;
+	inode_t *inode;
+
+	if (!local) {
+		STACK_UNWIND_STRICT(link,
+				    frame,
+				    -1,
+				    ENOMEM,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL);
+		return;
+	}
+	xdata = local->xdata;
+	xattr = local->xattr;
+	inode = local->inode;
+
+	if (local->loc){
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	if (local->newloc) {
+		loc_wipe(local->newloc);
+		GF_FREE(local->newloc);
+	}
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->format)
+		GF_FREE(local->format);
+
+	STACK_UNWIND_STRICT(link,
+			    frame,
+			    local->op_ret,
+			    local->op_errno,
+			    inode,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    xdata);
+	if (xdata)
+		dict_unref(xdata);
+	if (xattr)
+		dict_unref(xattr);
+	if (inode)
+		inode_unref(inode);
+}
+
+void link_wind(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_WIND(frame,
+		   link_flush,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->link,
+		   local->loc,
+		   local->newloc,
+		   local->xdata);
+}
+
+/*
+ * unlink()
+ */
+static int32_t unlink_flush(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    struct iatt *preparent,
+			    struct iatt *postparent, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	local->prebuf = *preparent;
+	local->postbuf = *postparent;
+	if (local->xdata) {
+		dict_unref(local->xdata);
+		local->xdata = NULL;
+	}
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	STACK_WIND(frame,
+		   linkop_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	unlink_unwind(frame);
+	return 0;
+}
+
+void unlink_unwind(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	dict_t *xdata;
+	dict_t *xattr;
+
+	if (!local) {
+		STACK_UNWIND_STRICT(unlink,
+				    frame,
+				    -1,
+				    ENOMEM,
+				    NULL,
+				    NULL,
+				    NULL);
+		return;
+	}
+	xdata = local->xdata;
+	xattr = local->xattr;
+	if (local->loc){
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->format)
+		GF_FREE(local->format);
+
+	STACK_UNWIND_STRICT(unlink,
+			    frame,
+			    local->op_ret,
+			    local->op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    xdata);
+	if (xdata)
+		dict_unref(xdata);
+	if (xattr)
+		dict_unref(xattr);
+}
+
+void unlink_wind(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_WIND(frame,
+		   unlink_flush,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->unlink,
+		   local->loc,
+		   local->flags,
+		   local->xdata);
+}
+
+void rename_unwind(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	dict_t *xdata;
+	dict_t *xattr;
+	struct iatt *prenewparent;
+	struct iatt *postnewparent;
+
+	if (!local) {
+		STACK_UNWIND_STRICT(rename,
+				    frame,
+				    -1,
+				    ENOMEM,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL,
+				    NULL);	
+		return;
+	}
+	xdata = local->xdata;
+	xattr = local->xattr;
+	prenewparent = local->prenewparent;
+	postnewparent = local->postnewparent;	
+
+	if (local->loc){
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	if (local->newloc){
+		loc_wipe(local->newloc);
+		GF_FREE(local->newloc);
+	}
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->format)
+		GF_FREE(local->format);
+
+	STACK_UNWIND_STRICT(rename,
+			    frame,
+			    local->op_ret,
+			    local->op_errno,
+			    &local->buf,
+			    &local->prebuf,
+			    &local->postbuf,
+			    prenewparent,
+			    postnewparent,
+			    xdata);
+	if (xdata)
+		dict_unref(xdata);
+	if (xattr)
+		dict_unref(xattr);
+	if (prenewparent)
+		GF_FREE(prenewparent);
+	if (postnewparent)
+		GF_FREE(postnewparent);
+}
+
+/*
+ * called as flush_cbk()
+ */
+static int32_t rename_end(call_frame_t *frame,
+			  void *cookie,
+			  xlator_t *this,
+			  int32_t op_ret,
+			  int32_t op_errno,
+			  dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	rename_unwind(frame);
+	return 0;
+}
+
+static int32_t rename_flush(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    struct iatt *buf,
+			    struct iatt *preoldparent,
+			    struct iatt *postoldparent,
+			    struct iatt *prenewparent,
+			    struct iatt *postnewparent,
+			    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto error;
+	dict_unref(local->xdata);
+	local->xdata = NULL;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	local->buf = *buf;
+	local->prebuf = *preoldparent;
+	local->postbuf = *postoldparent;
+	if (prenewparent) {
+		local->prenewparent = GF_CALLOC(1, sizeof(*prenewparent),
+						gf_crypt_mt_iatt);
+		if (!local->prenewparent) {
+			op_errno = ENOMEM;
+			goto error;
+		}
+		*local->prenewparent = *prenewparent;
+	}
+	if (postnewparent) {
+		local->postnewparent = GF_CALLOC(1, sizeof(*postnewparent),
+						 gf_crypt_mt_iatt);
+		if (!local->postnewparent) {
+			op_errno = ENOMEM;
+			goto error;
+		}
+		*local->postnewparent = *postnewparent;
+	}
+	STACK_WIND(frame,
+		   rename_end,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->flush,
+		   local->fd,
+		   NULL);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	rename_unwind(frame);
+	return 0;
+}
+
+void rename_wind(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+
+	STACK_WIND(frame,
+		   rename_flush,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->rename,
+		   local->loc,
+		   local->newloc,
+		   local->xdata);
+}
+
+static int32_t __do_linkop(call_frame_t *frame,
+			   void *cookie,
+			   xlator_t *this,
+			   int32_t op_ret,
+			   int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	linkop_wind_handler_t wind_fn;
+	linkop_unwind_handler_t unwind_fn;
+
+	wind_fn = linkop_wind_dispatch(local->fop);
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if (op_ret >= 0)
+		wind_fn(frame, this);
+	else {
+		gf_log(this->name, GF_LOG_WARNING, "mtd unlock failed (%d)",
+		       op_errno);
+		unwind_fn(frame);
+	}
+	return 0;
+}
+
+static int32_t do_linkop(call_frame_t *frame,
+			 void *cookie,
+			 xlator_t *this,
+			 int32_t op_ret,
+			 int32_t op_errno,
+			 dict_t *xdata)
+{
+	struct gf_flock  lock  = {0, };
+	crypt_local_t *local = frame->local;
+	linkop_unwind_handler_t unwind_fn;
+
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+	local->op_ret = op_ret;
+	local->op_errno = op_errno;
+
+	if(op_ret < 0)
+		goto error;
+
+	lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   __do_linkop,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+ error:		
+	unwind_fn(frame);
+	return 0;
+}
+
+/*
+ * Update the metadata string (against the new pathname);
+ * submit the result
+ */
+static int32_t linkop_begin(call_frame_t *frame,
+			    void *cookie,
+			    xlator_t *this,
+			    int32_t op_ret,
+			    int32_t op_errno,
+			    fd_t *fd,
+			    dict_t *xdata)
+{
+	gf_boolean_t upload_info;
+	crypt_local_t *local = frame->local;
+	crypt_private_t *priv = this->private;
+	struct crypt_inode_info *info;
+	data_t *old_mtd;
+	uint32_t new_mtd_size;
+	uint64_t value = 0;
+	void (*unwind_fn)(call_frame_t *frame);
+	void (*wind_fn)(call_frame_t *frame, xlator_t *this);
+	mtd_op_t mop;
+
+	wind_fn = linkop_wind_dispatch(local->fop);
+	unwind_fn = linkop_unwind_dispatch(local->fop);
+	mop = linkop_mtdop_dispatch(local->fop);
+
+	if (local->fd->inode->ia_type == IA_IFLNK)
+		goto wind;
+	if (op_ret < 0)
+		/*
+		 * verification failed
+		 */
+		goto error;
+
+	old_mtd = dict_get(xdata, CRYPTO_FORMAT_PREFIX);
+	if (!old_mtd) {
+		op_errno = EIO;
+		gf_log (this->name, GF_LOG_DEBUG,
+			"Metadata string wasn't found");
+		goto error;
+	}
+	new_mtd_size = format_size(mop, old_mtd->len);
+	op_errno = alloc_format(local, new_mtd_size);
+	if (op_errno)
+		goto error;
+	/*
+	 * check for cached info 
+	 */
+	op_ret = inode_ctx_get(fd->inode, this, &value);
+	if (op_ret != -1) {
+		info = (struct crypt_inode_info *)(long)value;
+		if (info == NULL) {
+			gf_log (this->name, GF_LOG_WARNING,
+				"Inode info was not found");
+			op_errno = EINVAL;
+			goto error;
+		}
+		/*
+		 * info was found in the cache
+		 */
+		local->info = info;
+		upload_info = _gf_false;
+	}
+	else {
+		/*
+		 * info wasn't found in the cache;
+		 */
+		info = alloc_inode_info(local, local->loc);
+		if (!info)
+			goto error;
+		init_inode_info_head(info, fd);
+		local->info = info;
+		upload_info = _gf_true;
+	}
+	op_errno = open_format((unsigned char *)old_mtd->data,
+			       old_mtd->len,
+			       local->loc,
+			       info,
+			       get_master_cinfo(priv),
+			       local,
+			       upload_info);
+	if (op_errno)
+		goto error;
+	if (upload_info == _gf_true) {
+		op_errno = init_inode_info_tail(info,
+						get_master_cinfo(priv));
+		if (op_errno)
+			goto error;
+		op_errno = inode_ctx_put(fd->inode, this,
+					 (uint64_t)(long)(info));
+		if (op_errno == -1) {
+			op_errno = EIO;
+			goto error;
+		}
+	}
+	/*
+	 * update the format string (append/update/cup a MAC)
+	 */
+	op_errno = update_format(local->format,
+				 (unsigned char *)old_mtd->data,
+				 old_mtd->len,
+				 local->mac_idx,
+				 mop,
+				 local->newloc,
+				 info,
+				 get_master_cinfo(priv),
+				 local);
+	if (op_errno)
+		goto error;
+	/*
+	 * store the new format string on the server
+	 */
+	if (new_mtd_size) {
+		op_errno = dict_set_static_bin(local->xattr,
+					       CRYPTO_FORMAT_PREFIX,
+					       local->format,
+					       new_mtd_size);
+		if (op_errno)
+			goto error;
+	}
+	STACK_WIND(frame,
+		   do_linkop,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->setxattr,
+		   local->loc,
+		   local->xattr,
+		   0,
+		   NULL);
+	return 0;
+ wind:
+	wind_fn(frame, this);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = op_errno;
+	unwind_fn(frame);
+	return 0;
+}
+
+static int32_t linkop_grab_local(call_frame_t *frame,
+				 xlator_t *this,
+				 loc_t *oldloc,
+				 loc_t *newloc,
+				 int flags, dict_t *xdata,
+				 glusterfs_fop_t op)
+{
+	int32_t ret = ENOMEM;
+	fd_t *fd;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, op);
+	if (!local)
+		goto error;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	fd = fd_create(oldloc->inode, frame->root->pid);
+	if (!fd) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create fd");
+		goto error;
+	}
+	local->fd = fd;
+	local->flags = flags;
+	local->loc = GF_CALLOC(1, sizeof(*oldloc), gf_crypt_mt_loc);
+	if (!local->loc)
+		goto error;
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, oldloc);
+	if (ret) {
+		GF_FREE(local->loc);
+		local->loc = NULL;
+		goto error;
+	}
+	if (newloc) {
+		local->newloc = GF_CALLOC(1, sizeof(*newloc), gf_crypt_mt_loc);
+		if (!local->newloc) {
+			loc_wipe(local->loc);
+			GF_FREE(local->loc);
+			goto error;
+		}
+		memset(local->newloc, 0, sizeof(*local->newloc));
+		ret = loc_copy(local->newloc, newloc);
+		if (ret) {
+			loc_wipe(local->loc);
+			GF_FREE(local->loc);
+			GF_FREE(local->newloc);
+			goto error;
+		}
+	}
+	local->xattr = dict_new();
+	if (!local->xattr) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create dict");
+		ret = ENOMEM;
+		goto error;
+	}
+	return 0;
+
+error:
+        if (local) {
+                if (local->xdata)
+                        dict_unref(local->xdata);
+                if (local->fd)
+                        fd_unref(local->fd);
+                local->fd = 0;
+                local->loc = NULL;
+                local->newloc = NULL;
+                local->op_ret = -1;
+                local->op_errno = ret;
+        }
+
+        return ret;
+}
+
+/*
+ * read and verify locked metadata against the old pathname (via open);
+ * update the metadata string in accordance with the new pathname;
+ * submit modified metadata;
+ * wind;
+ */
+static int32_t linkop(call_frame_t *frame,
+		      xlator_t *this,
+		      loc_t *oldloc,
+		      loc_t *newloc,
+		      int flags,
+		      dict_t *xdata,
+		      glusterfs_fop_t op)
+{
+	int32_t ret;
+	dict_t *dict;
+	crypt_local_t *local;
+	void (*unwind_fn)(call_frame_t *frame);
+
+	unwind_fn = linkop_unwind_dispatch(op);
+
+	ret = linkop_grab_local(frame, this, oldloc, newloc, flags, xdata, op);
+	local = frame->local;
+	if (ret)
+		goto error;
+	dict = dict_new();
+	if (!dict) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not create dict");
+		ret = ENOMEM;
+		goto error;
+	}
+	/*
+	 * Set a message to crypt_open() that we need
+	 * locked metadata string.
+	 * All link operations (link, unlink, rename)
+	 * need write lock
+	 */
+	msgflags_set_mtd_wlock(&local->msgflags);
+	ret = dict_set_static_bin(dict,
+				  MSGFLAGS_PREFIX,
+				  &local->msgflags,
+				  sizeof(local->msgflags));
+	if (ret) {
+		gf_log(this->name, GF_LOG_ERROR, "Can not set dict");
+		dict_unref(dict);
+		goto error;
+	}
+	/*
+	 * verify metadata against the old pathname
+	 * and retrieve locked metadata string
+	 */
+	STACK_WIND(frame,
+		   linkop_begin,
+		   this,
+		   this->fops->open, /* crypt_open() */
+		   oldloc,
+		   O_RDWR,
+		   local->fd,
+		   dict);
+	dict_unref(dict);
+	return 0;
+ error:
+	local->op_ret = -1;
+	local->op_errno = ret;
+	unwind_fn(frame);
+	return 0;
+}
+
+static int32_t crypt_link(call_frame_t *frame, xlator_t *this,
+			  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+	return linkop(frame, this, oldloc, newloc, 0, xdata, GF_FOP_LINK);
+}
+
+static int32_t crypt_unlink(call_frame_t *frame, xlator_t *this,
+			    loc_t *loc, int flags, dict_t *xdata)
+{
+	return linkop(frame, this, loc, NULL, flags, xdata, GF_FOP_UNLINK);
+}
+
+static int32_t crypt_rename(call_frame_t *frame, xlator_t *this,
+			    loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+	return linkop(frame, this, oldloc, newloc, 0, xdata, GF_FOP_RENAME);
+}
+
+static void put_one_call_open(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local)) {
+		fd_t *fd = local->fd;
+		loc_t *loc = local->loc;
+		dict_t *xdata = local->xdata;
+
+		STACK_UNWIND_STRICT(open,
+				    frame,
+				    local->op_ret,
+				    local->op_errno,
+				    fd,
+				    xdata);
+		fd_unref(fd);
+		if (xdata)
+			dict_unref(xdata);
+		loc_wipe(loc);
+		GF_FREE(loc);
+	}
+}
+
+static int32_t __crypt_readv_done(call_frame_t *frame,
+				  void *cookie,
+				  xlator_t *this,
+				  int32_t op_ret,
+				  int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	/* read deals with data configs only */
+	struct iovec *avec = local->data_conf.avec;
+	char **pool = local->data_conf.pool;
+	int blocks_in_pool = local->data_conf.blocks_in_pool;
+	struct iobref *iobref = local->iobref;
+	struct iobref *iobref_data = local->iobref_data;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "readv unlock failed (%d)", op_errno);
+		if (local->op_ret >= 0) {
+			local->op_ret = op_ret;
+			local->op_errno = op_errno;
+		}
+	}
+	dump_plain_text(local, avec);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "readv: ret_to_user: %d, iovec len: %d, ia_size: %llu",
+	       (int)(local->rw_count > 0 ? local->rw_count : local->op_ret),
+	       (int)(local->rw_count > 0 ? iovec_get_size(avec, local->data_conf.acount) : 0),
+	       (unsigned long long)local->buf.ia_size);
+
+	STACK_UNWIND_STRICT(readv,
+			    frame,
+			    local->rw_count > 0 ? local->rw_count : local->op_ret,
+			    local->op_errno,
+			    avec,
+			    avec ? local->data_conf.acount : 0,
+			    &local->buf,
+			    local->iobref,
+			    local_xdata);
+
+	free_avec(avec, pool, blocks_in_pool);
+	fd_unref(local_fd);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	if (iobref)
+		iobref_unref(iobref);
+	if (iobref_data)
+		iobref_unref(iobref_data);
+	return 0;
+}
+
+static void crypt_readv_done(call_frame_t *frame, xlator_t *this)
+{
+	if (parent_is_crypt_xlator(frame, this))
+		/*
+		 *  don't unlock (it will be done by the parent)
+		 */
+		__crypt_readv_done(frame, NULL, this, 0, 0, NULL);
+	else {
+		crypt_local_t *local = frame->local;
+		struct gf_flock  lock  = {0, };
+
+		lock.l_type   = F_UNLCK;
+		lock.l_whence = SEEK_SET;
+		lock.l_start  = 0;
+		lock.l_len    = 0;
+		lock.l_pid    = 0;
+
+		STACK_WIND(frame,
+			   __crypt_readv_done,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->finodelk,
+			   this->name,
+			   local->fd,
+			   F_SETLKW,
+			   &lock,
+			   NULL);
+	}
+}
+
+static void put_one_call_readv(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local))
+		crypt_readv_done(frame, this);
+}
+
+static int32_t __crypt_writev_done(call_frame_t *frame,
+				   void *cookie,
+				   xlator_t *this,
+				   int32_t op_ret,
+				   int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	int32_t ret_to_user;
+
+	if (local->xattr)
+		dict_unref(local->xattr);
+	/*
+	 * Calculate amout of butes to be returned
+	 * to user. We need to subtract paddings that
+	 * have been written as a part of atom.
+	 */
+	/*
+	 * subtract head padding
+	 */
+	if (local->rw_count == 0)
+		/*
+		 * Nothing has been written, it must be an error
+		 */
+		ret_to_user = local->op_ret;
+	else if (local->rw_count <= local->data_conf.off_in_head) {
+		gf_log("crypt", GF_LOG_WARNING, "Incomplete write");
+		ret_to_user = 0;
+	}
+	else
+		ret_to_user = local->rw_count -
+			local->data_conf.off_in_head;
+	/*
+	 * subtract tail padding
+	 */
+	if (ret_to_user > local->data_conf.orig_size)
+		ret_to_user = local->data_conf.orig_size;
+
+	if (local->iobref)
+		iobref_unref(local->iobref);
+	if (local->iobref_data)
+		iobref_unref(local->iobref_data);
+	free_avec_data(local);
+	free_avec_hole(local);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "writev: ret_to_user: %d", ret_to_user);
+
+	STACK_UNWIND_STRICT(writev,
+			    frame,
+			    ret_to_user,
+			    local->op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+	fd_unref(local_fd);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	return 0;
+}
+
+static int32_t crypt_writev_done(call_frame_t *frame,
+				 void *cookie,
+				 xlator_t *this,
+				 int32_t op_ret,
+				 int32_t op_errno,
+				 dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		gf_log("crypt", GF_LOG_WARNING, "can not update file size");
+
+	if (parent_is_crypt_xlator(frame, this))
+		/*
+		 * don't unlock (it will be done by the parent)
+		 */
+		__crypt_writev_done(frame, NULL, this, 0, 0, NULL);
+	else {
+		struct gf_flock  lock  = {0, };
+
+		lock.l_type   = F_UNLCK;
+		lock.l_whence = SEEK_SET;
+		lock.l_start  = 0;
+		lock.l_len    = 0;
+		lock.l_pid    = 0;
+
+		STACK_WIND(frame,
+			   __crypt_writev_done,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->finodelk,
+			   this->name,
+			   local->fd,
+			   F_SETLKW,
+			   &lock,
+			   NULL);
+	}
+	return 0;
+}
+
+static void put_one_call_writev(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local)) {
+		if (local->update_disk_file_size) {
+			int32_t ret;
+			/*
+			 * update file size, unlock the file and unwind
+			 */
+			ret = dict_set(local->xattr,
+				       FSIZE_XATTR_PREFIX,
+				       data_from_uint64(local->cur_file_size));
+			if (ret) {
+				gf_log("crypt", GF_LOG_WARNING,
+				       "can not set key to update file size");
+				crypt_writev_done(frame, NULL,
+						  this, 0, 0, NULL);
+				return;
+			}
+			gf_log("crypt", GF_LOG_DEBUG,
+			       "Updating disk file size to %llu",
+			       (unsigned long long)local->cur_file_size);
+			STACK_WIND(frame,
+				   crypt_writev_done,
+				   FIRST_CHILD(this),
+				   FIRST_CHILD(this)->fops->fsetxattr,
+				   local->fd,
+				   local->xattr, /* CRYPTO_FORMAT_PREFIX */
+				   0,
+				   NULL);
+		}
+		else
+			crypt_writev_done(frame, NULL, this, 0, 0, NULL);
+	}
+}
+
+static int32_t __crypt_ftruncate_done(call_frame_t *frame,
+				      void *cookie,
+				      xlator_t *this,
+				      int32_t op_ret,
+				      int32_t op_errno, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	fd_t *local_fd = local->fd;
+	dict_t *local_xdata = local->xdata;
+	char *iobase = local->vec.iov_base;
+
+	if (op_ret < 0) {
+		gf_log(this->name, GF_LOG_WARNING,
+		       "ftruncate unlock failed (%d)", op_errno);
+		if (local->op_ret >= 0) {
+			local->op_ret = op_ret;
+			local->op_errno = op_errno;
+		}
+	}
+	if (local->iobref_data)
+		iobref_unref(local->iobref_data);
+	free_avec_data(local);
+	free_avec_hole(local);
+
+	gf_log("crypt", GF_LOG_DEBUG,
+	       "ftruncate, return to user: presize=%llu, postsize=%llu",
+	       (unsigned long long)local->prebuf.ia_size,
+	       (unsigned long long)local->postbuf.ia_size);
+
+	STACK_UNWIND_STRICT(ftruncate,
+			    frame,
+			    local->op_ret < 0 ? -1 : 0,
+			    local->op_errno,
+			    &local->prebuf,
+			    &local->postbuf,
+			    local_xdata);
+	fd_unref(local_fd);
+	if (local_xdata)
+		dict_unref(local_xdata);
+	if (iobase)
+		GF_FREE(iobase);
+	return 0;
+}
+
+static int32_t crypt_ftruncate_done(call_frame_t *frame,
+				    void *cookie,
+				    xlator_t *this,
+				    int32_t op_ret,
+				    int32_t op_errno,
+				    dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+	struct gf_flock  lock  = {0, };
+
+	dict_unref(local->xattr);
+	if (op_ret < 0)
+		gf_log("crypt", GF_LOG_WARNING, "can not update file size");
+
+	lock.l_type   = F_UNLCK;
+	lock.l_whence = SEEK_SET;
+	lock.l_start  = 0;
+	lock.l_len    = 0;
+	lock.l_pid    = 0;
+
+	STACK_WIND(frame,
+		   __crypt_ftruncate_done,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->finodelk,
+		   this->name,
+		   local->fd,
+		   F_SETLKW,
+		   &lock,
+		   NULL);
+	return 0;
+}
+
+static void put_one_call_ftruncate(call_frame_t *frame, xlator_t *this)
+{
+	crypt_local_t *local = frame->local;
+	if (put_one_call(local)) {
+		if (local->update_disk_file_size) {
+			int32_t ret;
+			/*
+			 * update file size, unlock the file and unwind
+			 */
+			ret = dict_set(local->xattr,
+				       FSIZE_XATTR_PREFIX,
+				       data_from_uint64(local->cur_file_size));
+			if (ret) {
+				gf_log("crypt", GF_LOG_WARNING,
+				       "can not set key to update file size");
+				crypt_ftruncate_done(frame, NULL,
+						     this, 0, 0, NULL);
+				return;
+			}
+			gf_log("crypt", GF_LOG_DEBUG,
+			       "Updating disk file size to %llu",
+			       (unsigned long long)local->cur_file_size);
+			STACK_WIND(frame,
+				   crypt_ftruncate_done,
+				   FIRST_CHILD(this),
+				   FIRST_CHILD(this)->fops->fsetxattr,
+				   local->fd,
+				   local->xattr, /* CRYPTO_FORMAT_PREFIX */
+				   0,
+				   NULL);
+		}
+		else
+			crypt_ftruncate_done(frame, NULL, this, 0, 0, NULL);
+	}
+}
+
+/*
+ * load regular file size for some FOPs
+ */
+static int32_t load_file_size(call_frame_t *frame,
+			      void *cookie,
+			      xlator_t *this,
+			      int32_t op_ret,
+			      int32_t op_errno,
+			      dict_t *dict,
+			      dict_t *xdata)
+{
+	data_t *data;
+	crypt_local_t *local = frame->local;
+
+	dict_t *local_xdata = local->xdata;
+	inode_t *local_inode = local->inode;
+
+	if (op_ret < 0)
+		goto unwind;
+	/*
+	 * load regular file size
+	 */
+	data = dict_get(dict, FSIZE_XATTR_PREFIX);
+	if (!data) {
+		if (local->xdata)
+			dict_unref(local->xdata);
+		gf_log("crypt", GF_LOG_WARNING, "Regular file size not found");
+		op_ret = -1;
+		op_errno = EIO;
+		goto unwind;
+	}
+	local->buf.ia_size = data_to_uint64(data);
+
+	gf_log(this->name, GF_LOG_DEBUG,
+	       "FOP %d: Translate regular file to %llu",
+	       local->fop,
+	       (unsigned long long)local->buf.ia_size);
+ unwind:	
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->loc) {
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	switch (local->fop) {
+	case GF_FOP_FSTAT:
+		STACK_UNWIND_STRICT(fstat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? &local->buf : NULL,
+				    local->xdata);
+		break;
+	case GF_FOP_STAT:
+		STACK_UNWIND_STRICT(stat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? &local->buf : NULL,
+				    local->xdata);
+		break;
+	case GF_FOP_LOOKUP:
+		STACK_UNWIND_STRICT(lookup,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? local->inode : NULL,
+				    op_ret >= 0 ? &local->buf : NULL,				    
+				    local->xdata,
+				    op_ret >= 0 ? &local->postbuf : NULL);
+		break;
+	case GF_FOP_READ:
+		STACK_UNWIND_STRICT(readv,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    NULL,
+				    0,
+				    op_ret >= 0 ? &local->buf : NULL,
+				    NULL,
+				    NULL);
+		break;
+	default:
+		gf_log(this->name, GF_LOG_WARNING,
+		       "Improper file operation %d", local->fop);
+	}
+	if (local_xdata)
+		dict_unref(local_xdata);
+	if (local_inode)
+		inode_unref(local_inode);
+	return 0;
+}
+
+static int32_t crypt_stat_common_cbk(call_frame_t *frame,
+				     void *cookie,
+				     xlator_t *this,
+				     int32_t op_ret,
+				     int32_t op_errno,
+				     struct iatt *buf, dict_t *xdata)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto unwind;
+	if (!IA_ISREG(buf->ia_type))
+		goto unwind;
+
+	local->buf = *buf;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+
+	switch (local->fop) {
+	case GF_FOP_FSTAT:
+		STACK_WIND(frame,
+			   load_file_size,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->fgetxattr,
+			   local->fd,
+			   FSIZE_XATTR_PREFIX,
+			   NULL);
+		break;
+	case GF_FOP_STAT:
+		STACK_WIND(frame,
+			   load_file_size,
+			   FIRST_CHILD(this),
+			   FIRST_CHILD(this)->fops->getxattr,
+			   local->loc,
+			   FSIZE_XATTR_PREFIX,
+			   NULL);
+		break;
+	default:
+		gf_log (this->name, GF_LOG_WARNING,
+			"Improper file operation %d", local->fop);
+	}
+	return 0;
+ unwind:
+	if (local->fd)
+		fd_unref(local->fd);
+	if (local->loc) {
+		loc_wipe(local->loc);
+		GF_FREE(local->loc);
+	}
+	switch (local->fop) {
+	case GF_FOP_FSTAT:
+		STACK_UNWIND_STRICT(fstat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? buf : NULL,
+				    op_ret >= 0 ? xdata : NULL);
+		break;
+	case GF_FOP_STAT:
+		STACK_UNWIND_STRICT(stat,
+				    frame,
+				    op_ret,
+				    op_errno,
+				    op_ret >= 0 ? buf : NULL,
+				    op_ret >= 0 ? xdata : NULL);
+		break;
+	default:
+		gf_log (this->name, GF_LOG_WARNING,
+			"Improper file operation %d", local->fop);
+	}
+	return 0;
+}
+
+static int32_t crypt_fstat(call_frame_t *frame,
+			   xlator_t *this,
+			   fd_t *fd, dict_t *xdata)
+{
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_FSTAT);
+	if (!local)
+		goto error;
+	local->fd = fd_ref(fd);
+	STACK_WIND(frame,
+		   crypt_stat_common_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->fstat,
+		   fd,
+		   xdata);	
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(fstat,
+			    frame,
+			    -1,
+			    ENOMEM,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t crypt_stat(call_frame_t *frame,
+			  xlator_t *this,
+			  loc_t *loc, dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_STAT);
+	if (!local)
+		goto error;
+	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!local->loc)
+		goto error;
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, loc);
+	if (ret) {
+		GF_FREE(local->loc);
+		goto error;
+	}
+	STACK_WIND(frame,
+		   crypt_stat_common_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->stat,
+		   loc,
+		   xdata);	
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(stat,
+			    frame,
+			    -1,
+			    ENOMEM,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+static int32_t crypt_lookup_cbk(call_frame_t *frame,
+				void *cookie,
+				xlator_t *this,
+				int32_t op_ret,
+				int32_t op_errno,
+				inode_t *inode,
+				struct iatt *buf, dict_t *xdata,
+				struct iatt *postparent)
+{
+	crypt_local_t *local = frame->local;
+
+	if (op_ret < 0)
+		goto unwind;
+	if (!IA_ISREG(buf->ia_type))
+		goto unwind;
+
+	local->inode = inode_ref(inode);
+	local->buf = *buf;
+	local->postbuf = *postparent;
+	if (xdata)
+		local->xdata = dict_ref(xdata);
+	uuid_copy(local->loc->gfid, buf->ia_gfid);
+
+	STACK_WIND(frame,
+		   load_file_size,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->getxattr,
+		   local->loc,
+		   FSIZE_XATTR_PREFIX,
+		   NULL);
+	return 0;
+ unwind:
+	loc_wipe(local->loc);
+	GF_FREE(local->loc);
+	STACK_UNWIND_STRICT(lookup,
+			    frame,
+			    op_ret,
+			    op_errno,
+			    inode,
+			    buf,
+			    xdata,
+			    postparent);
+	return 0;
+}
+
+static int32_t crypt_lookup(call_frame_t *frame,
+			    xlator_t *this,
+			    loc_t *loc, dict_t *xdata)
+{
+	int32_t ret;
+	crypt_local_t *local;
+
+	local = crypt_alloc_local(frame, this, GF_FOP_LOOKUP);
+	if (!local)
+		goto error;
+	local->loc = GF_CALLOC(1, sizeof(*loc), gf_crypt_mt_loc);
+	if (!local->loc)
+		goto error;
+	memset(local->loc, 0, sizeof(*local->loc));
+	ret = loc_copy(local->loc, loc);
+	if (ret) {
+		GF_FREE(local->loc);
+		goto error;
+	}
+	gf_log(this->name, GF_LOG_DEBUG, "Lookup %s", loc->path);
+	STACK_WIND(frame,
+		   crypt_lookup_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->lookup,
+		   loc,
+		   xdata);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(lookup,
+			    frame,
+			    -1,
+			    ENOMEM,
+			    NULL,
+			    NULL,
+			    NULL,
+			    NULL);
+	return 0;
+}
+
+/*
+ * for every regular directory entry find its real file size
+ * and update stat's buf properly
+ */
+static int32_t crypt_readdirp_cbk(call_frame_t *frame,
+				  void *cookie,
+				  xlator_t *this,
+				  int32_t op_ret,
+				  int32_t op_errno,
+				  gf_dirent_t *entries, dict_t *xdata)
+{
+	gf_dirent_t *entry = NULL;
+
+	if (op_ret < 0)
+		goto unwind;
+
+	list_for_each_entry (entry, (&entries->list), list) {
+		data_t *data;
+
+		if (!IA_ISREG(entry->d_stat.ia_type))
+			continue;
+		data = dict_get(entry->dict, FSIZE_XATTR_PREFIX);
+		if (!data){
+			gf_log("crypt", GF_LOG_WARNING,
+			       "Regular file size of direntry not found");
+			op_errno = EIO;
+			op_ret = -1;
+			break;
+		}
+		entry->d_stat.ia_size = data_to_uint64(data);
+	}
+ unwind:
+	STACK_UNWIND_STRICT(readdirp, frame, op_ret, op_errno, entries, xdata);
+	return 0;
+}
+
+/*
+ * ->readdirp() fills in-core inodes, so we need to set proper
+ * file sizes for all directory entries of the parent @fd.
+ * Actual updates take place in ->crypt_readdirp_cbk()
+ */
+static int32_t crypt_readdirp(call_frame_t *frame, xlator_t *this,
+			      fd_t *fd, size_t size, off_t offset,
+			      dict_t *xdata)
+{
+	int32_t ret = ENOMEM;
+
+	if (!xdata) {
+		xdata = dict_new();
+		if (!xdata)
+			goto error;
+	}
+	else
+		dict_ref(xdata);
+	/*
+	 * make sure that we'll have real file sizes at ->readdirp_cbk()
+	 */
+	ret = dict_set(xdata, FSIZE_XATTR_PREFIX, data_from_uint64(0));
+	if (ret) {
+		dict_unref(xdata);
+		goto error;
+	}
+	STACK_WIND(frame,
+		   crypt_readdirp_cbk,
+		   FIRST_CHILD(this),
+		   FIRST_CHILD(this)->fops->readdirp,
+		   fd,
+		   size,
+		   offset,
+		   xdata);
+	dict_unref(xdata);
+	return 0;
+ error:
+	STACK_UNWIND_STRICT(readdirp, frame, -1, ret, NULL, NULL);
+	return 0;
+}
+
+static int32_t crypt_access(call_frame_t *frame,
+			    xlator_t *this,
+			    loc_t *loc,
+			    int32_t mask, dict_t *xdata)
+{
+	gf_log(this->name, GF_LOG_WARNING,
+	       "NFS mounts of encrypted volumes are unsupported");
+	STACK_UNWIND_STRICT(access, frame, -1, EPERM, NULL);
+	return 0;
+}
+
+int32_t master_set_block_size (xlator_t *this, crypt_private_t *priv,
+			       dict_t *options)
+{
+	uint64_t block_size = 0;
+	struct master_cipher_info *master = get_master_cinfo(priv);
+
+	if (options != NULL)
+		GF_OPTION_RECONF("block-size", block_size, options,
+				 size_uint64, error);
+	else
+		GF_OPTION_INIT("block-size", block_size, size_uint64, error);
+
+	switch (block_size) {
+	case 512:
+		master->m_block_bits = 9;
+		break;
+	case 1024:
+		master->m_block_bits = 10;
+		break;
+	case 2048:
+		master->m_block_bits = 11;
+		break;
+	case 4096:
+		master->m_block_bits = 12;
+		break;
+	default:
+		gf_log("crypt", GF_LOG_ERROR,
+		       "FATAL: unsupported block size %llu",
+		       (unsigned long long)block_size);
+		goto error;
+	}
+	return 0;
+ error:
+	return -1;
+}
+
+int32_t master_set_alg(xlator_t *this, crypt_private_t *priv)
+{
+	struct master_cipher_info *master = get_master_cinfo(priv);
+	master->m_alg = AES_CIPHER_ALG;
+	return 0;
+}
+
+int32_t master_set_mode(xlator_t *this, crypt_private_t *priv)
+{
+	struct master_cipher_info *master = get_master_cinfo(priv);
+	master->m_mode = XTS_CIPHER_MODE;
+	return 0;
+}
+
+/*
+ * set key size in bits to the master info
+ * Pre-conditions: cipher mode in the master info is uptodate.
+ */
+static int master_set_data_key_size (xlator_t *this, crypt_private_t *priv,
+				     dict_t *options)
+{
+	int32_t ret;
+	uint64_t key_size = 0;
+	struct master_cipher_info *master = get_master_cinfo(priv);
+
+	if (options != NULL)
+		GF_OPTION_RECONF("data-key-size", key_size, options,
+				 uint64, error);
+	else
+		GF_OPTION_INIT("data-key-size", key_size, uint64, error);
+
+	ret = data_cipher_algs[master->m_alg][master->m_mode].check_key(key_size);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "FATAL: wrong bin key size %llu for alg %d mode %d",
+		       (unsigned long long)key_size,
+		       (int)master->m_alg,
+		       (int)master->m_mode);
+		goto error;
+	}
+	master->m_dkey_size = key_size;
+	return 0;
+ error:
+	return -1;
+}
+
+static int is_hex(char *s) {
+	return  ('0' <= *s && *s <= '9') || ('a' <= *s && *s <= 'f');
+}
+
+static int parse_hex_buf(xlator_t *this, char *src, unsigned char *dst,
+			 int hex_size)
+{
+	int i;
+	int hex_byte = 0;
+
+	for (i = 0; i < (hex_size / 2); i++) {
+		if (!is_hex(src + i*2) || !is_hex(src + i*2 + 1)) {
+			gf_log("crypt", GF_LOG_ERROR,
+			       "FATAL: not hex symbol in key");
+			return -1;
+		}
+		if (sscanf(src + i*2, "%2x", &hex_byte) != 1) {
+			gf_log("crypt", GF_LOG_ERROR,
+			       "FATAL: can not parse hex key");
+			return -1;
+		}
+		dst[i] = hex_byte & 0xff;
+	}
+	return 0;
+}
+
+/*
+ * Parse options;
+ * install master volume key
+ */
+int32_t master_set_master_vol_key(xlator_t *this, crypt_private_t *priv)
+{
+	int32_t ret;
+	FILE *file = NULL;
+
+	int32_t key_size;
+	char *opt_key_file_pathname = NULL;
+
+	unsigned char bin_buf[MASTER_VOL_KEY_SIZE];
+	char hex_buf[2 * MASTER_VOL_KEY_SIZE];
+
+	struct master_cipher_info *master = get_master_cinfo(priv);
+	/*
+	 * extract master key passed via option
+	 */
+	GF_OPTION_INIT("master-key", opt_key_file_pathname, path, bad_key);
+
+	if (!opt_key_file_pathname) {
+		gf_log(this->name, GF_LOG_ERROR, "FATAL: missing master key");
+		return -1;
+	}
+	gf_log(this->name, GF_LOG_DEBUG, "handling file key %s",
+	       opt_key_file_pathname);
+
+	file = fopen(opt_key_file_pathname, "r");
+	if (file == NULL) {
+		gf_log(this->name, GF_LOG_ERROR,
+		       "FATAL: can not open file with master key");
+		return -1;
+	}
+	/*
+	 * extract hex key
+	 */
+	key_size = fread(hex_buf, 1, sizeof(hex_buf), file);
+	if (key_size < sizeof(hex_buf)) {
+		gf_log(this->name, GF_LOG_ERROR,
+		       "FATAL: master key is too short");
+		goto bad_key;
+	}
+	ret = parse_hex_buf(this, hex_buf, bin_buf, key_size);
+	if (ret)
+		goto bad_key;
+	memcpy(master->m_key, bin_buf, MASTER_VOL_KEY_SIZE);
+	memset(hex_buf, 0, sizeof(hex_buf));
+	fclose(file);
+
+ 	memset(bin_buf, 0, sizeof(bin_buf));
+	return 0;
+ bad_key:
+	gf_log(this->name, GF_LOG_ERROR, "FATAL: bad master key");
+	if (file)
+		fclose(file);
+	memset(bin_buf, 0, sizeof(bin_buf));
+	return -1;
+}
+
+/*
+ * Derive volume key for object-id authentication
+ */
+int32_t master_set_nmtd_vol_key(xlator_t *this, crypt_private_t *priv)
+{
+	return get_nmtd_vol_key(get_master_cinfo(priv));
+}
+
+int32_t crypt_init_xlator(xlator_t *this)
+{
+	int32_t ret;
+	crypt_private_t *priv = this->private; 
+
+	ret = master_set_alg(this, priv);
+	if (ret)
+		return ret;
+	ret = master_set_mode(this, priv);
+	if (ret)
+		return ret;
+	ret = master_set_block_size(this, priv, NULL);
+	if (ret)
+		return ret;
+	ret = master_set_data_key_size(this, priv, NULL);
+	if (ret)
+		return ret;
+	ret = master_set_master_vol_key(this, priv);
+	if (ret)
+		return ret;
+	return master_set_nmtd_vol_key(this, priv);
+}
+
+static int32_t crypt_alloc_private(xlator_t *this)
+{
+	this->private = GF_CALLOC(1, sizeof(crypt_private_t), gf_crypt_mt_priv);
+	if (!this->private) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "Can not allocate memory for private data");
+		return ENOMEM;
+	}
+	return 0;
+}
+
+static void crypt_free_private(xlator_t *this)
+{
+	crypt_private_t *priv = this->private;
+	if (priv) {
+		memset(priv, 0, sizeof(*priv));
+		GF_FREE(priv);
+	}
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_crypt_mt_end);
+
+        if (ret != 0) {
+                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
+                       "failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+int32_t reconfigure (xlator_t *this, dict_t *options)
+{
+	int32_t ret = -1;
+	crypt_private_t *priv = NULL;
+
+	GF_VALIDATE_OR_GOTO ("crypt", this, error);
+	GF_VALIDATE_OR_GOTO (this->name, this->private, error);
+	GF_VALIDATE_OR_GOTO (this->name, options, error);
+
+	priv = this->private;
+
+	ret = master_set_block_size(this, priv, options);
+	if (ret) {
+		gf_log("this->name", GF_LOG_ERROR,
+		       "Failed to reconfure block size");
+		goto error;
+	}
+	ret = master_set_data_key_size(this, priv, options);
+	if (ret) {
+		gf_log("this->name", GF_LOG_ERROR,
+		       "Failed to reconfure data key size");
+		goto error;
+	}
+	return 0;
+ error:
+	return ret;
+}
+
+int32_t init(xlator_t *this)
+{
+	int32_t ret;
+
+	if (!this->children || this->children->next) {
+		gf_log ("crypt", GF_LOG_ERROR,
+			"FATAL: crypt should have exactly one child");
+		return EINVAL;
+	}
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+	ret = crypt_alloc_private(this);
+	if (ret)
+		return ret;
+	ret = crypt_init_xlator(this);
+	if (ret)
+		goto error;
+	this->local_pool = mem_pool_new(crypt_local_t, 64);
+        if (!this->local_pool) {
+		gf_log(this->name, GF_LOG_ERROR,
+		       "failed to create local_t's memory pool");
+		ret = ENOMEM;
+                goto error;
+        }
+	gf_log ("crypt", GF_LOG_INFO, "crypt xlator loaded");
+	return 0;
+ error:
+	crypt_free_private(this);
+	return ret;
+}
+
+void fini (xlator_t *this)
+{
+	crypt_free_private(this);
+}
+
+struct xlator_fops fops = {
+	.readv        = crypt_readv,
+	.writev       = crypt_writev,
+	.truncate     = crypt_truncate,
+	.ftruncate    = crypt_ftruncate,
+	.setxattr     = crypt_setxattr,
+	.fsetxattr    = crypt_fsetxattr,
+	.link         = crypt_link,
+	.unlink       = crypt_unlink,
+	.rename       = crypt_rename,
+	.open         = crypt_open,
+	.create       = crypt_create,
+	.stat         = crypt_stat,
+	.fstat        = crypt_fstat,
+	.lookup       = crypt_lookup,
+	.readdirp     = crypt_readdirp,
+	.access       = crypt_access   
+};
+
+struct xlator_cbks cbks = {
+	.forget       = crypt_forget
+};
+
+struct volume_options options[] = {
+	{ .key = {"master-key"},
+	  .type = GF_OPTION_TYPE_PATH,
+	  .description = "Pathname of regular file which contains master volume key"
+	},
+	{ .key = {"data-key-size"},
+	  .type = GF_OPTION_TYPE_SIZET,
+	  .description = "Data key size (bits)",
+	  .min = 256,
+	  .max = 512,
+	  .default_value = "256",
+	},
+	{ .key = {"block-size"},
+	  .type = GF_OPTION_TYPE_SIZET,
+	  .description = "Atom size (bits)",
+	  .min = 512,
+	  .max = 4096,
+	  .default_value = "4096"
+	},
+	{ .key  = {NULL} },
+};
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/crypt.h b/xlators/encryption/crypt/src/crypt.h
new file mode 100644
index 000000000..eb7291f13
--- /dev/null
+++ b/xlators/encryption/crypt/src/crypt.h
@@ -0,0 +1,908 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CRYPT_H__
+#define __CRYPT_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+#include <openssl/aes.h>
+#include <openssl/evp.h>
+#include <openssl/sha.h>
+#include <openssl/hmac.h>
+#include <openssl/cmac.h>
+#include <openssl/modes.h>
+#include "crypt-mem-types.h"
+#include "compat.h"
+
+#define CRYPT_XLATOR_ID  (0)
+
+#define MAX_IOVEC_BITS (3)
+#define MAX_IOVEC (1 << MAX_IOVEC_BITS)
+#define KEY_FACTOR_BITS  (6)
+
+#define DEBUG_CRYPT (0)
+#define TRIVIAL_TFM (0)
+
+#define CRYPT_MIN_BLOCK_BITS (9)
+#define CRYPT_MAX_BLOCK_BITS (12)
+
+#define MASTER_VOL_KEY_SIZE (32)
+#define NMTD_VOL_KEY_SIZE (16)
+
+#if defined(__NetBSD__)
+typedef off_t loff_t;
+#endif
+
+#if defined(GF_DARWIN_HOST_OS)
+typedef uint64_t loff_t;
+#endif
+
+struct crypt_key {
+	uint32_t len;
+	const char *label;
+};
+
+/*
+ * Add new key types to the end of this
+ * enumeration but before LAST_KEY_TYPE
+ */
+typedef enum {
+	MASTER_VOL_KEY,
+	NMTD_VOL_KEY,
+	NMTD_LINK_KEY,
+	EMTD_FILE_KEY,
+	DATA_FILE_KEY_256,
+	DATA_FILE_KEY_512,
+	LAST_KEY_TYPE
+}crypt_key_type;
+
+struct kderive_context {
+	const unsigned char *pkey;/* parent key */
+	uint32_t pkey_len;        /* parent key size, bits */
+	uint32_t ckey_len;        /* child key size, bits */
+	unsigned char *fid;       /* fixed input data, NIST 800-108, 5.1 */
+	uint32_t fid_len;         /* fid len, bytes */
+	unsigned char *out;       /* contains child keying material */
+	uint32_t out_len;         /* out len, bytes */
+};
+
+typedef enum {
+	DATA_ATOM,
+	HOLE_ATOM,
+	LAST_DATA_TYPE
+}atom_data_type;
+
+typedef enum {
+	HEAD_ATOM,
+	TAIL_ATOM,
+	FULL_ATOM,
+	LAST_LOCALITY_TYPE
+}atom_locality_type;
+
+typedef enum {
+	MTD_CREATE,
+	MTD_APPEND,
+	MTD_OVERWRITE,
+	MTD_CUT,
+	MTD_LAST_OP
+} mtd_op_t;
+
+struct xts128_context {
+	void      *key1, *key2;
+	block128_f block1,block2;
+};
+
+struct object_cipher_info {
+	cipher_alg_t  o_alg;
+        cipher_mode_t o_mode;
+	uint32_t      o_block_bits;
+        uint32_t      o_dkey_size; /* raw data key size in bits */
+	union {
+		struct {
+			unsigned char ivec[16];
+			AES_KEY dkey[2];
+			AES_KEY tkey;    /* key used for tweaking */
+			XTS128_CONTEXT xts;
+		} aes_xts;
+	} u;
+};
+
+struct master_cipher_info {
+	/*
+	 * attributes inherited by newly created regular files
+	 */
+	cipher_alg_t  m_alg;
+        cipher_mode_t m_mode;
+	uint32_t      m_block_bits;
+        uint32_t      m_dkey_size; /* raw key size in bits */
+	/*
+	 * master key
+	 */
+	unsigned char m_key[MASTER_VOL_KEY_SIZE];
+	/*
+	 * volume key for oid authentication
+	 */
+	unsigned char m_nmtd_key[NMTD_VOL_KEY_SIZE];
+};
+
+/*
+* This info is not changed during file's life
+ */
+struct crypt_inode_info {
+#if DEBUG_CRYPT
+	loc_t *loc; /* pathname that the file has been
+		       opened, or created with */
+#endif
+	uint16_t nr_minor;
+	uuid_t oid;
+	struct object_cipher_info cinfo;
+};
+
+/*
+ * this should locate in secure memory
+ */
+typedef struct {
+	struct master_cipher_info master;
+} crypt_private_t;
+
+static inline struct master_cipher_info *get_master_cinfo(crypt_private_t *priv)
+{
+	return &priv->master;
+}
+
+static inline struct object_cipher_info *get_object_cinfo(struct crypt_inode_info
+							 *info)
+{
+	return &info->cinfo;
+}
+
+/*
+ * this describes layouts and properties
+ * of atoms in an aligned vector
+ */
+struct avec_config {
+	uint32_t atom_size;
+	atom_data_type type;
+	size_t orig_size;
+	off_t orig_offset;
+	size_t expanded_size;
+	off_t aligned_offset;
+
+	uint32_t off_in_head;
+	uint32_t off_in_tail;
+	uint32_t gap_in_tail;
+	uint32_t nr_full_blocks;
+
+	struct iovec *avec; /* aligned vector */
+	uint32_t acount; /* number of avec components. The same
+			  * as number of occupied logical blocks */
+	char **pool;
+	uint32_t blocks_in_pool;
+	uint32_t cursor; /* makes sense only for ordered writes,
+			  * so there is no races on this counter.
+			  *
+			  * Cursor is per-config object, we don't
+			  * reset cursor for atoms of different
+			  * localities (head, tail, full)
+			  */
+};
+
+
+typedef struct {
+	glusterfs_fop_t fop; /* code of FOP this local info built for */
+	fd_t *fd;
+	inode_t *inode;
+	loc_t *loc;
+	int32_t mac_idx;
+	loc_t *newloc;
+	int32_t flags;
+	int32_t wbflags;
+	struct crypt_inode_info *info;
+	struct iobref *iobref;
+	struct iobref *iobref_data;
+	off_t offset;
+
+	uint64_t old_file_size; /* per FOP, retrieved under lock held */
+	uint64_t cur_file_size; /* per iteration, before issuing IOs */
+	uint64_t new_file_size; /* per iteration, after issuing IOs */
+
+	uint64_t io_offset; /* offset of IOs issued per iteration */
+	uint64_t io_offset_nopad; /* offset of user's data in the atom */
+	uint32_t io_size; /* size of IOs issued per iteration */
+	uint32_t io_size_nopad; /* size of user's data in the IOs */
+	uint32_t eof_padding_size; /* size od EOF padding in the IOs */
+
+	gf_lock_t call_lock; /* protect nr_calls from many cbks */
+	int32_t nr_calls;
+
+	atom_data_type active_setup; /* which setup (hole or date)
+					is currently active */
+	/* data setup */
+	struct avec_config data_conf;
+
+	/* hole setup */
+	int hole_conv_in_proggress;
+	gf_lock_t hole_lock; /* protect hole config from many cbks */
+	int hole_handled;
+	struct avec_config hole_conf;
+	struct iatt buf;
+	struct iatt prebuf;
+	struct iatt postbuf;
+	struct iatt *prenewparent;
+	struct iatt *postnewparent;
+	int32_t op_ret;
+	int32_t op_errno;
+	int32_t rw_count; /* total read or written */
+	gf_lock_t rw_count_lock; /* protect the counter above */
+	unsigned char *format; /* for create, update format string */
+	uint32_t format_size;
+	uint32_t msgflags; /* messages for crypt_open() */
+	dict_t *xdata;
+	dict_t *xattr;
+	struct iovec vec; /* contains last file's atom for
+			     read-prune-write sequence */
+	gf_boolean_t custom_mtd;
+	/*
+	 * the next 3 fields are used by readdir and friends
+	 */
+	gf_dirent_t *de; /* directory entry */
+	char *de_path; /* pathname of directory entry */
+	uint32_t de_prefix_len; /* lenght of the parent's pathname */
+	gf_dirent_t *entries;
+
+	uint32_t update_disk_file_size:1;
+} crypt_local_t;
+
+/* This represents a (read)modify-write atom */
+struct rmw_atom {
+	atom_locality_type locality;
+	/*
+	 * read-modify-write sequence of the atom
+	 */
+	int32_t (*rmw)(call_frame_t *frame,
+		       void *cookie,
+		       xlator_t *this,
+		       int32_t op_ret,
+		       int32_t op_errno,
+		       struct iovec *vec,
+		       int32_t count,
+		       struct iatt *stbuf,
+		       struct iobref *iobref,
+		       dict_t *xdata);
+	/*
+	 * offset of the logical block in a file
+	 */
+	loff_t (*offset_at)(call_frame_t *frame,
+			    struct object_cipher_info *object);
+	/*
+	 * IO offset in an atom
+	 */
+	uint32_t (*offset_in)(call_frame_t *frame,
+			      struct object_cipher_info *object);
+	/*
+	 * number of bytes of plain text of this atom that user
+	 * wants to read/write.
+	 * It can be smaller than atom_size in the case of head
+	 * or tail atoms.
+	 */
+	uint32_t (*io_size_nopad)(call_frame_t *frame,
+				  struct object_cipher_info *object);
+	/*
+	 * which iovec represents the atom
+	 */
+	struct iovec *(*get_iovec)(call_frame_t *frame, uint32_t count);
+	/*
+	 * how many bytes of partial block should be uptodated by
+	 * reading from disk.
+	 * This is used to perform a read component of RMW (read-modify-write).
+	 */
+	uint32_t (*count_to_uptodate)(call_frame_t *frame, struct object_cipher_info *object);
+	struct avec_config *(*get_config)(call_frame_t *frame);
+};
+
+struct data_cipher_alg {
+	gf_boolean_t atomic; /* true means that algorithm requires
+				to pad data before cipher transform */
+	gf_boolean_t should_pad; /* true means that algorithm requires
+				    to pad the end of file with extra-data */
+	uint32_t blkbits; /* blksize = 1 << blkbits */
+	/*
+	 * any preliminary sanity checks goes here
+	 */
+	int32_t (*init)(void);
+	/*
+	 * set alg-mode specific inode info
+	 */
+	int32_t (*set_private)(struct crypt_inode_info *info,
+			       struct master_cipher_info *master);
+	/*
+	 * check alg-mode specific data key
+	 */
+	int32_t (*check_key)(uint32_t key_size);
+	void (*set_iv)(off_t offset, struct object_cipher_info *object);
+	int32_t (*encrypt)(const unsigned char *from, unsigned char *to,
+			   size_t length, off_t offset, const int enc,
+			   struct object_cipher_info *object);
+};
+
+/*
+ * version-dependent metadata loader
+ */
+struct crypt_mtd_loader {
+	/*
+	 * return core format size
+	 */
+	size_t (*format_size)(mtd_op_t op, size_t old_size);
+	/*
+	 * pack version-specific metadata of an object
+	 * at ->create()
+	 */
+	int32_t (*create_format)(unsigned char *wire,
+				 loc_t *loc,
+				 struct crypt_inode_info *info,
+				 struct master_cipher_info *master);
+	/*
+	 * extract version-specific metadata of an object
+	 * at ->open() time
+	 */
+	int32_t (*open_format)(unsigned char *wire,
+			       int32_t len,
+			       loc_t *loc,
+			       struct crypt_inode_info *info,
+			       struct master_cipher_info *master,
+			       crypt_local_t *local,
+			       gf_boolean_t load_info);
+	int32_t (*update_format)(unsigned char *new,
+				 unsigned char *old,
+				 size_t old_len,
+				 int32_t mac_idx,
+				 mtd_op_t op,
+				 loc_t *loc,
+				 struct crypt_inode_info *info,
+				 struct master_cipher_info *master,
+				 crypt_local_t *local);
+};
+
+typedef int32_t (*end_writeback_handler_t)(call_frame_t *frame,
+					   void *cookie,
+					   xlator_t *this,
+					   int32_t op_ret,
+					   int32_t op_errno,
+					   struct iatt *prebuf,
+					   struct iatt *postbuf,
+					   dict_t *xdata);
+typedef void (*linkop_wind_handler_t)(call_frame_t *frame, xlator_t *this);
+typedef void (*linkop_unwind_handler_t)(call_frame_t *frame);
+
+
+/* Declarations */
+
+/* keys.c */
+extern struct crypt_key crypt_keys[LAST_KEY_TYPE];
+int32_t get_nmtd_vol_key(struct master_cipher_info *master);
+int32_t get_nmtd_link_key(loc_t *loc,
+			  struct master_cipher_info *master,
+			  unsigned char *result);
+int32_t get_emtd_file_key(struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  unsigned char *result);
+int32_t get_data_file_key(struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  uint32_t keysize,
+			  unsigned char *key);
+/* data.c */
+extern struct data_cipher_alg data_cipher_algs[LAST_CIPHER_ALG][LAST_CIPHER_MODE];
+void encrypt_aligned_iov(struct object_cipher_info *object,
+			 struct iovec *vec,
+			 int count,
+			 off_t off);
+void decrypt_aligned_iov(struct object_cipher_info *object,
+			 struct iovec *vec,
+			 int count,
+			 off_t off);
+int32_t align_iov_by_atoms(xlator_t *this,
+			   crypt_local_t *local,
+			   struct object_cipher_info *object,
+			   struct iovec *vec /* input vector */,
+			   int32_t count /* number of vec components */,
+			   struct iovec *avec /* aligned vector */,
+			   char **blocks /* pool of blocks */,
+			   uint32_t *blocks_allocated,
+			   struct avec_config *conf);
+int32_t set_config_avec_data(xlator_t *this,
+			     crypt_local_t *local,
+			     struct avec_config *conf,
+			     struct object_cipher_info *object,
+			     struct iovec *vec,
+			     int32_t vec_count);
+int32_t set_config_avec_hole(xlator_t *this,
+			     crypt_local_t *local,
+			     struct avec_config *conf,
+			     struct object_cipher_info *object,
+			     glusterfs_fop_t fop);
+void set_gap_at_end(call_frame_t *frame, struct object_cipher_info *object,
+		    struct avec_config *conf, atom_data_type dtype);
+void set_config_offsets(call_frame_t *frame,
+			xlator_t *this,
+			uint64_t offset,
+			uint64_t count,
+			atom_data_type dtype,
+			int32_t setup_gap_in_tail);
+
+/* metadata.c */
+extern struct crypt_mtd_loader mtd_loaders [LAST_MTD_LOADER];
+
+int32_t alloc_format(crypt_local_t *local, size_t size);
+int32_t alloc_format_create(crypt_local_t *local);
+void free_format(crypt_local_t *local);
+size_t format_size(mtd_op_t op, size_t old_size);
+size_t new_format_size(void);
+int32_t open_format(unsigned char *str, int32_t len, loc_t *loc,
+		    struct crypt_inode_info *info,
+		    struct master_cipher_info *master, crypt_local_t *local,
+		    gf_boolean_t load_info);
+int32_t update_format(unsigned char *new, unsigned char *old,
+		      size_t old_len, int32_t mac_idx, mtd_op_t op, loc_t *loc,
+		      struct crypt_inode_info *info,
+		      struct master_cipher_info *master,
+		      crypt_local_t *local);
+int32_t create_format(unsigned char *wire,
+		      loc_t *loc,
+		      struct crypt_inode_info *info,
+		      struct master_cipher_info *master);
+
+/* atom.c */
+struct rmw_atom *atom_by_types(atom_data_type data,
+			       atom_locality_type locality);
+void submit_partial(call_frame_t *frame,
+		    xlator_t *this,
+		    fd_t *fd,
+		    atom_locality_type ltype);
+void submit_full(call_frame_t *frame, xlator_t *this);
+
+/* crypt.c */
+
+end_writeback_handler_t dispatch_end_writeback(glusterfs_fop_t fop);
+static size_t iovec_get_size(struct iovec *vec, uint32_t count);
+void set_local_io_params_writev(call_frame_t *frame,
+				struct object_cipher_info *object,
+				struct rmw_atom *atom, off_t io_offset,
+				uint32_t io_size);
+void link_wind(call_frame_t *frame, xlator_t *this);
+void unlink_wind(call_frame_t *frame, xlator_t *this);
+void link_unwind(call_frame_t *frame);
+void unlink_unwind(call_frame_t *frame);
+void rename_wind(call_frame_t *frame, xlator_t *this);
+void rename_unwind(call_frame_t *frame);
+
+/* Inline functions */
+
+static inline size_t iovec_get_size(struct iovec *vec, uint32_t count)
+{
+	int i;
+	size_t size = 0;
+	for (i = 0; i < count; i++)
+		size += vec[i].iov_len;
+	return size;
+}
+
+static inline int32_t crypt_xlator_id(void)
+{
+	return CRYPT_XLATOR_ID;
+}
+
+static inline mtd_loader_id current_mtd_loader(void)
+{
+	return MTD_LOADER_V1;
+}
+
+static inline uint32_t master_key_size (void)
+{
+	return crypt_keys[MASTER_VOL_KEY].len >> 3;
+}
+
+static inline uint32_t nmtd_vol_key_size (void)
+{
+	return crypt_keys[NMTD_VOL_KEY].len >> 3;
+}
+
+static inline uint32_t alg_mode_blkbits(cipher_alg_t alg,
+					cipher_mode_t mode)
+{
+	return data_cipher_algs[alg][mode].blkbits;
+}
+
+static inline uint32_t alg_mode_blksize(cipher_alg_t alg,
+					cipher_mode_t mode)
+{
+	return 1 << alg_mode_blkbits(alg, mode);
+}
+
+static inline gf_boolean_t alg_mode_atomic(cipher_alg_t alg,
+					   cipher_mode_t mode)
+{
+	return data_cipher_algs[alg][mode].atomic;
+}
+
+static inline gf_boolean_t alg_mode_should_pad(cipher_alg_t alg,
+					       cipher_mode_t mode)
+{
+	return data_cipher_algs[alg][mode].should_pad;
+}
+
+static inline uint32_t master_alg_blksize(struct master_cipher_info *mr)
+{
+	return alg_mode_blksize(mr->m_alg, mr->m_mode);
+}
+
+static inline uint32_t master_alg_blkbits(struct master_cipher_info *mr)
+{
+	return alg_mode_blkbits(mr->m_alg, mr->m_mode);
+}
+
+static inline gf_boolean_t master_alg_atomic(struct master_cipher_info *mr)
+{
+	return alg_mode_atomic(mr->m_alg, mr->m_mode);
+}
+
+static inline gf_boolean_t master_alg_should_pad(struct master_cipher_info *mr)
+{
+	return alg_mode_should_pad(mr->m_alg, mr->m_mode);
+}
+
+static inline uint32_t object_alg_blksize(struct object_cipher_info *ob)
+{
+	return alg_mode_blksize(ob->o_alg, ob->o_mode);
+}
+
+static inline uint32_t object_alg_blkbits(struct object_cipher_info *ob)
+{
+	return alg_mode_blkbits(ob->o_alg, ob->o_mode);
+}
+
+static inline gf_boolean_t object_alg_atomic(struct object_cipher_info *ob)
+{
+	return alg_mode_atomic(ob->o_alg, ob->o_mode);
+}
+
+static inline gf_boolean_t object_alg_should_pad(struct object_cipher_info *ob)
+{
+	return alg_mode_should_pad(ob->o_alg, ob->o_mode);
+}
+
+static inline uint32_t aes_raw_key_size(struct master_cipher_info *master)
+{
+	return master->m_dkey_size >> 3;
+}
+
+static inline struct avec_config *get_hole_conf(call_frame_t *frame)
+{
+	return &(((crypt_local_t *)frame->local)->hole_conf);
+}
+
+static inline struct avec_config *get_data_conf(call_frame_t *frame)
+{
+	return &(((crypt_local_t *)frame->local)->data_conf);
+}
+
+static inline int32_t get_atom_bits (struct object_cipher_info *object)
+{
+	return object->o_block_bits;
+}
+
+static inline int32_t get_atom_size (struct object_cipher_info *object)
+{
+	return 1 << get_atom_bits(object);
+}
+
+static inline int32_t has_head_block(struct avec_config *conf)
+{
+	return conf->off_in_head ||
+		(conf->acount == 1 && conf->off_in_tail);
+}
+
+static inline int32_t has_tail_block(struct avec_config *conf)
+{
+	return conf->off_in_tail && conf->acount > 1;
+}
+
+static inline int32_t has_full_blocks(struct avec_config *conf)
+{
+	return conf->nr_full_blocks;
+}
+
+static inline int32_t should_submit_head_block(struct avec_config *conf)
+{
+	return has_head_block(conf) && (conf->cursor == 0);
+}
+
+static inline int32_t should_submit_tail_block(struct avec_config *conf)
+{
+	return has_tail_block(conf) && (conf->cursor == conf->acount - 1);
+}
+
+static inline int32_t should_submit_full_block(struct avec_config *conf)
+{
+	uint32_t start = has_head_block(conf) ? 1 : 0;
+
+	return has_full_blocks(conf) &&
+		conf->cursor >= start &&
+		conf->cursor < start + conf->nr_full_blocks;
+}
+
+#if DEBUG_CRYPT
+static inline void crypt_check_input_len(size_t len,
+					 struct object_cipher_info *object)
+{
+	if (object_alg_should_pad(object) && (len & (object_alg_blksize(object) - 1)))
+		gf_log ("crypt", GF_LOG_DEBUG, "bad input len: %d", (int)len);
+}
+
+static inline void check_head_block(struct avec_config *conf)
+{
+	if (!has_head_block(conf))
+		gf_log("crypt", GF_LOG_DEBUG, "not a head atom");
+}
+
+static inline void check_tail_block(struct avec_config *conf)
+{
+	if (!has_tail_block(conf))
+		gf_log("crypt", GF_LOG_DEBUG, "not a tail atom");
+}
+
+static inline void check_full_block(struct avec_config *conf)
+{
+	if (!has_full_blocks(conf))
+		gf_log("crypt", GF_LOG_DEBUG, "not a full atom");
+}
+
+static inline void check_cursor_head(struct avec_config *conf)
+{
+	if (!has_head_block(conf))
+		gf_log("crypt",
+		       GF_LOG_DEBUG, "Illegal call of head atom method");
+	else if (conf->cursor != 0)
+		gf_log("crypt",
+		       GF_LOG_DEBUG, "Cursor (%d) is not at head atom",
+		       conf->cursor);
+}
+
+static inline void check_cursor_full(struct avec_config *conf)
+{
+	if (!has_full_blocks(conf))
+		gf_log("crypt",
+		       GF_LOG_DEBUG, "Illegal call of full atom method");
+	if (has_head_block(conf) && (conf->cursor == 0))
+		gf_log("crypt",
+		       GF_LOG_DEBUG, "Cursor is not at full atom");
+}
+
+/*
+ * FIXME: use avec->iov_len to check setup
+ */
+static inline int data_local_invariant(crypt_local_t *local)
+{
+	return 0;
+}
+
+#else
+#define crypt_check_input_len(len, object) noop
+#define check_head_block(conf) noop
+#define check_tail_block(conf) noop
+#define check_full_block(conf) noop
+#define check_cursor_head(conf) noop
+#define check_cursor_full(conf) noop
+
+#endif /* DEBUG_CRYPT */
+
+static inline struct avec_config *conf_by_type(call_frame_t *frame,
+					       atom_data_type dtype)
+{
+	struct avec_config *conf = NULL;
+
+	switch (dtype) {
+	case HOLE_ATOM:
+		conf = get_hole_conf(frame);
+		break;
+	case DATA_ATOM:
+		conf = get_data_conf(frame);
+		break;
+	default:
+		gf_log("crypt", GF_LOG_DEBUG, "bad atom type");
+	}
+	return conf;
+}
+
+static inline uint32_t nr_calls_head(struct avec_config *conf)
+{
+	return has_head_block(conf) ? 1 : 0;
+}
+
+static inline uint32_t nr_calls_tail(struct avec_config *conf)
+{
+	return has_tail_block(conf) ? 1 : 0;
+}
+
+static inline uint32_t nr_calls_full(struct avec_config *conf)
+{
+	switch(conf->type) {
+	case HOLE_ATOM:
+		return has_full_blocks(conf);
+	case DATA_ATOM:
+		return has_full_blocks(conf) ?
+			logical_blocks_occupied(0,
+						conf->nr_full_blocks,
+						MAX_IOVEC_BITS)	: 0;
+	default:
+		gf_log("crypt", GF_LOG_DEBUG, "bad atom data type");
+		return 0;
+	}
+}
+
+static inline uint32_t nr_calls(struct avec_config *conf)
+{
+	return nr_calls_head(conf) + nr_calls_tail(conf) + nr_calls_full(conf);
+}
+
+static inline uint32_t nr_calls_data(call_frame_t *frame)
+{
+	return nr_calls(get_data_conf(frame));
+}
+
+static inline uint32_t nr_calls_hole(call_frame_t *frame)
+{
+	return nr_calls(get_hole_conf(frame));
+}
+
+static inline void get_one_call_nolock(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+
+	++local->nr_calls;
+
+	//gf_log("crypt", GF_LOG_DEBUG, "get %d calls", 1);
+}
+
+static inline void get_one_call(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+
+	LOCK(&local->call_lock);
+	get_one_call_nolock(frame);
+	UNLOCK(&local->call_lock);
+}
+
+static inline void get_nr_calls_nolock(call_frame_t *frame, int32_t nr)
+{
+	crypt_local_t *local = frame->local;
+
+	local->nr_calls += nr;
+
+	//gf_log("crypt", GF_LOG_DEBUG, "get %d calls", nr);
+}
+
+static inline void get_nr_calls(call_frame_t *frame, int32_t nr)
+{
+	crypt_local_t *local = frame->local;
+
+	LOCK(&local->call_lock);
+	get_nr_calls_nolock(frame, nr);
+	UNLOCK(&local->call_lock);
+}
+
+static inline int put_one_call(crypt_local_t *local)
+{
+	uint32_t last = 0;
+
+	LOCK(&local->call_lock);
+	if (--local->nr_calls == 0)
+		last = 1;
+
+	//gf_log("crypt", GF_LOG_DEBUG, "put %d calls", 1);
+
+	UNLOCK(&local->call_lock);
+	return last;
+}
+
+static inline int is_appended_write(call_frame_t *frame)
+{
+	crypt_local_t *local = frame->local;
+	struct avec_config *conf = get_data_conf(frame);
+
+	return conf->orig_offset + conf->orig_size > local->old_file_size;
+}
+
+static inline int is_ordered_mode(call_frame_t *frame)
+{
+#if 0
+	crypt_local_t *local = frame->local;
+	return local->fop == GF_FOP_FTRUNCATE ||
+		(local->fop == GF_FOP_WRITE && is_appended_write(frame));
+#endif
+	return 1;
+}
+
+static inline int32_t hole_conv_completed(crypt_local_t *local)
+{
+	struct avec_config *conf = &local->hole_conf;
+	return conf->cursor == conf->acount;
+}
+
+static inline int32_t data_write_in_progress(crypt_local_t *local)
+{
+	return local->active_setup == DATA_ATOM;
+}
+
+static inline int32_t parent_is_crypt_xlator(call_frame_t *frame,
+					     xlator_t *this)
+{
+	return frame->parent->this == this;
+}
+
+static inline linkop_wind_handler_t linkop_wind_dispatch(glusterfs_fop_t fop)
+{
+	switch(fop){
+	case GF_FOP_LINK:
+		return link_wind;
+	case GF_FOP_UNLINK:
+		return unlink_wind;
+	case GF_FOP_RENAME:
+		return rename_wind;
+	default:
+		gf_log("crypt", GF_LOG_ERROR, "Bad link operation %d", fop);
+		return NULL;
+	}
+}
+
+static inline linkop_unwind_handler_t linkop_unwind_dispatch(glusterfs_fop_t fop)
+{
+	switch(fop){
+	case GF_FOP_LINK:
+		return link_unwind;
+	case GF_FOP_UNLINK:
+		return unlink_unwind;
+	case GF_FOP_RENAME:
+		return rename_unwind;
+	default:
+		gf_log("crypt", GF_LOG_ERROR, "Bad link operation %d", fop);
+		return NULL;
+	}
+}
+
+static inline mtd_op_t linkop_mtdop_dispatch(glusterfs_fop_t fop)
+{
+	switch (fop) {
+	case GF_FOP_LINK:
+		return MTD_APPEND;
+	case GF_FOP_UNLINK:
+		return MTD_CUT;
+	case GF_FOP_RENAME:
+		return MTD_OVERWRITE;
+	default:
+		gf_log("crypt", GF_LOG_WARNING, "Bad link operation %d", fop);
+		return MTD_LAST_OP;
+	}
+}
+
+#endif /* __CRYPT_H__ */
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/data.c b/xlators/encryption/crypt/src/data.c
new file mode 100644
index 000000000..762fa554a
--- /dev/null
+++ b/xlators/encryption/crypt/src/data.c
@@ -0,0 +1,769 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+
+static void set_iv_aes_xts(off_t offset, struct object_cipher_info *object)
+{
+	unsigned char *ivec;
+
+	ivec = object->u.aes_xts.ivec;
+
+	/* convert the tweak into a little-endian byte
+	 * array (IEEE P1619/D16, May 2007, section 5.1)
+	 */
+
+	*((uint64_t *)ivec) = htole64(offset);
+
+	/* ivec is padded with zeroes */
+}
+
+static int32_t aes_set_keys_common(unsigned char *raw_key, uint32_t key_size,
+				   AES_KEY *keys)
+{
+	int32_t ret;
+
+	ret = AES_set_encrypt_key(raw_key,
+				  key_size,
+				  &keys[AES_ENCRYPT]);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Set encrypt key failed");
+		return ret;
+	}
+	ret = AES_set_decrypt_key(raw_key,
+				  key_size,
+				  &keys[AES_DECRYPT]);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Set decrypt key failed");
+		return ret;
+	}
+	return 0;
+}
+
+/*
+ * set private cipher info for xts mode
+ */
+static int32_t set_private_aes_xts(struct crypt_inode_info *info,
+				   struct master_cipher_info *master)
+{
+	int ret;
+	struct object_cipher_info *object = get_object_cinfo(info);
+	unsigned char *data_key;
+	uint32_t subkey_size;
+
+	/* init tweak value */
+	memset(object->u.aes_xts.ivec, 0, 16);
+
+	data_key = GF_CALLOC(1, object->o_dkey_size, gf_crypt_mt_key);
+	if (!data_key)
+		return ENOMEM;
+
+	/*
+	 * retrieve data keying meterial
+	 */
+	ret = get_data_file_key(info, master, object->o_dkey_size, data_key);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Failed to retrieve data key");
+		GF_FREE(data_key);
+		return ret;
+	}
+	/*
+	 * parse compound xts key
+	 */
+	subkey_size = object->o_dkey_size >> 4; /* (xts-key-size-in-bytes / 2) */
+	/*
+	 * install key for data encryption
+	 */
+	ret = aes_set_keys_common(data_key,
+				  subkey_size << 3, object->u.aes_xts.dkey);
+	if (ret) {
+		GF_FREE(data_key);
+		return ret;
+	}
+	/*
+	 * set up key used to encrypt tweaks
+	 */
+	ret = AES_set_encrypt_key(data_key + subkey_size,
+				  object->o_dkey_size / 2,
+				  &object->u.aes_xts.tkey);
+	if (ret < 0)
+		gf_log("crypt", GF_LOG_ERROR, "Set tweak key failed");
+
+	GF_FREE(data_key);
+	return ret;
+}
+
+static int32_t aes_xts_init(void)
+{
+	cassert(AES_BLOCK_SIZE == (1 << AES_BLOCK_BITS));
+	return 0;
+}
+
+static int32_t check_key_aes_xts(uint32_t keysize)
+{
+	switch(keysize) {
+	case 256:
+	case 512:
+		return 0;
+	default:
+		break;
+	}
+	return -1;
+}
+
+static int32_t encrypt_aes_xts(const unsigned char *from,
+			       unsigned char *to, size_t length,
+			       off_t offset, const int enc,
+			       struct object_cipher_info *object)
+{
+	XTS128_CONTEXT ctx;
+	if (enc) {
+		ctx.key1 = &object->u.aes_xts.dkey[AES_ENCRYPT];
+		ctx.block1 = (block128_f)AES_encrypt;
+	}
+	else {
+		ctx.key1 = &object->u.aes_xts.dkey[AES_DECRYPT];
+		ctx.block1 = (block128_f)AES_decrypt;
+	}
+	ctx.key2 = &object->u.aes_xts.tkey;
+	ctx.block2 = (block128_f)AES_encrypt;
+
+	return CRYPTO_xts128_encrypt(&ctx,
+				     object->u.aes_xts.ivec,
+				     from,
+				     to,
+				     length, enc);
+}
+
+/*
+ * Cipher input chunk @from of length @len;
+ * @to: result of cipher transform;
+ * @off: offset in a file (must be cblock-aligned);
+ */
+static void cipher_data(struct object_cipher_info *object,
+			char *from,
+			char *to,
+			off_t off,
+			size_t len,
+			const int enc)
+{
+	crypt_check_input_len(len, object);
+
+#if TRIVIAL_TFM && DEBUG_CRYPT
+	return;
+#endif
+	data_cipher_algs[object->o_alg][object->o_mode].set_iv(off, object);
+	data_cipher_algs[object->o_alg][object->o_mode].encrypt
+		((const unsigned char *)from,
+		 (unsigned char *)to,
+		 len,
+		 off,
+		 enc,
+		 object);
+}
+
+#define MAX_CIPHER_CHUNK (1 << 30)
+
+/*
+ * Do cipher (encryption/decryption) transform of a
+ * continuous region of memory.
+ *
+ * @len: a number of bytes to transform;
+ * @buf: data to transform;
+ * @off: offset in a file, should be block-aligned
+ *       for atomic cipher modes and ksize-aligned
+ *       for other modes).
+ * @dir: direction of transform (encrypt/decrypt).
+ */
+static void cipher_region(struct object_cipher_info *object,
+			  char *from,
+			  char *to,
+			  off_t off,
+			  size_t len,
+			  int dir)
+{
+	while (len > 0) {
+		size_t to_cipher;
+
+		to_cipher = len;
+		if (to_cipher > MAX_CIPHER_CHUNK)
+			to_cipher = MAX_CIPHER_CHUNK;
+
+		/* this will reset IV */
+		cipher_data(object,
+			    from,
+			    to,
+			    off,
+			    to_cipher,
+			    dir);
+		from += to_cipher;
+		to   += to_cipher;
+		off  += to_cipher;
+		len  -= to_cipher;
+	}
+}
+
+/*
+ * Do cipher transform (encryption/decryption) of
+ * plaintext/ciphertext represented by @vec.
+ *
+ * Pre-conditions: @vec represents a continuous piece
+ * of data in a file at offset @off to be ciphered
+ * (encrypted/decrypted).
+ * @count is the number of vec's components. All the
+ * components must be block-aligned, the caller is
+ * responsible for this. @dir is "direction" of
+ * transform (encrypt/decrypt).
+ */
+static void cipher_aligned_iov(struct object_cipher_info *object,
+			       struct iovec *vec,
+			       int count,
+			       off_t off,
+			       int32_t dir)
+{
+	int i;
+	int len = 0;
+
+	for (i = 0; i < count; i++) {
+		cipher_region(object,
+			      vec[i].iov_base,
+			      vec[i].iov_base,
+			      off + len,
+			      vec[i].iov_len,
+			      dir);
+		len += vec[i].iov_len;
+	}
+}
+
+void encrypt_aligned_iov(struct object_cipher_info *object,
+			 struct iovec *vec,
+			 int count,
+			 off_t off)
+{
+	cipher_aligned_iov(object, vec, count, off, 1);
+}
+
+void decrypt_aligned_iov(struct object_cipher_info *object,
+			 struct iovec *vec,
+			 int count,
+			 off_t off)
+{
+	cipher_aligned_iov(object, vec, count, off, 0);
+}
+
+#if DEBUG_CRYPT
+static void compound_stream(struct iovec *vec, int count, char *buf, off_t skip)
+{
+	int i;
+	int off = 0;
+	for (i = 0; i < count; i++) {
+		memcpy(buf + off,
+		       vec[i].iov_base + skip,
+		       vec[i].iov_len - skip);
+
+		off += (vec[i].iov_len - skip);
+		skip = 0;
+	}
+}
+
+static void check_iovecs(struct iovec *vec, int cnt,
+			 struct iovec *avec, int acnt, uint32_t off_in_head)
+{
+	char *s1, *s2;
+	uint32_t size, asize;
+
+	size = iovec_get_size(vec, cnt);
+	asize = iovec_get_size(avec, acnt) - off_in_head;
+	if (size != asize) {
+		gf_log("crypt", GF_LOG_DEBUG, "size %d is not eq asize %d",
+		       size, asize);
+		return;
+	}
+	s1 = GF_CALLOC(1, size, gf_crypt_mt_data);
+	if (!s1) {
+		gf_log("crypt", GF_LOG_DEBUG, "Can not allocate stream ");
+		return;
+	}
+	s2 = GF_CALLOC(1, asize, gf_crypt_mt_data);
+	if (!s2) {
+		GF_FREE(s1);
+		gf_log("crypt", GF_LOG_DEBUG, "Can not allocate stream ");
+		return;
+	}
+	compound_stream(vec, cnt, s1, 0);
+	compound_stream(avec, acnt, s2, off_in_head);
+	if (memcmp(s1, s2, size))
+		gf_log("crypt", GF_LOG_DEBUG, "chunks of different data");
+	GF_FREE(s1);
+	GF_FREE(s2);
+}
+
+#else
+#define check_iovecs(vec, count, avec, avecn, off) noop
+#endif /* DEBUG_CRYPT */
+
+static char *data_alloc_block(xlator_t *this, crypt_local_t *local,
+			      int32_t block_size)
+{
+	struct iobuf *iobuf = NULL;
+
+	iobuf = iobuf_get2(this->ctx->iobuf_pool, block_size);
+	if (!iobuf) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "Failed to get iobuf");
+		return NULL;
+	}
+	if (!local->iobref_data) {
+		local->iobref_data = iobref_new();
+		if (!local->iobref_data) {
+			gf_log("crypt", GF_LOG_ERROR,
+			       "Failed to get iobref");
+			iobuf_unref(iobuf);
+			return NULL;
+		}
+	}
+	iobref_add(local->iobref_data, iobuf);
+	return iobuf->ptr;
+}
+
+/*
+ * Compound @avec, which represent the same data
+ * chunk as @vec, but has aligned components of
+ * specified block size. Alloc blocks, if needed.
+ * In particular, incomplete head and tail blocks
+ * must be allocated.
+ * Put number of allocated blocks to @num_blocks.
+ *
+ * Example:
+ *
+ * input: data chunk represented by 4 components
+ * [AB],[BC],[CD],[DE];
+ * output: 5 logical blocks (0, 1, 2, 3, 4).
+ *
+ *   A     B       C     D               E
+ *   *-----*+------*-+---*----+--------+-*
+ *   |     ||      | |   |    |        | |
+ * *-+-----+*------+-*---+----*--------*-+------*
+ * 0        1        2        3        4
+ *
+ * 0 - incomplete compound (head);
+ * 1, 2 - full compound;
+ * 3 - full non-compound (the case of reuse);
+ * 4 - incomplete non-compound (tail).
+ */
+int32_t align_iov_by_atoms(xlator_t *this,
+			   crypt_local_t *local,
+			   struct object_cipher_info *object,
+			   struct iovec *vec /* input vector */,
+			   int32_t count /* number of vec components */,
+			   struct iovec *avec /* aligned vector */,
+			   char **blocks /* pool of blocks */,
+			   uint32_t *blocks_allocated,
+			   struct avec_config *conf)
+{
+	int vecn = 0;      /* number of the current component in vec */
+	int avecn = 0;     /* number of the current component in avec */
+	off_t vec_off = 0; /* offset in the current vec component,
+			    * i.e. the number of bytes have already
+			    * been copied */
+	int32_t block_size = get_atom_size(object);
+	size_t to_process; /* number of vec's bytes to copy and(or) re-use */
+	int32_t off_in_head = conf->off_in_head;
+
+	to_process = iovec_get_size(vec, count);
+
+	while (to_process > 0) {
+		if (off_in_head ||
+		    vec[vecn].iov_len - vec_off < block_size) {
+			/*
+			 * less than block_size:
+			 * the case of incomplete (head or tail),
+			 * or compound block
+			 */
+			size_t copied = 0;
+			/*
+			 * populate the pool with a new block
+			 */
+			blocks[*blocks_allocated] = data_alloc_block(this,
+								    local,
+								    block_size);
+			if (!blocks[*blocks_allocated])
+				return -ENOMEM;
+			memset(blocks[*blocks_allocated], 0, off_in_head);
+			/*
+			 * fill the block with vec components
+			 */
+			do {
+				size_t to_copy;
+
+				to_copy = vec[vecn].iov_len - vec_off;
+				if (to_copy > block_size - off_in_head)
+					to_copy = block_size - off_in_head;
+
+				memcpy(blocks[*blocks_allocated] + off_in_head + copied,
+				       vec[vecn].iov_base + vec_off,
+				       to_copy);
+
+				copied += to_copy;
+				to_process -= to_copy;
+
+				vec_off += to_copy;
+				if (vec_off == vec[vecn].iov_len) {
+					/* finished with this vecn */
+					vec_off = 0;
+					vecn++;
+				}
+			} while (copied < (block_size - off_in_head) && to_process > 0);
+			/*
+			 * update avec
+			 */
+			avec[avecn].iov_len = off_in_head + copied;
+			avec[avecn].iov_base = blocks[*blocks_allocated];
+
+			(*blocks_allocated)++;
+			off_in_head = 0;
+		} else {
+			/*
+			 * the rest of the current vec component
+			 * is not less than block_size, so reuse
+			 * the memory buffer of the component.
+			 */
+			size_t to_reuse;
+			to_reuse = (to_process > block_size ?
+				    block_size :
+				    to_process);
+			avec[avecn].iov_len = to_reuse;
+			avec[avecn].iov_base = vec[vecn].iov_base + vec_off;
+
+			vec_off += to_reuse;
+			if (vec_off == vec[vecn].iov_len) {
+				/* finished with this vecn */
+				vec_off = 0;
+				vecn++;
+			}
+			to_process -= to_reuse;
+		}
+		avecn++;
+	}
+	check_iovecs(vec, count, avec, avecn, conf->off_in_head);
+	return 0;
+}
+
+/*
+ * allocate and setup aligned vector for data submission
+ * Pre-condition: @conf is set.
+ */
+int32_t set_config_avec_data(xlator_t *this,
+			     crypt_local_t *local,
+			     struct avec_config *conf,
+			     struct object_cipher_info *object,
+			     struct iovec *vec,
+			     int32_t vec_count)
+{
+	int32_t ret = ENOMEM;
+	struct iovec *avec;
+	char **pool;
+	uint32_t blocks_in_pool = 0;
+
+	conf->type = DATA_ATOM;
+
+	avec = GF_CALLOC(conf->acount, sizeof(*avec), gf_crypt_mt_iovec);
+	if (!avec)
+		return ret;
+	pool = GF_CALLOC(conf->acount, sizeof(pool), gf_crypt_mt_char);
+	if (!pool) {
+		GF_FREE(avec);
+		return ret;
+	}
+	if (!vec) {
+		/*
+		 * degenerated case: no data
+		 */
+		pool[0] = data_alloc_block(this, local, get_atom_size(object));
+		if (!pool[0])
+			goto free;
+		blocks_in_pool = 1;
+		avec->iov_base = pool[0];
+		avec->iov_len = conf->off_in_tail;
+	}
+	else {
+		ret = align_iov_by_atoms(this, local, object, vec, vec_count,
+					 avec, pool, &blocks_in_pool, conf);
+		if (ret)
+			goto free;
+	}
+	conf->avec = avec;
+	conf->pool = pool;
+	conf->blocks_in_pool = blocks_in_pool;
+	return 0;
+ free:
+	GF_FREE(avec);
+	GF_FREE(pool);
+	return ret;
+}
+
+/*
+ * allocate and setup aligned vector for hole submission
+ */
+int32_t set_config_avec_hole(xlator_t *this,
+			     crypt_local_t *local,
+			     struct avec_config *conf,
+			     struct object_cipher_info *object,
+			     glusterfs_fop_t fop)
+{
+	uint32_t i, idx;
+	struct iovec *avec;
+	char **pool;
+	uint32_t num_blocks;
+	uint32_t blocks_in_pool = 0;
+
+	conf->type = HOLE_ATOM;
+
+	num_blocks = conf->acount -
+		(conf->nr_full_blocks ? conf->nr_full_blocks - 1 : 0);
+
+	switch (fop) {
+	case GF_FOP_WRITE:
+		/*
+		 * hole goes before data
+		 */
+		if (num_blocks == 1 && conf->off_in_tail != 0)
+		/*
+		 * we won't submit a hole which fits into
+		 * a data atom: this part of hole will be
+		 * submitted with data write
+		 */
+			return 0;
+		break;
+	case GF_FOP_FTRUNCATE:
+		/*
+		 * expanding truncate, hole goes after data,
+		 * and will be submited in any case.
+ 		 */
+		break;
+	default:
+		gf_log("crypt", GF_LOG_WARNING,
+		       "bad file operation %d", fop);
+		return 0;
+	}
+	avec = GF_CALLOC(num_blocks, sizeof(*avec), gf_crypt_mt_iovec);
+	if (!avec)
+		return ENOMEM;
+	pool = GF_CALLOC(num_blocks, sizeof(pool), gf_crypt_mt_char);
+	if (!pool) {
+		GF_FREE(avec);
+		return ENOMEM;
+	}
+	for (i = 0; i < num_blocks; i++) {
+		pool[i] = data_alloc_block(this, local, get_atom_size(object));
+		if (pool[i] == NULL)
+			goto free;
+		blocks_in_pool++;
+	}
+	if (has_head_block(conf)) {
+		/* set head block */
+		idx = 0;
+		avec[idx].iov_base = pool[idx];
+		avec[idx].iov_len = get_atom_size(object);
+		memset(avec[idx].iov_base + conf->off_in_head,
+		       0,
+		       get_atom_size(object) - conf->off_in_head);
+	}
+	if (has_tail_block(conf)) {
+		/* set tail block */
+		idx = num_blocks - 1;
+		avec[idx].iov_base = pool[idx];
+		avec[idx].iov_len = get_atom_size(object);
+		memset(avec[idx].iov_base, 0, conf->off_in_tail);
+	}
+	if (has_full_blocks(conf)) {
+		/* set full block */
+		idx = conf->off_in_head ? 1 : 0;
+		avec[idx].iov_base = pool[idx];
+		avec[idx].iov_len = get_atom_size(object);
+		/*
+		 * since we re-use the buffer,
+		 * zeroes will be set every time
+		 * before encryption, see submit_full()
+		 */
+	}
+	conf->avec = avec;
+	conf->pool = pool;
+	conf->blocks_in_pool = blocks_in_pool;
+	return 0;
+ free:
+	GF_FREE(avec);
+	GF_FREE(pool);
+	return ENOMEM;
+}
+
+/* A helper for setting up config of partial atoms (which
+ * participate in read-modify-write sequence).
+ *
+ * Calculate and setup precise amount of "extra-bytes"
+ * that should be uptodated at the end of partial (not
+ * necessarily tail!) block.
+ *
+ * Pre-condition: local->old_file_size is valid!
+ * @conf contains setup, which is enough for correct calculation
+ * of has_tail_block(), ->get_offset().
+ */
+void set_gap_at_end(call_frame_t *frame, struct object_cipher_info *object,
+		    struct avec_config *conf, atom_data_type dtype)
+{
+	uint32_t to_block;
+	crypt_local_t *local = frame->local;
+	uint64_t old_file_size = local->old_file_size;
+	struct rmw_atom *partial = atom_by_types(dtype,
+						 has_tail_block(conf) ?
+						 TAIL_ATOM : HEAD_ATOM);
+
+	if (old_file_size <= partial->offset_at(frame, object))
+		to_block = 0;
+	else {
+		to_block = old_file_size - partial->offset_at(frame, object);
+		if (to_block > get_atom_size(object))
+			to_block = get_atom_size(object);
+	}
+	if (to_block > conf->off_in_tail)
+		conf->gap_in_tail = to_block - conf->off_in_tail;
+	else
+		/*
+		 * nothing to uptodate
+		 */
+		conf->gap_in_tail = 0;
+}
+
+/*
+ * fill struct avec_config with offsets layouts
+ */
+void set_config_offsets(call_frame_t *frame,
+			xlator_t *this,
+			uint64_t offset,
+			uint64_t count,
+			atom_data_type dtype,
+			int32_t set_gap)
+{
+	crypt_local_t *local;
+	struct object_cipher_info *object;
+	struct avec_config *conf;
+	uint32_t resid;
+
+	uint32_t atom_size;
+	uint32_t atom_bits;
+
+	size_t orig_size;
+	off_t orig_offset;
+	size_t expanded_size;
+	off_t aligned_offset;
+
+	uint32_t off_in_head = 0;
+	uint32_t off_in_tail = 0;
+	uint32_t nr_full_blocks;
+	int32_t size_full_blocks;
+
+	uint32_t acount; /* number of alifned components to write.
+			  * The same as number of occupied logical
+			  * blocks (atoms)
+			  */
+	local = frame->local;
+	object = &local->info->cinfo;
+	conf = (dtype == DATA_ATOM ?
+		get_data_conf(frame) : get_hole_conf(frame));
+
+	orig_offset = offset;
+	orig_size = count;
+
+	atom_size = get_atom_size(object);
+	atom_bits = get_atom_bits(object);
+
+	/*
+	 * Round-down the start,
+	 * round-up the end.
+	 */
+	resid = offset & (uint64_t)(atom_size - 1);
+
+	if (resid)
+		off_in_head = resid;
+	aligned_offset = offset - off_in_head;
+	expanded_size = orig_size + off_in_head;
+
+	/* calculate tail,
+	   expand size forward  */
+	resid = (offset + orig_size) & (uint64_t)(atom_size - 1);
+
+	if (resid) {
+		off_in_tail = resid;
+		expanded_size += (atom_size - off_in_tail);
+	}
+	/*
+	 * calculate number of occupied blocks
+	 */
+	acount = expanded_size >> atom_bits;
+	/*
+	 * calculate number of full blocks
+	 */
+	size_full_blocks = expanded_size;
+	if (off_in_head)
+		size_full_blocks -= atom_size;
+	if (off_in_tail && size_full_blocks > 0)
+		size_full_blocks -= atom_size;
+	nr_full_blocks = size_full_blocks >> atom_bits;
+
+	conf->atom_size = atom_size;
+	conf->orig_size = orig_size;
+	conf->orig_offset = orig_offset;
+	conf->expanded_size = expanded_size;
+	conf->aligned_offset = aligned_offset;
+
+	conf->off_in_head = off_in_head;
+	conf->off_in_tail = off_in_tail;
+	conf->nr_full_blocks = nr_full_blocks;
+	conf->acount = acount;
+	/*
+	 * Finally, calculate precise amount of
+	 * "extra-bytes" that should be uptodated
+	 * at the end.
+	 * Only if RMW is expected.
+	 */
+	if (off_in_tail && set_gap)
+		set_gap_at_end(frame, object, conf, dtype);
+}
+
+struct data_cipher_alg data_cipher_algs[LAST_CIPHER_ALG][LAST_CIPHER_MODE] = {
+	[AES_CIPHER_ALG][XTS_CIPHER_MODE] =
+	{ .atomic = _gf_true,
+	  .should_pad = _gf_true,
+	  .blkbits = AES_BLOCK_BITS,
+	  .init = aes_xts_init,
+	  .set_private = set_private_aes_xts,
+	  .check_key = check_key_aes_xts,
+	  .set_iv = set_iv_aes_xts,
+	  .encrypt = encrypt_aes_xts
+	}
+};
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/keys.c b/xlators/encryption/crypt/src/keys.c
new file mode 100644
index 000000000..4a1d3bb5a
--- /dev/null
+++ b/xlators/encryption/crypt/src/keys.c
@@ -0,0 +1,302 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+
+/* Key hierarchy
+                  
+                   +----------------+
+                   | MASTER_VOL_KEY |
+                   +-------+--------+
+                           |
+                           |
+          +----------------+----------------+
+          |                |                |
+          |                |                |
+  +-------+------+ +-------+-------+ +------+--------+
+  | NMTD_VOL_KEY | | EMTD_FILE_KEY | | DATA_FILE_KEY |
+  +-------+------+ +---------------+ +---------------+
+          |
+          |
+  +-------+-------+
+  | NMTD_LINK_KEY |
+  +---------------+
+
+ */
+
+#if DEBUG_CRYPT
+static void check_prf_iters(uint32_t num_iters)
+{
+	if (num_iters == 0)
+		gf_log ("crypt", GF_LOG_DEBUG,
+			"bad number of prf iterations : %d", num_iters);
+}
+#else
+#define check_prf_iters(num_iters) noop
+#endif /* DEBUG_CRYPT */
+
+unsigned char crypt_fake_oid[16] =
+	{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+
+/*
+ * derive key in the counter mode using
+ * sha256-based HMAC as PRF, see
+ * NIST Special Publication 800-108, 5.1)
+ */
+
+#define PRF_OUTPUT_SIZE  SHA256_DIGEST_LENGTH
+
+static int32_t kderive_init(struct kderive_context *ctx,
+			    const unsigned char *pkey, /* parent key */
+			    uint32_t pkey_size,  /* parent key size */
+			    const unsigned char *idctx, /* id-context */
+			    uint32_t idctx_size,
+			    crypt_key_type type  /* type of child key */)
+{
+	unsigned char *pos;
+	uint32_t llen = strlen(crypt_keys[type].label);	
+	/*
+	 * Compoud the fixed input data for KDF:
+	 * [i]_2 || Label || 0x00 || Id-Context || [L]_2),
+	 * NIST SP 800-108, 5.1
+	 */
+	ctx->fid_len =
+		sizeof(uint32_t) +
+		llen +
+		1 +
+		idctx_size +
+		sizeof(uint32_t);
+
+	ctx->fid = GF_CALLOC(ctx->fid_len, 1, gf_crypt_mt_key);
+	if (!ctx->fid)
+		return ENOMEM;
+	ctx->out_len = round_up(crypt_keys[type].len >> 3,
+				PRF_OUTPUT_SIZE);
+	ctx->out = GF_CALLOC(ctx->out_len, 1, gf_crypt_mt_key);
+	if (!ctx->out) {
+		GF_FREE(ctx->fid);
+		return ENOMEM;
+	}
+	ctx->pkey = pkey;
+	ctx->pkey_len = pkey_size;
+	ctx->ckey_len = crypt_keys[type].len;
+
+	pos = ctx->fid;
+
+	/* counter will be set up in kderive_rfn() */
+	pos += sizeof(uint32_t);
+
+	memcpy(pos, crypt_keys[type].label, llen);
+	pos += llen;
+
+	/* set up zero octet */
+	*pos = 0;
+	pos += 1;
+
+	memcpy(pos, idctx, idctx_size);
+	pos += idctx_size;
+
+	*((uint32_t *)pos) = htobe32(ctx->ckey_len);
+
+	return 0;
+}
+
+static void kderive_update(struct kderive_context *ctx)
+{	
+	uint32_t i;
+	HMAC_CTX hctx;
+	unsigned char *pos = ctx->out;
+	uint32_t *p_iter = (uint32_t *)ctx->fid;
+	uint32_t num_iters = ctx->out_len / PRF_OUTPUT_SIZE;
+
+	check_prf_iters(num_iters);
+
+	HMAC_CTX_init(&hctx);
+	for (i = 0; i < num_iters; i++) {
+		/*
+		 * update the iteration number in the fid
+		 */
+		*p_iter = htobe32(i);
+		HMAC_Init_ex(&hctx,
+			     ctx->pkey, ctx->pkey_len >> 3,
+			     EVP_sha256(),
+			     NULL);
+		HMAC_Update(&hctx, ctx->fid, ctx->fid_len);
+		HMAC_Final(&hctx, pos, NULL);
+
+		pos += PRF_OUTPUT_SIZE;
+	}
+	HMAC_CTX_cleanup(&hctx);
+}
+
+static void kderive_final(struct kderive_context *ctx, unsigned char *child)
+{
+	memcpy(child, ctx->out, ctx->ckey_len >> 3);
+	GF_FREE(ctx->fid);
+	GF_FREE(ctx->out);
+	memset(ctx, 0, sizeof(*ctx));
+}
+
+/*
+ * derive per-volume key for object ids aithentication
+ */
+int32_t get_nmtd_vol_key(struct master_cipher_info *master)
+{
+	int32_t ret;
+	struct kderive_context ctx;
+
+	ret = kderive_init(&ctx,
+			   master->m_key,
+			   master_key_size(),
+			   crypt_fake_oid, sizeof(uuid_t), NMTD_VOL_KEY);
+	if (ret)
+		return ret;
+	kderive_update(&ctx);
+	kderive_final(&ctx, master->m_nmtd_key);
+	return 0;
+}
+
+/*
+ * derive per-link key for aithentication of non-encrypted
+ * meta-data (nmtd)
+ */
+int32_t get_nmtd_link_key(loc_t *loc,
+			  struct master_cipher_info *master,
+			  unsigned char *result)
+{
+	int32_t ret;
+	struct kderive_context ctx;
+
+	ret = kderive_init(&ctx,
+			   master->m_nmtd_key,
+			   nmtd_vol_key_size(),
+			   (const unsigned char *)loc->path,
+			   strlen(loc->path), NMTD_LINK_KEY);
+	if (ret)
+		return ret;
+	kderive_update(&ctx);
+	kderive_final(&ctx, result);
+	return 0;
+}
+
+/*
+ * derive per-file key for encryption and authentication
+ * of encrypted part of metadata (emtd)
+ */
+int32_t get_emtd_file_key(struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  unsigned char *result)
+{
+	int32_t ret;
+	struct kderive_context ctx;
+
+	ret = kderive_init(&ctx,
+			   master->m_key,
+			   master_key_size(),
+			   info->oid, sizeof(uuid_t), EMTD_FILE_KEY);
+	if (ret)
+		return ret;
+	kderive_update(&ctx);
+	kderive_final(&ctx, result);
+	return 0;
+}
+
+static int32_t data_key_type_by_size(uint32_t keysize, crypt_key_type *type)
+{
+	int32_t ret = 0;
+	switch (keysize) {
+	case 256:
+		*type = DATA_FILE_KEY_256;
+		break;
+	case 512:
+		*type = DATA_FILE_KEY_512;
+		break;
+	default:
+		gf_log("crypt", GF_LOG_ERROR, "Unsupported data key size %d",
+		       keysize);
+		ret = ENOTSUP;
+		break;
+	}
+	return ret;
+}
+
+/*
+ * derive per-file key for data encryption
+ */
+int32_t get_data_file_key(struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  uint32_t keysize,
+			  unsigned char *key)
+{
+	int32_t ret;
+	struct kderive_context ctx;
+	crypt_key_type type;
+
+	ret = data_key_type_by_size(keysize, &type);
+	if (ret)
+		return ret;
+	ret = kderive_init(&ctx,
+			   master->m_key,
+			   master_key_size(),
+			   info->oid, sizeof(uuid_t), type);
+	if (ret)
+		return ret;
+	kderive_update(&ctx);
+	kderive_final(&ctx, key);
+	return 0;
+}
+
+/*
+ * NOTE: Don't change existing keys: it will break compatibility;
+ */
+struct crypt_key crypt_keys[LAST_KEY_TYPE] = {
+	[MASTER_VOL_KEY] =
+	{ .len = MASTER_VOL_KEY_SIZE << 3,
+	  .label = "volume-master",
+	},
+	[NMTD_VOL_KEY] =
+	{ .len = NMTD_VOL_KEY_SIZE << 3,
+	  .label = "volume-nmtd-key-generation"
+	},
+	[NMTD_LINK_KEY] =
+	{ .len = 128,
+	  .label = "link-nmtd-authentication"
+	},
+	[EMTD_FILE_KEY] =
+	{ .len = 128,
+	  .label = "file-emtd-encryption-and-auth"
+	},
+	[DATA_FILE_KEY_256] =
+	{ .len = 256,
+	  .label = "file-data-encryption-256"
+	},
+	[DATA_FILE_KEY_512] =
+	{ .len = 512,
+	  .label = "file-data-encryption-512"
+	}
+};
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/metadata.c b/xlators/encryption/crypt/src/metadata.c
new file mode 100644
index 000000000..36b14c055
--- /dev/null
+++ b/xlators/encryption/crypt/src/metadata.c
@@ -0,0 +1,605 @@
+/*
+  Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "defaults.h"
+#include "crypt-common.h"
+#include "crypt.h"
+#include "metadata.h"
+
+int32_t alloc_format(crypt_local_t *local, size_t size)
+{
+	if (size > 0) {
+		local->format = GF_CALLOC(1, size, gf_crypt_mt_mtd);
+		if (!local->format)
+			return ENOMEM;
+	}
+	local->format_size = size;
+	return 0;
+}
+
+int32_t alloc_format_create(crypt_local_t *local)
+{
+	return alloc_format(local, new_format_size());
+}
+
+void free_format(crypt_local_t *local)
+{
+	GF_FREE(local->format);
+}
+
+/*
+ * Check compatibility with extracted metadata
+ */
+static int32_t check_file_metadata(struct crypt_inode_info *info)
+{
+	struct object_cipher_info *object = &info->cinfo;
+
+	if (info->nr_minor != CRYPT_XLATOR_ID) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "unsupported minor subversion %d", info->nr_minor);
+		return EINVAL;
+	}
+	if (object->o_alg > LAST_CIPHER_ALG) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "unsupported cipher algorithm %d",
+		       object->o_alg);
+		return EINVAL;
+	}
+	if (object->o_mode > LAST_CIPHER_MODE) {
+		gf_log("crypt", GF_LOG_WARNING,
+		       "unsupported cipher mode %d",
+		       object->o_mode);
+		return EINVAL;
+	}
+	if (object->o_block_bits < CRYPT_MIN_BLOCK_BITS ||
+	    object->o_block_bits > CRYPT_MAX_BLOCK_BITS) {
+		gf_log("crypt", GF_LOG_WARNING, "unsupported block bits %d",
+		       object->o_block_bits);
+		return EINVAL;
+	}
+	/* TBD: check data key size */
+	return 0;
+}
+
+static size_t format_size_v1(mtd_op_t op, size_t old_size)
+{
+	
+	switch (op) {
+	case MTD_CREATE:
+		return sizeof(struct mtd_format_v1);
+	case MTD_OVERWRITE:
+		return old_size;
+	case MTD_APPEND:
+		return old_size + NMTD_8_MAC_SIZE;
+	case MTD_CUT:
+		if (old_size > sizeof(struct mtd_format_v1))
+			return old_size - NMTD_8_MAC_SIZE;
+		else
+			return 0;
+	default:
+		gf_log("crypt", GF_LOG_WARNING, "Bad mtd operation");
+		return 0;
+	}
+}
+
+/*
+ * Calculate size of the updated format string.
+ * Returned zero means that we don't need to update the format string.
+ */
+size_t format_size(mtd_op_t op, size_t old_size)
+{
+	size_t versioned;
+
+	versioned = mtd_loaders[current_mtd_loader()].format_size(op,
+				 old_size - sizeof(struct crypt_format));
+	if (versioned != 0)
+		return versioned + sizeof(struct crypt_format);
+	return 0;
+}
+
+/*
+ * size of the format string of newly created file (nr_links = 1)
+ */
+size_t new_format_size(void)
+{
+	return format_size(MTD_CREATE, 0);
+}
+
+/*
+ * Calculate per-link MAC by pathname
+ */
+static int32_t calc_link_mac_v1(struct mtd_format_v1 *fmt,
+				loc_t *loc,
+				unsigned char *result,
+				struct crypt_inode_info *info,
+				struct master_cipher_info *master)
+{
+	int32_t ret;	
+	unsigned char nmtd_link_key[16];
+	CMAC_CTX *cctx;
+	size_t len;
+
+	ret = get_nmtd_link_key(loc, master, nmtd_link_key);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Can not get nmtd link key");
+		return -1;
+	}
+	cctx = CMAC_CTX_new();
+	if (!cctx) {
+		gf_log("crypt", GF_LOG_ERROR, "CMAC_CTX_new failed");
+		return -1; 
+	}
+	ret = CMAC_Init(cctx, nmtd_link_key, sizeof(nmtd_link_key),
+			EVP_aes_128_cbc(), 0);
+	if (!ret) {
+		gf_log("crypt", GF_LOG_ERROR, "CMAC_Init failed");
+		CMAC_CTX_free(cctx);
+		return -1;
+	}
+	ret = CMAC_Update(cctx, get_NMTD_V1(info), SIZE_OF_NMTD_V1);
+	if (!ret) {
+		gf_log("crypt", GF_LOG_ERROR, "CMAC_Update failed");
+		CMAC_CTX_free(cctx);
+		return -1;
+	}
+	ret = CMAC_Final(cctx, result, &len);
+	CMAC_CTX_free(cctx);
+	if (!ret) {
+		gf_log("crypt", GF_LOG_ERROR, "CMAC_Final failed");
+		return -1;
+	}
+	return 0;
+}
+
+/*
+ * Create per-link MAC of index @idx by pathname
+ */
+static int32_t create_link_mac_v1(struct mtd_format_v1 *fmt,
+				  uint32_t idx,
+				  loc_t *loc,
+				  struct crypt_inode_info *info,
+				  struct master_cipher_info *master)
+{
+	int32_t ret;
+	unsigned char *mac;
+	unsigned char cmac[16];
+
+	mac = get_NMTD_V1_MAC(fmt) + idx * SIZE_OF_NMTD_V1_MAC;
+
+	ret = calc_link_mac_v1(fmt, loc, cmac, info, master);
+	if (ret)
+		return -1;
+	memcpy(mac, cmac, SIZE_OF_NMTD_V1_MAC);
+	return 0;
+}
+
+static int32_t create_format_v1(unsigned char *wire,
+				loc_t *loc,
+				struct crypt_inode_info *info,
+				struct master_cipher_info *master)
+{
+	int32_t ret;
+	struct mtd_format_v1 *fmt;
+	unsigned char mtd_key[16];
+	AES_KEY EMTD_KEY;
+	unsigned char nmtd_link_key[16];
+	uint32_t ad;
+	GCM128_CONTEXT *gctx;
+
+	fmt = (struct mtd_format_v1 *)wire;
+
+	fmt->minor_id = info->nr_minor;
+	fmt->alg_id = AES_CIPHER_ALG;
+	fmt->dkey_factor = master->m_dkey_size >> KEY_FACTOR_BITS;
+	fmt->block_bits = master->m_block_bits;
+	fmt->mode_id = master->m_mode;
+	/*
+	 * retrieve keys for the parts of metadata
+	 */
+	ret = get_emtd_file_key(info, master, mtd_key);
+	if (ret)
+		return ret;
+	ret = get_nmtd_link_key(loc, master, nmtd_link_key);
+	if (ret)
+		return ret;
+	
+	AES_set_encrypt_key(mtd_key, sizeof(mtd_key)*8, &EMTD_KEY);
+
+	gctx = CRYPTO_gcm128_new(&EMTD_KEY, (block128_f)AES_encrypt);
+
+	/* TBD: Check return values */
+
+	CRYPTO_gcm128_setiv(gctx, info->oid, sizeof(uuid_t));
+
+	ad = htole32(MTD_LOADER_V1);
+	ret = CRYPTO_gcm128_aad(gctx, (const unsigned char *)&ad, sizeof(ad));
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_aad failed");
+		CRYPTO_gcm128_release(gctx);
+		return ret;
+	}
+	ret = CRYPTO_gcm128_encrypt(gctx,
+				    get_EMTD_V1(fmt),
+				    get_EMTD_V1(fmt),
+				    SIZE_OF_EMTD_V1);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_encrypt failed");
+		CRYPTO_gcm128_release(gctx);
+		return ret;
+	}
+	/*
+	 * set MAC of encrypted part of metadata
+	 */
+	CRYPTO_gcm128_tag(gctx, get_EMTD_V1_MAC(fmt), SIZE_OF_EMTD_V1_MAC);
+	CRYPTO_gcm128_release(gctx);
+	/*
+	 * set the first MAC of non-encrypted part of metadata 
+	 */
+	return create_link_mac_v1(fmt, 0, loc, info, master);
+}
+
+/*
+ * Called by fops:
+ * ->create();
+ * ->link();
+ *
+ * Pack common and version-specific parts of file's metadata
+ * Pre-conditions: @info contains valid object-id.
+ */
+int32_t create_format(unsigned char *wire,
+		      loc_t *loc,
+		      struct crypt_inode_info *info,
+		      struct master_cipher_info *master)
+{
+	struct crypt_format *fmt = (struct crypt_format *)wire;
+
+	fmt->loader_id = current_mtd_loader();
+
+	wire += sizeof(struct crypt_format);
+	return mtd_loaders[current_mtd_loader()].create_format(wire, loc,
+							       info, master);
+}
+
+/*
+ * Append or overwrite per-link mac of @mac_idx index
+ * in accordance with the new pathname
+ */
+int32_t appov_link_mac_v1(unsigned char *new,
+			  unsigned char *old,
+			  uint32_t old_size,
+			  int32_t mac_idx,
+			  loc_t *loc,
+			  struct crypt_inode_info *info,
+			  struct master_cipher_info *master,
+			  crypt_local_t *local)
+{
+	memcpy(new, old, old_size);
+	return create_link_mac_v1((struct mtd_format_v1 *)new, mac_idx,
+				  loc, info, master);
+}
+
+/*
+ * Cut per-link mac of @mac_idx index
+ */
+static int32_t cut_link_mac_v1(unsigned char *new,
+			       unsigned char *old,			       
+			       uint32_t old_size,
+			       int32_t mac_idx,
+			       loc_t *loc,
+			       struct crypt_inode_info *info,
+			       struct master_cipher_info *master,
+			       crypt_local_t *local)
+{
+	memcpy(new,
+	       old,
+	       sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * (mac_idx - 1));
+
+	memcpy(new + sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * (mac_idx - 1),
+	       old + sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * mac_idx,
+	       old_size - (sizeof(struct mtd_format_v1) + NMTD_8_MAC_SIZE * mac_idx));
+	return 0;
+}
+
+int32_t update_format_v1(unsigned char *new,
+			 unsigned char *old,
+				size_t old_len,
+				int32_t mac_idx, /* of old name */
+				mtd_op_t op,
+				loc_t *loc,
+				struct crypt_inode_info *info,
+				struct master_cipher_info *master,
+				crypt_local_t *local)
+{
+	switch (op) {
+	case MTD_APPEND:
+		mac_idx = 1 + (old_len - sizeof(struct mtd_format_v1))/8;
+	case MTD_OVERWRITE:
+		return appov_link_mac_v1(new, old, old_len, mac_idx,
+					 loc, info, master, local);
+	case MTD_CUT:
+		return cut_link_mac_v1(new, old, old_len, mac_idx,
+				       loc, info, master, local);
+	default:
+		gf_log("crypt", GF_LOG_ERROR, "Bad  mtd operation %d", op);
+		return -1;
+	}
+}
+
+/*
+ * Called by fops:
+ *
+ * ->link()
+ * ->unlink()
+ * ->rename()
+ *
+ */
+int32_t update_format(unsigned char *new,
+		      unsigned char *old,
+		      size_t old_len,
+		      int32_t mac_idx,
+		      mtd_op_t op,
+		      loc_t *loc,
+		      struct crypt_inode_info *info,
+		      struct master_cipher_info *master,
+		      crypt_local_t *local)
+{
+	if (!new)
+		return 0;
+	memcpy(new, old, sizeof(struct crypt_format));
+
+	old += sizeof(struct crypt_format);
+	new += sizeof(struct crypt_format);
+	old_len -= sizeof(struct crypt_format);
+
+	return mtd_loaders[current_mtd_loader()].update_format(new, old,
+							       old_len,
+							       mac_idx, op,
+							       loc, info,
+							       master, local);
+}
+
+/*
+ * Perform preliminary checks of found metadata
+ * Return < 0 on errors;
+ * Return number of object-id MACs (>= 1) on success
+ */
+int32_t check_format_v1(uint32_t len, unsigned char *wire)
+{
+	uint32_t nr_links;
+
+	if (len < sizeof(struct mtd_format_v1)) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "v1-loader: bad metadata size %d", len);
+		goto error;
+	}
+	len -= sizeof(struct mtd_format_v1);
+	if (len % sizeof(nmtd_8_mac_t)) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "v1-loader: bad metadata format");
+		goto error;
+	}
+	nr_links = 1 + len / sizeof(nmtd_8_mac_t);
+	if (nr_links > _POSIX_LINK_MAX)
+		goto error;
+	return nr_links;
+ error:
+	return EIO;
+}
+
+/*
+ * Verify per-link MAC specified by index @idx
+ *
+ * return:
+ * -1 on errors;
+ *  0 on failed verification;
+ *  1 on sucessful verification
+ */
+static int32_t verify_link_mac_v1(struct mtd_format_v1 *fmt,
+				  uint32_t idx /* index of the mac to verify */,
+				  loc_t *loc,
+				  struct crypt_inode_info *info,
+				  struct master_cipher_info *master)
+{
+	int32_t ret;
+	unsigned char *mac;
+	unsigned char cmac[16];
+
+	mac = get_NMTD_V1_MAC(fmt) + idx * SIZE_OF_NMTD_V1_MAC;
+
+	ret = calc_link_mac_v1(fmt, loc, cmac, info, master);
+	if (ret)
+		return -1;
+	if (memcmp(cmac, mac, SIZE_OF_NMTD_V1_MAC))
+		return 0;
+	return 1;
+}
+
+/*
+ * Lookup per-link MAC by pathname.
+ *
+ * return index of the MAC, if it was found;
+ * return < 0 on errors, or if the MAC wasn't found
+ */
+static int32_t lookup_link_mac_v1(struct mtd_format_v1 *fmt,
+				  uint32_t nr_macs,
+				  loc_t *loc,
+				  struct crypt_inode_info *info,
+				  struct master_cipher_info *master)
+{
+	int32_t ret;
+	uint32_t idx;
+
+	for (idx = 0; idx < nr_macs; idx++) {
+		ret = verify_link_mac_v1(fmt, idx, loc, info, master);
+		if (ret < 0)
+			return ret;
+		if (ret > 0)
+			return idx;
+	}
+	return -ENOENT;
+}
+
+/*
+ * Extract version-specific part of metadata
+ */
+static int32_t open_format_v1(unsigned char *wire,
+			      int32_t len,
+			      loc_t *loc,
+			      struct crypt_inode_info *info,
+			      struct master_cipher_info *master,
+			      crypt_local_t *local,
+			      gf_boolean_t load_info)
+{
+	int32_t ret;
+	int32_t num_nmtd_macs;
+	struct mtd_format_v1 *fmt;
+	unsigned char mtd_key[16];
+	AES_KEY EMTD_KEY;
+	GCM128_CONTEXT *gctx;
+	uint32_t ad;
+	emtd_8_mac_t gmac;
+	struct object_cipher_info *object;
+
+	num_nmtd_macs = check_format_v1(len, wire);
+	if (num_nmtd_macs <= 0)
+		return EIO;
+	fmt = (struct mtd_format_v1 *)wire;
+
+	ret = lookup_link_mac_v1(fmt, num_nmtd_macs, loc, info, master);
+	if (ret < 0) {
+		gf_log("crypt", GF_LOG_ERROR, "NMTD verification failed");
+		return EINVAL;
+	}
+	local->mac_idx = ret;
+	if (load_info == _gf_false)
+		/* the case of partial open */
+		return 0;
+
+	object = &info->cinfo;
+
+	ret = get_emtd_file_key(info, master, mtd_key);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, "Can not retrieve metadata key");
+		return ret;
+	}
+	/*
+	 * decrypt encrypted meta-data
+	 */
+	ret = AES_set_encrypt_key(mtd_key, sizeof(mtd_key)*8, &EMTD_KEY);
+	if (ret < 0) {
+		gf_log("crypt", GF_LOG_ERROR, "Can not set encrypt key");
+		return ret;
+	}
+	gctx = CRYPTO_gcm128_new(&EMTD_KEY, (block128_f)AES_encrypt);
+	if (!gctx) {
+		gf_log("crypt", GF_LOG_ERROR, "Can not alloc gcm context");
+		return ENOMEM;
+	}
+	CRYPTO_gcm128_setiv(gctx, info->oid, sizeof(uuid_t));
+
+	ad = htole32(MTD_LOADER_V1);
+	ret = CRYPTO_gcm128_aad(gctx, (const unsigned char *)&ad, sizeof(ad));
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_aad failed");
+		CRYPTO_gcm128_release(gctx);
+		return ret;	
+	}
+	ret = CRYPTO_gcm128_decrypt(gctx,
+				    get_EMTD_V1(fmt),
+				    get_EMTD_V1(fmt),
+				    SIZE_OF_EMTD_V1);
+	if (ret) {
+		gf_log("crypt", GF_LOG_ERROR, " CRYPTO_gcm128_decrypt failed");
+		CRYPTO_gcm128_release(gctx);
+		return ret;
+	}
+	/*
+	 * verify metadata
+	 */
+	CRYPTO_gcm128_tag(gctx, gmac, sizeof(gmac));
+	CRYPTO_gcm128_release(gctx);
+	if (memcmp(gmac, get_EMTD_V1_MAC(fmt), SIZE_OF_EMTD_V1_MAC)) {
+		gf_log("crypt", GF_LOG_ERROR, "EMTD verification failed");
+		return EINVAL;
+	}
+	/*
+	 * load verified metadata to the private part of inode
+	 */
+	info->nr_minor = fmt->minor_id;
+
+	object->o_alg = fmt->alg_id;
+	object->o_dkey_size = fmt->dkey_factor << KEY_FACTOR_BITS;
+	object->o_block_bits = fmt->block_bits;
+	object->o_mode = fmt->mode_id;
+
+	return check_file_metadata(info);
+}
+
+/*
+ * perform metadata authentication against @loc->path;
+ * extract crypt-specific attribtes and populate @info
+ * with them (optional)
+ */
+int32_t open_format(unsigned char *str,
+		    int32_t len,
+		    loc_t *loc,
+		    struct crypt_inode_info *info,
+		    struct master_cipher_info *master,
+		    crypt_local_t *local,
+		    gf_boolean_t load_info)
+{
+	struct crypt_format *fmt;
+	if (len < sizeof(*fmt)) {
+		gf_log("crypt", GF_LOG_ERROR, "Bad core format");
+		return EIO;
+	}
+	fmt = (struct crypt_format *)str;
+
+	if (fmt->loader_id >= LAST_MTD_LOADER) {
+		gf_log("crypt", GF_LOG_ERROR,
+		       "Unsupported loader id %d", fmt->loader_id);
+		return EINVAL;
+	}
+	str += sizeof(*fmt);
+	len -= sizeof(*fmt);
+
+	return mtd_loaders[fmt->loader_id].open_format(str,
+						       len,
+						       loc,
+						       info,
+						       master,
+						       local,
+						       load_info);
+}
+
+struct crypt_mtd_loader mtd_loaders [LAST_MTD_LOADER] = {
+	[MTD_LOADER_V1] = 
+	{.format_size = format_size_v1,
+	 .create_format = create_format_v1,
+	 .open_format = open_format_v1,
+	 .update_format = update_format_v1
+	}
+};
+
+/*
+  Local variables:
+  c-indentation-style: "K&R"
+  mode-name: "LC"
+  c-basic-offset: 8
+  tab-width: 8
+  fill-column: 80
+  scroll-step: 1
+  End:
+*/
diff --git a/xlators/encryption/crypt/src/metadata.h b/xlators/encryption/crypt/src/metadata.h
new file mode 100644
index 000000000..a92f149ef
--- /dev/null
+++ b/xlators/encryption/crypt/src/metadata.h
@@ -0,0 +1,74 @@
+/*
+  Copyright (c) 2008-2013 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  This file is licensed to you under your choice of the GNU Lesser
+  General Public License, version 3 or any later version (LGPLv3 or
+  later), or the GNU General Public License, version 2 (GPLv2), in all
+  cases as published by the Free Software Foundation.
+*/
+
+#ifndef __METADATA_H__
+#define __METADATA_H__
+
+#define NMTD_8_MAC_SIZE   (8)
+#define EMTD_8_MAC_SIZE   (8)
+
+typedef uint8_t nmtd_8_mac_t[NMTD_8_MAC_SIZE];
+typedef uint8_t emtd_8_mac_t[EMTD_8_MAC_SIZE] ;
+
+/*
+ * Version "v1" of file's metadata.
+ * Metadata of this version has 4 components:
+ * 
+ * 1) EMTD (Encrypted part of MeTaData);
+ * 2) NMTD (Non-encrypted part of MeTaData);
+ * 3) EMTD_MAC; (EMTD Message Authentication Code);
+ * 4) Array of per-link NMTD MACs (for every (hard)link it includes
+ *    exactly one MAC)
+ */
+struct mtd_format_v1 {
+	/* EMTD, encrypted part of meta-data */
+	uint8_t            alg_id;  /* cipher algorithm id (only AES for now) */
+        uint8_t            mode_id; /* cipher mode id; (only XTS for now) */
+	uint8_t            block_bits; /* encoded block size */
+	uint8_t            minor_id; /* client translator id */
+	uint8_t            dkey_factor; /* encoded size of the data key */
+	/* MACs */
+	emtd_8_mac_t       gmac; /* MAC of the encrypted meta-data, 8 bytes */
+	nmtd_8_mac_t       omac; /* per-link MACs of the non-encrypted
+				  * meta-data: at least one such MAC is always
+				  * present */
+} __attribute__((packed));
+
+/*
+ * NMTD, the non-encrypted part of metadata of version "v1"
+ * is file's gfid, which is generated on trusted machines.
+ */
+#define SIZE_OF_NMTD_V1 (sizeof(uuid_t))
+#define SIZE_OF_EMTD_V1 (offsetof(struct mtd_format_v1, gmac) -		\
+			 offsetof(struct mtd_format_v1, alg_id))
+#define SIZE_OF_NMTD_V1_MAC (NMTD_8_MAC_SIZE)
+#define SIZE_OF_EMTD_V1_MAC (EMTD_8_MAC_SIZE)
+
+static inline unsigned char *get_EMTD_V1(struct mtd_format_v1 *format)
+{
+        return &format->alg_id;
+}
+
+static inline unsigned char *get_NMTD_V1(struct crypt_inode_info *info)
+{
+        return info->oid;
+}
+
+static inline unsigned char *get_EMTD_V1_MAC(struct mtd_format_v1 *format)
+{
+        return format->gmac;
+}
+
+static inline unsigned char *get_NMTD_V1_MAC(struct mtd_format_v1 *format)
+{
+        return format->omac;
+}
+
+#endif /* __METADATA_H__ */
diff --git a/xlators/encryption/rot-13/src/Makefile.am b/xlators/encryption/rot-13/src/Makefile.am
index ba5e623d8..94e8d18e7 100644
--- a/xlators/encryption/rot-13/src/Makefile.am
+++ b/xlators/encryption/rot-13/src/Makefile.am
@@ -1,14 +1,15 @@
 xlator_LTLIBRARIES = rot-13.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/encryption
 
-rot_13_la_LDFLAGS = -module -avoidversion
+rot_13_la_LDFLAGS = -module -avoid-version
 
 rot_13_la_SOURCES = rot-13.c
 rot_13_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
 
 noinst_HEADERS = rot-13.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
diff --git a/xlators/encryption/rot-13/src/rot-13.c b/xlators/encryption/rot-13/src/rot-13.c
index 39549d8be..1bcfe0192 100644
--- a/xlators/encryption/rot-13/src/rot-13.c
+++ b/xlators/encryption/rot-13/src/rot-13.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2009 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <ctype.h>
 #include <sys/uio.h>
 
@@ -32,13 +22,13 @@
 #include "rot-13.h"
 
 /*
- * This is a rot13 ``encryption'' xlator. It rot13's data when 
- * writing to disk and rot13's it back when reading it. 
+ * This is a rot13 ``encryption'' xlator. It rot13's data when
+ * writing to disk and rot13's it back when reading it.
  * This xlator is meant as an example, NOT FOR PRODUCTION
  * USE ;) (hence no error-checking)
  */
 
-void 
+void
 rot13 (char *buf, int len)
 {
 	int i;
@@ -68,14 +58,15 @@ rot13_readv_cbk (call_frame_t *frame,
                  struct iovec *vector,
                  int32_t count,
 		 struct iatt *stbuf,
-                 struct iobref *iobref)
+                 struct iobref *iobref, dict_t *xdata)
 {
 	rot_13_private_t *priv = (rot_13_private_t *)this->private;
-  
+
 	if (priv->decrypt_read)
 		rot13_iovec (vector, count);
 
-	STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count, stbuf, iobref);
+	STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector, count,
+                             stbuf, iobref, xdata);
 	return 0;
 }
 
@@ -84,13 +75,13 @@ rot13_readv (call_frame_t *frame,
              xlator_t *this,
              fd_t *fd,
              size_t size,
-             off_t offset)
+             off_t offset, uint32_t flags, dict_t *xdata)
 {
 	STACK_WIND (frame,
 		    rot13_readv_cbk,
 		    FIRST_CHILD (this),
 		    FIRST_CHILD (this)->fops->readv,
-		    fd, size, offset);
+		    fd, size, offset, flags, xdata);
 	return 0;
 }
 
@@ -101,9 +92,10 @@ rot13_writev_cbk (call_frame_t *frame,
                   int32_t op_ret,
                   int32_t op_errno,
                   struct iatt *prebuf,
-		  struct iatt *postbuf)
+		  struct iatt *postbuf, dict_t *xdata)
 {
-	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+	STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
 	return 0;
 }
 
@@ -112,20 +104,20 @@ rot13_writev (call_frame_t *frame,
               xlator_t *this,
               fd_t *fd,
               struct iovec *vector,
-              int32_t count, 
-              off_t offset,
-              struct iobref *iobref)
+              int32_t count,
+              off_t offset, uint32_t flags,
+              struct iobref *iobref, dict_t *xdata)
 {
 	rot_13_private_t *priv = (rot_13_private_t *)this->private;
 	if (priv->encrypt_write)
 		rot13_iovec (vector, count);
 
-	STACK_WIND (frame, 
+	STACK_WIND (frame,
 		    rot13_writev_cbk,
 		    FIRST_CHILD (this),
 		    FIRST_CHILD (this)->fops->writev,
-		    fd, vector, count, offset,
-                    iobref);
+		    fd, vector, count, offset, flags,
+                    iobref, xdata);
 	return 0;
 }
 
@@ -136,7 +128,7 @@ init (xlator_t *this)
 	rot_13_private_t *priv = NULL;
 
 	if (!this->children || this->children->next) {
-		gf_log ("rot13", GF_LOG_ERROR, 
+		gf_log ("rot13", GF_LOG_ERROR,
 			"FATAL: rot13 should have exactly one child");
 		return -1;
 	}
@@ -158,6 +150,7 @@ init (xlator_t *this)
 		if (gf_string2boolean (data->data, &priv->encrypt_write) == -1) {
 			gf_log (this->name, GF_LOG_ERROR,
 				"encrypt-write takes only boolean options");
+			GF_FREE (priv);
 			return -1;
 		}
 	}
@@ -167,6 +160,7 @@ init (xlator_t *this)
 		if (gf_string2boolean (data->data, &priv->decrypt_read) == -1) {
 			gf_log (this->name, GF_LOG_ERROR,
 				"decrypt-read takes only boolean options");
+			GF_FREE (priv);
 			return -1;
 		}
 	}
@@ -194,15 +188,14 @@ struct xlator_fops fops = {
 	.writev       = rot13_writev
 };
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks;
 
 struct volume_options options[] = {
-	{ .key  = {"encrypt-write"}, 
+	{ .key  = {"encrypt-write"},
 	  .type = GF_OPTION_TYPE_BOOL
 	},
-	{ .key  = {"decrypt-read"}, 
-	  .type = GF_OPTION_TYPE_BOOL 
+	{ .key  = {"decrypt-read"},
+	  .type = GF_OPTION_TYPE_BOOL
 	},
 	{ .key  = {NULL} },
 };
diff --git a/xlators/encryption/rot-13/src/rot-13.h b/xlators/encryption/rot-13/src/rot-13.h
index 1fee3e44d..3e9fc19c7 100644
--- a/xlators/encryption/rot-13/src/rot-13.h
+++ b/xlators/encryption/rot-13/src/rot-13.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2006-2009 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __ROT_13_H__
 #define __ROT_13_H__
 
diff --git a/xlators/features/Makefile.am b/xlators/features/Makefile.am
index 8b23b939b..1fdd474c2 100644
--- a/xlators/features/Makefile.am
+++ b/xlators/features/Makefile.am
@@ -1,3 +1,4 @@
-SUBDIRS = locks trash quota read-only access-control mac-compat quiesce #path-converter # filter
+SUBDIRS = locks quota read-only mac-compat quiesce marker index barrier \
+	  protect compress changelog gfid-access $(GLUPY_SUBDIR) qemu-block # trash path-converter # filter
 
-CLEANFILES = 
+CLEANFILES =
diff --git a/xlators/features/access-control/src/Makefile.am b/xlators/features/access-control/src/Makefile.am
deleted file mode 100644
index 6ab8cc4ec..000000000
--- a/xlators/features/access-control/src/Makefile.am
+++ /dev/null
@@ -1,13 +0,0 @@
-xlator_LTLIBRARIES = access-control.la
-xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
-access_control_la_LDFLAGS = -module -avoidversion
-access_control_la_SOURCES = access-control.c
-access_control_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
-
-noinst_HEADERS = access-control.h
-
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS)\
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)\
-	-L$(xlatordir)/
-
-CLEANFILES =
diff --git a/xlators/features/access-control/src/access-control.c b/xlators/features/access-control/src/access-control.c
deleted file mode 100644
index 7fb065121..000000000
--- a/xlators/features/access-control/src/access-control.c
+++ /dev/null
@@ -1,1862 +0,0 @@
-/*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#include "access-control.h"
-#include "xlator.h"
-#include "call-stub.h"
-#include "defaults.h"
-#include "iatt.h"
-
-/* Careful, this function erases the stub from frame->local. Dont call this if
- * a subsequent callback requires retaining access to the stub. This should be
- * called at the end of all access-control related operations, i.e. once the
- * frame will be handed off to the actual fop and the next callback that will
- * be called is the default callback. IOW, the function where call_resume is
- * called.
- * NOTE: this is required because FRAME_DESTROY tries to free frame->local if
- * it finds it to be non-NULL.
- */
-call_stub_t *
-__get_frame_stub (call_frame_t *fr)
-{
-        call_stub_t     *st = NULL;
-
-        if (!fr)
-                return NULL;
-
-        st = fr->local;
-        fr->local = NULL;
-
-        return st;
-}
-
-
-int
-ac_test_owner_access (struct iatt *ia, uid_t uid, int accesstest)
-{
-        int     ret = -1;
-
-        if (!ia)
-                return -1;
-
-        /* First test permissions using the uid. */
-        if (ia->ia_uid != uid) {
-                ret = -1;
-                goto out;
-        }
-
-        /* At this point we know, the uid matches that of the stat structure, so
-         * if the caller does not care, we should return success.
-         */
-        if (ac_test_dontcare (accesstest)) {
-                ret = 0;
-                goto out;
-        }
-
-        if (ac_test_read (accesstest))
-                ret = IA_PROT_RUSR (ia->ia_prot);
-
-        if (ac_test_write (accesstest))
-                ret = IA_PROT_WUSR (ia->ia_prot);
-
-        if (ac_test_exec (accesstest))
-                ret = IA_PROT_XUSR (ia->ia_prot);
-
-        /* For failed access test for owner, we need to return EACCES */
-        if (!ret)
-                ret = -1;
-        else
-                ret = 0;
-out:
-        return ret;
-}
-
-
-int
-ac_test_group_access (struct iatt *ia, gid_t gid, gid_t *auxgids, int auxcount,
-                      int accesstest)
-{
-        int     ret = -1;
-        int     testgid = -1;
-        int     x = 0;
-
-        if (!ia)
-                return -1;
-        /* First, determine which gid to test against. This will be determined
-         * by first checking which of the gids given to us match the gid in the
-         * stat. If none match, then we go to checking with others as the user.
-         */
-
-        /* If we are only given the primary gid. Dont depend on @auxgids
-         * being NULL since I know users of this function can pass statically
-         * allocated arrays which cant be NULL and yet contain no valid gids.
-         */
-
-        if ((ia->ia_gid != gid) && (auxcount == 0)) {
-                ret = -1;
-                goto out;
-        }
-
-        if (ia->ia_gid == gid)
-                testgid = gid;
-        else {
-                for (; x < auxcount; ++x) {
-                        if (ia->ia_gid == auxgids[x]) {
-                                testgid = ia->ia_gid;
-                                break;
-                        }
-                }
-        }
-
-        /* None of the gids match with the gid in the stat. */
-        if (testgid == -1) {
-                ret = -1;
-                goto out;
-        }
-
-        /* At this point, at least one gid matches that in the stat, now we must
-         * check whether the caller is interested in the access check at all.
-         */
-        if (ac_test_dontcare (accesstest)) {
-                ret = 0;
-                goto out;
-        }
-
-        if (ac_test_read (accesstest))
-                ret = IA_PROT_RGRP (ia->ia_prot);
-
-        if (ac_test_write (accesstest))
-                ret = IA_PROT_WGRP (ia->ia_prot);
-
-        if (ac_test_exec (accesstest))
-                ret = IA_PROT_XGRP (ia->ia_prot);
-
-        if (!ret)
-                ret = -1;
-        else
-                ret = 0;
-
-out:
-        return ret;
-}
-
-
-int
-ac_test_other_access (struct iatt *ia, int accesstest)
-{
-        int     ret = 0;
-
-        if (!ia)
-                return -1;
-
-        if (ac_test_read (accesstest))
-                ret = IA_PROT_ROTH (ia->ia_prot);
-
-        if (ac_test_write (accesstest))
-                ret = IA_PROT_WOTH (ia->ia_prot);
-
-        if (ac_test_exec (accesstest))
-                ret = IA_PROT_XOTH (ia->ia_prot);
-
-        if (!ret)
-                ret = -1;
-        else
-                ret = 0;
-
-        return ret;
-}
-
-
-/* Returns -1 on a failed access test with @operrno set to the relevant error
- * number.
- */
-int
-ac_test_access (struct iatt *ia, uid_t uid, gid_t gid, gid_t *auxgids,
-                int auxcount, int accesstest, int testwho, int *operrno)
-{
-        int             ret = -1;
-
-        if ((!ia) || (!operrno))
-                return -1;
-
-        if ((uid == 0) && (gid == 0)) {
-                gf_log (ACTRL, GF_LOG_TRACE, "Root has access");
-                return 0;
-        }
-
-        if (ac_test_owner (testwho)) {
-                gf_log (ACTRL, GF_LOG_TRACE, "Testing owner access");
-                ret = ac_test_owner_access (ia, uid, accesstest);
-        }
-
-        if (ret == 0) {
-                gf_log (ACTRL, GF_LOG_TRACE, "Owner has access");
-                goto out;
-        }
-
-        if (ac_test_group (testwho)) {
-                gf_log (ACTRL, GF_LOG_TRACE, "Testing group access");
-                ret = ac_test_group_access (ia, gid, auxgids, auxcount,
-                                            accesstest);
-        }
-
-        if (ret == 0) {
-                gf_log (ACTRL, GF_LOG_TRACE, "Group has access");
-                goto out;
-        }
-
-        if (ac_test_other (testwho)) {
-                gf_log (ACTRL, GF_LOG_TRACE, "Testing other access");
-                ret = ac_test_other_access (ia, accesstest);
-        }
-
-        if (ret == 0)
-                gf_log (ACTRL, GF_LOG_TRACE, "Other has access");
-out:
-        if (ret == -1) {
-                gf_log (ACTRL, GF_LOG_TRACE, "No access allowed");
-                *operrno = EPERM;
-        }
-
-        return ret;
-}
-
-
-int
-ac_loc_fill (loc_t *loc, inode_t *inode, inode_t *parent, char *path)
-{
-        int     ret = -EFAULT;
-
-        if (!loc)
-                return ret;
-
-        if (inode) {
-                loc->inode = inode_ref (inode);
-                loc->ino = inode->ino;
-        }
-
-        if (parent)
-                loc->parent = inode_ref (parent);
-
-        loc->path = gf_strdup (path);
-        if (!loc->path) {
-                gf_log (ACTRL, GF_LOG_ERROR, "strdup failed");
-                goto loc_wipe;
-        }
-
-        loc->name = strrchr (loc->path, '/');
-        if (loc->name)
-                loc->name++;
-        else
-                goto loc_wipe;
-
-        ret = 0;
-loc_wipe:
-        if (ret < 0)
-                loc_wipe (loc);
-
-        return ret;
-}
-
-
-int
-ac_inode_loc_fill (inode_t *inode, loc_t *loc)
-{
-        char            *resolvedpath = NULL;
-        inode_t         *parent = NULL;
-        int             ret = -EFAULT;
-
-        if ((!inode) || (!loc))
-                return ret;
-
-        if ((inode) && (inode->ino == 1))
-                goto ignore_parent;
-
-        parent = inode_parent (inode, 0, NULL);
-        if (!parent)
-                goto err;
-
-ignore_parent:
-        ret = inode_path (inode, NULL, &resolvedpath);
-        if (ret < 0)
-                goto err;
-
-        ret = ac_loc_fill (loc, inode, parent, resolvedpath);
-        if (ret < 0)
-                goto err;
-
-err:
-        if (parent)
-                inode_unref (parent);
-
-        if (resolvedpath)
-                GF_FREE (resolvedpath);
-
-        return ret;
-}
-
-
-int
-ac_parent_loc_fill (loc_t *parentloc, loc_t *childloc)
-{
-        if ((!parentloc) || (!childloc))
-                return -1;
-
-        return ac_inode_loc_fill (childloc->parent, parentloc);
-}
-
-
-int32_t
-ac_truncate_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    off_t offset)
-{
-        STACK_WIND (frame, default_truncate_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->truncate, loc, offset);
-        return 0;
-}
-
-
-int32_t
-ac_truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        call_stub_t     *stub = NULL;
-
-        stub = __get_frame_stub (frame);
-        if (op_ret == -1)
-                goto out;
-
-        op_ret = ac_test_access (buf, frame->root->uid, frame->root->gid,
-                                 frame->root->groups, frame->root->ngrps,
-                                 ACCTEST_WRITE, ACCTEST_ANY, &op_errno);
-        if (op_ret == -1)
-                goto out;
-
-        call_resume (stub);
-out:
-        if (op_ret < 0) {
-                STACK_UNWIND_STRICT (truncate, frame, -1, op_errno, NULL, NULL);
-                if (stub)
-                        call_stub_destroy (stub);
-        }
-
-        return 0;
-}
-
-
-int32_t
-ac_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc, off_t offset)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -EFAULT;
-
-        stub = fop_truncate_stub (frame, ac_truncate_resume, loc, offset);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR, "cannot create call stub: "
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        frame->local = stub;
-        STACK_WIND (frame, ac_truncate_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, loc);
-
-        ret = 0;
-out:
-        if (ret < 0)
-                STACK_UNWIND_STRICT (truncate, frame, -1, -ret, NULL, NULL);
-
-        return 0;
-}
-
-
-int32_t
-ac_access_resume (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
-{
-        STACK_WIND (frame, default_access_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->access, loc, mask);
-        return 0;
-}
-
-
-int32_t
-ac_access_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        call_stub_t     *stub = NULL;
-        int32_t         mask = 0;
-        int             acctest = 0;
-
-        stub = __get_frame_stub (frame);
-        mask = stub->args.access.mask;
-
-        /* If mask requests test for file existence then do not
-         * return a failure with ENOENT, instead return a failed
-         * access test.
-         */
-        if (op_ret == -1) {
-                if (mask & F_OK)
-                        op_errno = EACCES;
-                else
-                        op_errno = errno;
-
-                goto out;
-        }
-
-        if (R_OK & mask)
-                acctest |= ACCTEST_READ;
-        else if (W_OK & mask)
-                acctest |= ACCTEST_WRITE;
-        else if (X_OK & mask)
-                acctest |= ACCTEST_EXEC;
-        else
-                acctest = 0;
-
-        op_ret = ac_test_access (buf, frame->root->uid, frame->root->gid,
-                                 frame->root->groups, frame->root->ngrps,
-                                 acctest, ACCTEST_ANY, &op_errno);
-        if (op_ret == -1)
-                goto out;
-
-        call_resume (stub);
-out:
-        if (op_ret < 0) {
-                STACK_UNWIND_STRICT (access, frame, -1, op_errno);
-                if (stub)
-                        call_stub_destroy (stub);
-        }
-
-        return 0;
-}
-
-
-int32_t
-ac_access (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t mask)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -EFAULT;
-
-        stub = fop_access_stub (frame, ac_access_resume, loc, mask);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR, "cannot create call stub: "
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        frame->local = stub;
-        STACK_WIND (frame, ac_access_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, loc);
-        ret = 0;
-
-out:
-        if (ret < 0)
-                STACK_UNWIND_STRICT (access, frame, -1, -ret);
-
-        return 0;
-}
-
-
-int32_t
-ac_readlink_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                    size_t size)
-{
-        STACK_WIND (frame, default_readlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->readlink, loc, size);
-        return 0;
-}
-
-
-int32_t
-ac_readlink_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        call_stub_t     *stub = NULL;
-
-        stub = __get_frame_stub (frame);
-        if (op_ret == -1)
-                goto out;
-
-        op_ret = ac_test_access (buf, frame->root->uid, frame->root->gid,
-                                 frame->root->groups, frame->root->ngrps,
-                                 ACCTEST_READ, ACCTEST_ANY, &op_errno);
-        if (op_ret == -1)
-                goto out;
-
-        call_resume (stub);
-out:
-        if (op_ret < 0) {
-                STACK_UNWIND_STRICT (readlink, frame, -1, op_errno, NULL, NULL);
-                if (stub)
-                        call_stub_destroy (stub);
-        }
-
-        return 0;
-}
-
-
-int32_t
-ac_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc, size_t size)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -EFAULT;
-
-        stub = fop_readlink_stub (frame, ac_readlink_resume, loc, size);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR, "cannot create call stub: "
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        frame->local = stub;
-        STACK_WIND (frame, ac_readlink_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, loc);
-        ret = 0;
-
-out:
-        if (ret < 0)
-                STACK_UNWIND_STRICT (readlink, frame, -1, -ret, NULL, NULL);
-
-        return 0;
-}
-
-
-int
-ac_mknod_resume (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-                 dev_t rdev, dict_t *params)
-{
-        STACK_WIND (frame, default_mknod_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev, params);
-        return 0;
-}
-
-
-int32_t
-ac_mknod_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        call_stub_t     *stub = NULL;
-
-        stub = __get_frame_stub (frame);
-        if (op_ret == -1)
-                goto out;
-
-        op_ret = ac_test_access (buf, frame->root->uid, frame->root->gid,
-                                 frame->root->groups, frame->root->ngrps,
-                                 ACCTEST_WRITE, ACCTEST_ANY, &op_errno);
-        if (op_ret == -1) {
-                op_errno = EACCES;
-                goto out;
-        }
-
-        call_resume (stub);
-out:
-        if (op_ret < 0) {
-                STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL);
-                if (stub)
-                        call_stub_destroy (stub);
-        }
-
-        return 0;
-}
-
-
-int
-ac_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          dev_t rdev, dict_t *params)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -EFAULT;
-        loc_t           parentloc = {0, };
-
-        stub = fop_mknod_stub (frame, ac_mknod_resume, loc, mode, rdev, params);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR, "cannot create call stub: "
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        frame->local = stub;
-        ret = ac_parent_loc_fill (&parentloc, loc);
-        if (ret < 0)
-                goto out;
-
-        STACK_WIND (frame, ac_mknod_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, &parentloc);
-        loc_wipe (&parentloc);
-        ret = 0;
-
-out:
-        if (ret < 0) {
-                /* Erase any stored frame before unwinding. */
-                stub = __get_frame_stub (frame);
-                if (stub)
-                        call_stub_destroy (stub);
-                STACK_UNWIND_STRICT (mknod, frame, -1, -ret, NULL, NULL, NULL,
-                                     NULL);
-        }
-
-        return 0;
-}
-
-
-int
-ac_mkdir_resume (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-                 dict_t *params)
-{
-        STACK_WIND (frame, default_mkdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->mkdir, loc, mode, params);
-        return 0;
-}
-
-
-int32_t
-ac_mkdir_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        call_stub_t     *stub = NULL;
-
-        stub = __get_frame_stub (frame);
-        if (op_ret == -1)
-                goto out;
-
-        op_ret = ac_test_access (buf, frame->root->uid, frame->root->gid,
-                                 frame->root->groups, frame->root->ngrps,
-                                 ACCTEST_WRITE, ACCTEST_ANY, &op_errno);
-        if (op_ret == -1) {
-                /* On a failed  write test on parent dir, we need to return
-                 * EACCES, not EPERM that is returned by default by
-                 * ac_test_access.
-                 */
-                op_errno = EACCES;
-                goto out;
-        }
-
-        call_resume (stub);
-out:
-        if (op_ret < 0) {
-                STACK_UNWIND_STRICT (mkdir, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL);
-                if (stub)
-                        call_stub_destroy (stub);
-        }
-
-        return 0;
-}
-
-
-int
-ac_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
-          dict_t *params)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -EFAULT;
-        loc_t           parentloc = {0, };
-
-        stub = fop_mkdir_stub (frame, ac_mkdir_resume, loc, mode, params);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR, "cannot create call stub: "
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        frame->local = stub;
-        ret = ac_parent_loc_fill (&parentloc, loc);
-        if (ret < 0)
-                goto out;
-
-        STACK_WIND (frame, ac_mkdir_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, &parentloc);
-        loc_wipe (&parentloc);
-        ret = 0;
-
-out:
-        if (ret < 0) {
-                /* Erase the stored stub before unwinding. */
-                stub = __get_frame_stub (frame);
-                if (stub)
-                        call_stub_destroy (stub);
-                STACK_UNWIND_STRICT (mkdir, frame, -1, -ret, NULL, NULL, NULL,
-                                     NULL);
-        }
-
-        return 0;
-}
-
-
-int32_t
-ac_unlink_resume (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        STACK_WIND (frame, default_unlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->unlink, loc);
-        return 0;
-}
-
-
-int32_t
-ac_unlink_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        call_stub_t     *stub = NULL;
-
-        stub = __get_frame_stub (frame);
-        if (op_ret == -1)
-                goto out;
-
-        op_ret = ac_test_access (buf, frame->root->uid, frame->root->gid,
-                                 frame->root->groups, frame->root->ngrps,
-                                 ACCTEST_WRITE, ACCTEST_ANY, &op_errno);
-        if (op_ret == -1) {
-                op_errno = EACCES;
-                goto out;
-        }
-
-        call_resume (stub);
-out:
-        if (op_ret < 0) {
-                STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL, NULL);
-                if (stub)
-                        call_stub_destroy (stub);
-        }
-
-        return 0;
-}
-
-
-int32_t
-ac_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -EFAULT;
-        loc_t           parentloc = {0, };
-
-        stub = fop_unlink_stub (frame, ac_unlink_resume, loc);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR, "cannot create call stub: "
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        frame->local = stub;
-        ret = ac_parent_loc_fill (&parentloc, loc);
-        if (ret < 0)
-                goto out;
-
-        STACK_WIND (frame, ac_unlink_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, &parentloc);
-        loc_wipe (&parentloc);
-        ret = 0;
-
-out:
-        if (ret < 0) {
-                /* Erase the stored stub before unwinding. */
-                stub = __get_frame_stub (frame);
-                if (stub)
-                        call_stub_destroy (stub);
-                STACK_UNWIND_STRICT (unlink, frame, -1, -ret, NULL, NULL);
-        }
-
-        return 0;
-}
-
-
-int
-ac_rmdir_resume (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
-{
-        STACK_WIND (frame, default_rmdir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rmdir, loc, flags);
-        return 0;
-}
-
-
-int32_t
-ac_rmdir_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        call_stub_t     *stub = NULL;
-
-        stub = __get_frame_stub (frame);
-        if (op_ret == -1)
-                goto out;
-
-        op_ret = ac_test_access (buf, frame->root->uid, frame->root->gid,
-                                 frame->root->groups, frame->root->ngrps,
-                                 ACCTEST_WRITE, ACCTEST_ANY, &op_errno);
-        if (op_ret == -1) {
-                op_errno = EACCES;
-                goto out;
-        }
-
-        call_resume (stub);
-out:
-        if (op_ret < 0) {
-                STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL, NULL);
-                if (stub)
-                        call_stub_destroy (stub);
-        }
-
-        return 0;
-}
-
-
-int
-ac_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -EFAULT;
-        loc_t           parentloc = {0, };
-
-        stub = fop_rmdir_stub (frame, ac_rmdir_resume, loc, flags);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR, "cannot create call stub: "
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        frame->local = stub;
-        ret = ac_parent_loc_fill (&parentloc, loc);
-        if (ret < 0)
-                goto out;
-
-        STACK_WIND (frame, ac_rmdir_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, &parentloc);
-        loc_wipe (&parentloc);
-        ret = 0;
-
-out:
-        if (ret < 0) {
-                /* Erase the stored stub before unwinding. */
-                stub = __get_frame_stub (frame);
-                if (stub)
-                        call_stub_destroy (stub);
-                STACK_UNWIND_STRICT (rmdir, frame, -1, -ret, NULL, NULL);
-        }
-
-        return 0;
-}
-
-
-int32_t
-ac_symlink_resume (call_frame_t *frame, xlator_t *this, const char *linkname,
-                   loc_t *loc, dict_t *params)
-{
-        STACK_WIND (frame, default_symlink_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->symlink, linkname, loc, params);
-        return 0;
-}
-
-
-int32_t
-ac_symlink_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        call_stub_t     *stub = NULL;
-
-        stub = __get_frame_stub (frame);
-        if (op_ret == -1)
-                goto out;
-
-        op_ret = ac_test_access (buf, frame->root->uid, frame->root->gid,
-                                 frame->root->groups, frame->root->ngrps,
-                                 ACCTEST_WRITE, ACCTEST_ANY, &op_errno);
-        if (op_ret == -1) {
-                op_errno = EACCES;
-                goto out;
-        }
-
-        call_resume (stub);
-out:
-        if (op_ret < 0) {
-                STACK_UNWIND_STRICT (symlink, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL);
-                if (stub)
-                        call_stub_destroy (stub);
-        }
-
-        return 0;
-}
-
-
-int
-ac_symlink (call_frame_t *frame, xlator_t *this, const char *linkname,
-            loc_t *loc, dict_t *params)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -EFAULT;
-        loc_t           parentloc = {0, };
-
-        stub = fop_symlink_stub (frame, ac_symlink_resume, linkname, loc,
-                                 params);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR, "cannot create call stub: "
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        frame->local = stub;
-        ret = ac_parent_loc_fill (&parentloc, loc);
-        if (ret < 0)
-                goto out;
-
-        STACK_WIND (frame, ac_symlink_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, &parentloc);
-        loc_wipe (&parentloc);
-        ret = 0;
-
-out:
-        if (ret < 0) {
-                /* Erase the stored stub before unwinding. */
-                stub = __get_frame_stub (frame);
-                if (stub)
-                        call_stub_destroy (stub);
-                STACK_UNWIND_STRICT (symlink, frame, -1, -ret, NULL, NULL, NULL,
-                                     NULL);
-        }
-
-        return 0;
-}
-
-
-int32_t
-ac_rename_resume (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                  loc_t *newloc)
-{
-        STACK_WIND (frame, default_rename_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->rename, oldloc, newloc);
-        return 0;
-}
-
-
-int32_t
-ac_rename_dst_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        call_stub_t     *stub = NULL;
-
-        stub = __get_frame_stub (frame);
-        if (op_ret == -1)
-                goto out;
-
-        op_ret = ac_test_access (buf, frame->root->uid,
-                                 frame->root->gid, frame->root->groups,
-                                 frame->root->ngrps, ACCTEST_WRITE,
-                                 ACCTEST_ANY, &op_errno);
-        if (op_ret == -1) {
-                op_errno = EACCES;
-               goto out;
-        }
-
-        call_resume (stub);
-out:
-        if (op_ret < 0) {
-                STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, NULL);
-                if (stub)
-                        call_stub_destroy (stub);
-        }
-
-        return 0;
-}
-
-
-int32_t
-ac_rename_src_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                        int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        call_stub_t     *stub = NULL;
-        loc_t           parentloc = {0, };
-
-        stub = frame->local;
-        if (op_ret == -1)
-                goto out;
-
-        op_ret = ac_test_access (buf, frame->root->uid,
-                                 frame->root->gid, frame->root->groups,
-                                 frame->root->ngrps, ACCTEST_WRITE,
-                                 ACCTEST_ANY, &op_errno);
-        if (op_ret == -1) {
-                op_errno = EACCES;
-                goto out;
-        }
-
-        op_ret = ac_parent_loc_fill (&parentloc, &stub->args.rename.new);
-        if (op_ret < 0) {
-                op_errno = -EFAULT;
-                goto out;
-        }
-
-        STACK_WIND (frame, ac_rename_dst_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, &parentloc);
-        loc_wipe (&parentloc);
-
-out:
-        if (op_ret < 0) {
-                /* Erase the stored stub before unwinding. */
-                stub = __get_frame_stub (frame);
-                if (stub)
-                        call_stub_destroy (stub);
-                STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, NULL);
-        }
-
-        return 0;
-}
-
-
-int32_t
-ac_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -EFAULT;
-        loc_t           parentloc = {0, };
-
-        stub = fop_rename_stub (frame, ac_rename_resume, oldloc, newloc);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR, "cannot create call stub: "
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        frame->local = stub;
-        ret = ac_parent_loc_fill (&parentloc, oldloc);
-        if (ret < 0)
-                goto out;
-
-        STACK_WIND (frame, ac_rename_src_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, &parentloc);
-        loc_wipe (&parentloc);
-        ret = 0;
-
-out:
-        if (ret < 0) {
-                /* Erase the stored stub before unwinding. */
-                stub = __get_frame_stub (frame);
-                if (stub)
-                        call_stub_destroy (stub);
-                STACK_UNWIND_STRICT (rename, frame, -1, -ret, NULL, NULL, NULL,
-                                     NULL, NULL);
-        }
-
-        return 0;
-}
-
-
-int32_t
-ac_link_resume (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
-                loc_t *newloc)
-{
-        STACK_WIND (frame, default_link_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->link, oldloc, newloc);
-        return 0;
-}
-
-
-int32_t
-ac_link_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        call_stub_t     *stub = NULL;
-
-        stub = __get_frame_stub (frame);
-        if (op_ret == -1)
-                goto out;
-
-        op_ret = ac_test_access (buf, frame->root->uid, frame->root->gid,
-                                 frame->root->groups, frame->root->ngrps,
-                                 ACCTEST_WRITE, ACCTEST_ANY, &op_errno);
-        if (op_ret == -1) {
-                /* By default ac_test_access sets the op_errno to EPERM
-                 * but in the case of link, we need to return EACCES to meet
-                 * posix requirements when a write permission is not available
-                 * for the new directory.
-                 */
-                op_errno = EACCES;
-                goto out;
-        }
-
-        call_resume (stub);
-out:
-        if (op_ret < 0) {
-                STACK_UNWIND_STRICT (link, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL);
-                if (stub)
-                        call_stub_destroy (stub);
-        }
-
-        return 0;
-}
-
-
-int32_t
-ac_link (call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -EFAULT;
-        loc_t           parentloc = {0, };
-
-        stub = fop_link_stub (frame, ac_link_resume, oldloc, newloc);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR, "cannot create call stub: "
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        frame->local = stub;
-        ret = ac_parent_loc_fill (&parentloc, newloc);
-        if (ret < 0)
-                goto out;
-
-        STACK_WIND (frame, ac_link_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, &parentloc);
-        loc_wipe (&parentloc);
-        ret = 0;
-
-out:
-        if (ret < 0) {
-                /* Erase the stored stub before unwinding. */
-                stub = __get_frame_stub (frame);
-                if (stub)
-                        call_stub_destroy (stub);
-                STACK_UNWIND_STRICT (link, frame, -1, -ret, NULL, NULL, NULL,
-                                     NULL);
-        }
-
-        return 0;
-}
-
-
-int32_t
-ac_create_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
-{
-        STACK_WIND (frame, default_create_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->create, loc, flags, mode,
-                    fd, params);
-        return 0;
-}
-
-
-int32_t
-ac_create_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        call_stub_t     *stub = NULL;
-
-        stub = __get_frame_stub (frame);
-        if (op_ret == -1)
-                goto out;
-
-        op_ret = ac_test_access (buf, frame->root->uid, frame->root->gid,
-                                 frame->root->groups, frame->root->ngrps,
-                                 ACCTEST_WRITE, ACCTEST_ANY, &op_errno);
-         if (op_ret == -1) {
-                op_errno = EACCES;
-                goto out;
-         }
-
-        call_resume (stub);
-out:
-        if (op_ret < 0) {
-                STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL, NULL,
-                                     NULL, NULL, NULL);
-                if (stub)
-                        call_stub_destroy (stub);
-        }
-
-        return 0;
-}
-
-
-int32_t
-ac_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-           mode_t mode, fd_t *fd, dict_t *params)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -EFAULT;
-        loc_t           parentloc = {0, };
-
-        stub = fop_create_stub (frame, ac_create_resume, loc, flags, mode,
-                                fd, params);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR, "cannot create call stub: "
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        frame->local = stub;
-        ret = ac_parent_loc_fill (&parentloc, loc);
-        if (ret < 0)
-                goto out;
-
-        STACK_WIND (frame, ac_create_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, &parentloc);
-        loc_wipe (&parentloc);
-        ret = 0;
-
-out:
-        if (ret < 0) {
-                /* Erase the stored stub before unwinding. */
-                stub = __get_frame_stub (frame);
-                if (stub)
-                        call_stub_destroy (stub);
-                STACK_UNWIND_STRICT (create, frame, -1, -ret, NULL, NULL, NULL,
-                                     NULL, NULL);
-        }
-
-        return 0;
-}
-
-
-int32_t
-ac_open_resume (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-                fd_t *fd, int32_t wbflags)
-{
-        STACK_WIND (frame, default_open_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->open, loc, flags, fd, wbflags);
-        return 0;
-}
-
-
-int32_t
-ac_open_create_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                         int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        call_stub_t     *stub = NULL;
-
-        stub = __get_frame_stub (frame);
-        if (op_ret == -1)
-                goto out;
-
-        op_ret = ac_test_access (buf, frame->root->uid, frame->root->gid,
-                                 frame->root->groups, frame->root->ngrps,
-                                 ACCTEST_WRITE, ACCTEST_ANY, &op_errno);
-         if (op_ret == -1) {
-                op_errno = EACCES;
-                goto out;
-         }
-
-        call_resume (stub);
-out:
-        if (op_ret < 0) {
-                STACK_UNWIND_STRICT (open, frame, -1, op_errno, NULL);
-                if (stub)
-                        call_stub_destroy (stub);
-        }
-
-        return 0;
-}
-
-
-int
-ac_open_create (call_stub_t *stub)
-{
-        int             ret = -EFAULT;
-        loc_t           parentloc = {0, };
-        xlator_t        *this = NULL;
-
-        if (!stub)
-                return ret;
-
-        ret = ac_parent_loc_fill (&parentloc, &stub->args.open.loc);
-        if (ret < 0)
-                goto out;
-
-        this = stub->frame->this;
-        STACK_WIND (stub->frame, ac_open_create_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, &parentloc);
-        loc_wipe (&parentloc);
-        ret = 0;
-
-out:
-        return ret;
-}
-
-
-int32_t
-ac_open_only_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        call_stub_t     *stub = NULL;
-        int             acctest = 0;
-        int32_t         flags = 0;
-
-        stub = __get_frame_stub (frame);
-        if (op_ret == -1)
-                goto out;
-
-        flags = stub->args.open.flags;
-        /* The permissions we test for depend on how the open needs to be
-         * performed. */
-        if ((flags & O_ACCMODE) == O_RDONLY)
-                acctest = ACCTEST_READ;
-        else if (((flags & O_ACCMODE) == O_RDWR) ||
-                 ((flags & O_ACCMODE) == O_WRONLY))
-                acctest = ACCTEST_WRITE;
-
-        op_ret = ac_test_access (buf, frame->root->uid, frame->root->gid,
-                                 frame->root->groups, frame->root->ngrps,
-                                 acctest, ACCTEST_ANY, &op_errno);
-        if (op_ret == -1)
-                goto out;
-
-        call_resume (stub);
-out:
-        if (op_ret < 0) {
-                STACK_UNWIND_STRICT (open, frame, -1, op_errno, NULL);
-                if (stub)
-                        call_stub_destroy (stub);
-        }
-
-        return 0;
-}
-
-
-int
-ac_open_only (call_stub_t *stub)
-{
-        int             ret = -EFAULT;
-        xlator_t        *this = NULL;
-
-        if (!stub)
-                return ret;
-
-        this = stub->frame->this;
-        STACK_WIND (stub->frame, ac_open_only_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, &stub->args.open.loc);
-        return 0;
-}
-
-int32_t
-ac_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, int32_t wbflags)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -EFAULT;
-
-        stub = fop_open_stub (frame, ac_open_resume, loc, flags, fd, wbflags);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR, "cannot create call stub: "
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        frame->local = stub;
-        /* If we are not supposed to create the file then there is no need to
-         * check the parent dir permissions. */
-        if (flags & O_CREAT)
-                ret = ac_open_create (stub);
-        else
-                ret = ac_open_only (stub);
-
-out:
-        if (ret < 0) {
-                /* Erase the stored stub before unwinding. */
-                stub = __get_frame_stub (frame);
-                if (stub)
-                        call_stub_destroy (stub);
-                STACK_UNWIND_STRICT (open, frame, -1, -ret, NULL);
-        }
-
-        return 0;
-}
-
-
-int32_t
-ac_readv_resume (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-                 off_t offset)
-{
-        STACK_WIND (frame, default_readv_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->readv, fd, size, offset);
-        return 0;
-}
-
-
-int32_t
-ac_readv_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                    int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        call_stub_t     *stub = NULL;
-
-        stub = __get_frame_stub (frame);
-        if (op_ret == -1)
-                goto out;
-
-        op_ret = ac_test_access (buf, frame->root->uid, frame->root->gid,
-                                 frame->root->groups, frame->root->ngrps,
-                                 ACCTEST_READ, ACCTEST_ANY, &op_errno);
-        if (op_ret == -1) {
-                op_errno = EACCES;
-                goto out;
-        }
-
-        call_resume (stub);
-out:
-        if (op_ret < 0) {
-                STACK_UNWIND_STRICT (readv, frame, -1, op_errno, NULL, 0, NULL,
-                                     NULL);
-                if (stub)
-                        call_stub_destroy (stub);
-        }
-
-        return 0;
-}
-
-
-int32_t
-ac_readv (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
-          off_t offset)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -EFAULT;
-
-        stub = fop_readv_stub (frame, ac_readv_resume, fd, size, offset);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR, "cannot create call stub: "
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        frame->local = stub;
-        STACK_WIND (frame, ac_readv_fstat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fstat, fd);
-        ret = 0;
-
-out:
-        if (ret < 0)
-                STACK_UNWIND_STRICT (readv, frame, -1, -ret, NULL, 0, NULL,
-                                     NULL);
-
-        return 0;
-}
-
-
-int32_t
-ac_writev_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                  struct iovec *vector, int32_t count, off_t offset,
-                  struct iobref *iobref)
-{
-        STACK_WIND (frame, default_writev_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->writev, fd, vector, count, offset,
-                    iobref);
-        return 0;
-}
-
-
-int32_t
-ac_writev_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        call_stub_t     *stub = NULL;
-
-        stub = __get_frame_stub (frame);
-        if (op_ret == -1)
-                goto out;
-
-        op_ret = ac_test_access (buf, frame->root->uid, frame->root->gid,
-                                 frame->root->groups, frame->root->ngrps,
-                                 ACCTEST_WRITE, ACCTEST_ANY, &op_errno);
-        if (op_ret == -1) {
-                op_errno = EACCES;
-                goto out;
-        }
-
-        call_resume (stub);
-out:
-        if (op_ret < 0) {
-                STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL);
-                if (stub)
-                        call_stub_destroy (stub);
-        }
-
-        return 0;
-}
-
-
-int32_t
-ac_writev (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iovec *vector,
-           int32_t count, off_t offset, struct iobref *iobref)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -EFAULT;
-
-        stub = fop_writev_stub (frame, ac_writev_resume, fd, vector, count,
-                                offset, iobref);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR, "cannot create call stub: "
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        frame->local = stub;
-        STACK_WIND (frame, ac_writev_fstat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fstat, fd);
-        ret = 0;
-
-out:
-        if (ret < 0)
-                STACK_UNWIND_STRICT (writev, frame, -1, -ret, NULL, NULL);
-
-        return 0;
-}
-
-
-int32_t
-ac_opendir_resume (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
-{
-        STACK_WIND (frame, default_opendir_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->opendir, loc, fd);
-        return 0;
-}
-
-
-int32_t
-ac_opendir_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        call_stub_t     *stub = NULL;
-
-        stub = __get_frame_stub (frame);
-        if (op_ret == -1)
-                goto out;
-
-        op_ret = ac_test_access (buf, frame->root->uid, frame->root->gid,
-                                 frame->root->groups, frame->root->ngrps,
-                                 ACCTEST_READ, ACCTEST_ANY, &op_errno);
-        if (op_ret == -1)
-                goto out;
-
-        call_resume (stub);
-out:
-        if (op_ret < 0) {
-                STACK_UNWIND_STRICT (opendir, frame, -1, op_errno, NULL);
-                if (stub)
-                        call_stub_destroy (stub);
-        }
-
-        return 0;
-}
-
-int32_t
-ac_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc, fd_t *fd)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -EFAULT;
-
-        stub = fop_opendir_stub (frame, ac_opendir_resume, loc, fd);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR, "cannot create call stub: "
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        frame->local = stub;
-        STACK_WIND (frame, ac_opendir_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, loc);
-        ret = 0;
-
-out:
-        if (ret < 0)
-                STACK_UNWIND_STRICT (opendir, frame, -1, -ret, NULL);
-
-        return 0;
-}
-
-
-int32_t
-ac_setattr_resume (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                   struct iatt *buf, int32_t valid)
-{
-        STACK_WIND (frame, default_setattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->setattr, loc, buf, valid);
-        return 0;
-}
-
-
-int32_t
-ac_setattr_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                     int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        call_stub_t     *stub = NULL;
-        int32_t         valid = 0;
-        struct iatt     *setbuf = NULL;
-
-        stub = __get_frame_stub (frame);
-        if (op_ret == -1)
-                goto out;
-
-        op_ret = ac_test_access (buf, frame->root->uid, frame->root->gid,
-                                 frame->root->groups, frame->root->ngrps,
-                                 ACCTEST_DONTCARE, ACCTEST_OWNER,
-                                 &op_errno);
-        if (op_ret == -1)
-                goto out;
-
-        valid = stub->args.setattr.valid;
-        setbuf = &stub->args.setattr.stbuf;
-        if (gf_attr_uid_set (valid) || gf_attr_gid_set (valid)) {
-                /* chown returns EPERM if the operation would change the
-                 * ownership, but the effective user ID is not the
-                 * super-user and the process is not an owner of the file.
-                 * Ref: posix-testsuite/chown/07.t
-                 */
-                if ((frame->root->uid != 0) && (gf_attr_uid_set (valid))) {
-                        if (buf->ia_uid != setbuf->ia_uid) {
-                                op_ret = -1;
-                                op_errno = EPERM;
-                                goto out;
-                        }
-                }
-
-                /* non-super-user can modify file group if he is owner of a
-                 * file and gid he is setting is in his groups list.
-                 * Ref: posix-testsuite/chown/00.t
-                 */
-                if ((frame->root->uid != 0) && (gf_attr_gid_set (valid))) {
-                        if (frame->root->uid != buf->ia_uid) {
-                                op_ret = -1;
-                                op_errno = EPERM;
-                                goto out;
-                        }
-
-                        op_ret = ac_test_access (setbuf, 0, frame->root->gid,
-                                                 frame->root->groups,
-                                                 frame->root->ngrps,
-                                                 ACCTEST_DONTCARE,
-                                                 ACCTEST_GROUP, &op_errno);
-                        if (op_ret == -1)
-                                goto out;
-                }
-        }
-
-        call_resume (stub);
-out:
-        if (op_ret < 0) {
-                STACK_UNWIND_STRICT (setattr, frame, -1, op_errno, NULL, NULL);
-                if (stub)
-                        call_stub_destroy (stub);
-        }
-
-        return 0;
-}
-
-int32_t
-ac_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *buf,
-            int32_t valid)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -EFAULT;
-
-        stub = fop_setattr_stub (frame, ac_setattr_resume, loc, buf, valid);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR, "cannot create call stub: "
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        frame->local = stub;
-        STACK_WIND (frame, ac_setattr_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, loc);
-        ret = 0;
-
-out:
-        if (ret < 0)
-                STACK_UNWIND_STRICT (setattr, frame, -1, -ret, NULL, NULL);
-
-        return 0;
-}
-
-
-int32_t
-ac_fsetattr_resume (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                    struct iatt *buf, int32_t valid)
-{
-        STACK_WIND (frame, default_fsetattr_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fsetattr, fd, buf, valid);
-        return 0;
-}
-
-
-int32_t
-ac_fsetattr_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                       int32_t op_ret, int32_t op_errno, struct iatt *buf)
-{
-        call_stub_t     *stub = NULL;
-        int32_t         valid = 0;
-        struct iatt     *setbuf = NULL;
-
-        stub = __get_frame_stub (frame);
-        if (op_ret == -1)
-                goto out;
-
-        op_ret = ac_test_access (buf, frame->root->uid, frame->root->gid,
-                                 frame->root->groups, frame->root->ngrps,
-                                 ACCTEST_DONTCARE, ACCTEST_OWNER,
-                                 &op_errno);
-        if (op_ret == -1)
-                goto out;
-
-        valid = stub->args.fsetattr.valid;
-        setbuf = &stub->args.fsetattr.stbuf;
-        if (gf_attr_uid_set (valid) && gf_attr_gid_set (valid)) {
-                /* chown returns EPERM if the operation would change the
-                 * ownership, but the effective user ID is not the
-                 * super-user and the process is not an owner of the file.
-                 * Ref: posix-testsuite/chown/07.t
-                 */
-                if ((frame->root->uid != 0) && (gf_attr_uid_set (valid))) {
-                        if (buf->ia_uid != setbuf->ia_uid) {
-                                op_ret = -1;
-                                op_errno = EPERM;
-                                goto out;
-                        }
-                }
-
-                /* non-super-user can modify file group if he is owner of a
-                 * file and gid he is setting is in his groups list.
-                 * Ref: posix-testsuite/chown/00.t
-                 */
-                if ((frame->root->uid != 0) && (gf_attr_gid_set (valid))) {
-                        if (frame->root->uid != buf->ia_uid) {
-                                op_ret = -1;
-                                op_errno = EPERM;
-                                goto out;
-                        }
-
-                        op_ret = ac_test_access (buf, 0, frame->root->gid,
-                                                 frame->root->groups,
-                                                 frame->root->ngrps,
-                                                 ACCTEST_DONTCARE,
-                                                 ACCTEST_GROUP, &op_errno);
-                        if (op_ret == -1)
-                                goto out;
-                }
-        }
-
-        call_resume (stub);
-out:
-        if (op_ret < 0) {
-                STACK_UNWIND_STRICT (fsetattr, frame, -1, op_errno, NULL, NULL);
-                if (stub)
-                        call_stub_destroy (stub);
-        }
-
-        return 0;
-}
-
-
-int32_t
-ac_fsetattr (call_frame_t *frame, xlator_t *this, fd_t *fd, struct iatt *buf,
-             int32_t valid)
-{
-        call_stub_t     *stub = NULL;
-        int             ret = -EFAULT;
-
-        stub = fop_fsetattr_stub (frame, ac_fsetattr_resume, fd, buf, valid);
-	if (!stub) {
-		gf_log (this->name, GF_LOG_ERROR, "cannot create call stub: "
-                        "(out of memory)");
-                ret = -ENOMEM;
-                goto out;
-	}
-
-        frame->local = stub;
-        STACK_WIND (frame, ac_fsetattr_fstat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->fstat, fd);
-        ret = 0;
-
-out:
-        if (ret < 0)
-                STACK_UNWIND_STRICT (fsetattr, frame, -1, -ret, NULL, NULL);
-
-        return 0;
-}
-
-
-struct xlator_fops fops = {
-        .truncate       = ac_truncate,
-        .access         = ac_access,
-        .readlink       = ac_readlink,
-        .mknod          = ac_mknod,
-        .mkdir          = ac_mkdir,
-        .unlink         = ac_unlink,
-        .rmdir          = ac_rmdir,
-        .symlink        = ac_symlink,
-        .rename         = ac_rename,
-        .link           = ac_link,
-        .create         = ac_create,
-        .open           = ac_open,
-        .readv          = ac_readv,
-        .writev         = ac_writev,
-        .opendir        = ac_opendir,
-        .setattr        = ac_setattr,
-        .fsetattr       = ac_fsetattr,
-};
-
-int
-init (xlator_t *this)
-{
-        int ret = -1;
-
-	if (!this->children || this->children->next) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"FATAL: access-control not configured with "
-                        "exactly one  child");
-                goto out;
-	}
-
-	if (!this->parents) {
-		gf_log (this->name, GF_LOG_WARNING,
-			"dangling volume. check volfile ");
-	}
-
-        ret = 0;
-out:
-        return ret;
-}
-
-void
-fini (xlator_t *this)
-{
-        return;
-}
-
-struct xlator_cbks cbks = {
-};
diff --git a/xlators/features/access-control/src/access-control.h b/xlators/features/access-control/src/access-control.h
deleted file mode 100644
index 8e2d6b6b9..000000000
--- a/xlators/features/access-control/src/access-control.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
-  Copyright (c) 2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
-
-
-#ifndef __ACCESS_CONTROL_H_
-#define __ACCESS_CONTROL_H_
-
-#ifndef _CONFIG_H
-#define _CONFIG_H
-#include "config.h"
-#endif
-
-#define ACTRL                   "access-control"
-#define ACCTEST_READ            0x1
-#define ACCTEST_WRITE           0x2
-#define ACCTEST_EXEC            0x4
-#define ACCTEST_DONTCARE        0x8
-
-/* Note if the caller is only interested in ownership test i.e. one of the below
-+ * in combination with GF_ACCTEST_DONTCARE, then only one type of user's owner
-+ * ship can be tested with one call to gf_test_access, i.e. we can only
-+ * check of either owner and group, if both need to be tested for a specific
-+ * (uid, gid) pair then two calls will be needed.
-+ */
-#define ACCTEST_OWNER           0x1
-#define ACCTEST_GROUP           0x2
-#define ACCTEST_OTHER           0x4
-
-/* Signifies any user, as long as we get access. */
-#define ACCTEST_ANY             (ACCTEST_OWNER | ACCTEST_GROUP | ACCTEST_OTHER)
-
-#define ac_test_owner(acc)      ((acc) & ACCTEST_OWNER)
-#define ac_test_group(acc)      ((acc) & ACCTEST_GROUP)
-#define ac_test_other(acc)      ((acc) & ACCTEST_OTHER)
-#define ac_test_dontcare(acc)   ((acc) & ACCTEST_DONTCARE)
-#define ac_test_read(acc)       ((acc) & ACCTEST_READ)
-#define ac_test_write(acc)      ((acc) & ACCTEST_WRITE)
-#define ac_test_exec(acc)       ((acc) & ACCTEST_EXEC)
-#endif
diff --git a/xlators/features/access-control/Makefile.am b/xlators/features/barrier/Makefile.am
index a985f42a8..a985f42a8 100644
--- a/xlators/features/access-control/Makefile.am
+++ b/xlators/features/barrier/Makefile.am
diff --git a/xlators/features/barrier/src/Makefile.am b/xlators/features/barrier/src/Makefile.am
new file mode 100644
index 000000000..8859be328
--- /dev/null
+++ b/xlators/features/barrier/src/Makefile.am
@@ -0,0 +1,16 @@
+xlator_LTLIBRARIES = barrier.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+barrier_la_LDFLAGS = -module -avoid-version
+
+barrier_la_SOURCES = barrier.c
+
+barrier_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = barrier.h barrier-mem-types.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/barrier/src/barrier-mem-types.h b/xlators/features/barrier/src/barrier-mem-types.h
new file mode 100644
index 000000000..36647a669
--- /dev/null
+++ b/xlators/features/barrier/src/barrier-mem-types.h
@@ -0,0 +1,20 @@
+/*
+   Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __BARRIER_MEM_TYPES_H__
+#define __BARRIER_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_barrier_mem_types_ {
+        gf_barrier_mt_priv_t = gf_common_mt_end + 1,
+        gf_barrier_mt_end
+};
+#endif
diff --git a/xlators/features/barrier/src/barrier.c b/xlators/features/barrier/src/barrier.c
new file mode 100644
index 000000000..5edb9cdd3
--- /dev/null
+++ b/xlators/features/barrier/src/barrier.c
@@ -0,0 +1,658 @@
+/*
+   Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "barrier.h"
+#include "defaults.h"
+#include "call-stub.h"
+
+#include "statedump.h"
+
+int32_t
+barrier_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf,
+                    dict_t *xdata)
+{
+        BARRIER_FOP_CBK (writev, out, frame, this, op_ret, op_errno,
+                         prebuf, postbuf, xdata);
+out:
+        return 0;
+}
+
+int32_t
+barrier_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                          int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        BARRIER_FOP_CBK (fremovexattr, out, frame, this, op_ret, op_errno,
+                         xdata);
+out:
+        return 0;
+}
+
+int32_t
+barrier_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        BARRIER_FOP_CBK (removexattr, out, frame, this, op_ret, op_errno,
+                         xdata);
+out:
+        return 0;
+}
+
+int32_t
+barrier_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                      struct iatt *postbuf, dict_t *xdata)
+{
+        BARRIER_FOP_CBK (truncate, out, frame, this, op_ret, op_errno, prebuf,
+                         postbuf, xdata);
+out:
+        return 0;
+}
+
+int32_t
+barrier_ftruncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                       struct iatt *postbuf, dict_t *xdata)
+{
+        BARRIER_FOP_CBK (ftruncate, out, frame, this, op_ret, op_errno, prebuf,
+                         postbuf, xdata);
+out:
+        return 0;
+}
+
+int32_t
+barrier_rename_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                    struct iatt *preoldparent, struct iatt *postoldparent,
+                    struct iatt *prenewparent, struct iatt *postnewparent,
+                    dict_t *xdata)
+{
+        BARRIER_FOP_CBK (rename, out, frame, this, op_ret, op_errno, buf,
+                         preoldparent, postoldparent, prenewparent,
+                         postnewparent, xdata);
+out:
+        return 0;
+}
+
+int32_t
+barrier_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+        BARRIER_FOP_CBK (rmdir, out, frame, this, op_ret, op_errno, preparent,
+                         postparent, xdata);
+out:
+        return 0;
+}
+
+int32_t
+barrier_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+        BARRIER_FOP_CBK (unlink, out, frame, this, op_ret, op_errno, preparent,
+                         postparent, xdata);
+out:
+        return 0;
+}
+
+int32_t
+barrier_fsync_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                   struct iatt *postbuf, dict_t *xdata)
+{
+        BARRIER_FOP_CBK (fsync, out, frame, this, op_ret, op_errno,
+                         prebuf, postbuf, xdata);
+out:
+        return 0;
+}
+
+int32_t
+barrier_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                struct iovec *vector, int32_t count, off_t off, uint32_t flags,
+                struct iobref *iobref, dict_t *xdata)
+{
+        if (!(flags & O_SYNC)) {
+                STACK_WIND_TAIL (frame, FIRST_CHILD(this),
+                                 FIRST_CHILD(this)->fops->writev,
+                                 fd, vector, count, off, flags, iobref, xdata);
+
+                return 0;
+        }
+
+        STACK_WIND (frame, barrier_writev_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->writev, fd, vector, count,
+                    off, flags, iobref, xdata);
+        return 0;
+}
+
+int32_t
+barrier_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                      const char *name, dict_t *xdata)
+{
+        STACK_WIND (frame, barrier_fremovexattr_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->fremovexattr,
+                    fd, name, xdata);
+        return 0;
+}
+
+int32_t
+barrier_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     const char *name, dict_t *xdata)
+{
+        STACK_WIND (frame, barrier_removexattr_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->removexattr,
+                    loc, name, xdata);
+        return 0;
+}
+
+int32_t
+barrier_truncate (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  off_t offset, dict_t *xdata)
+{
+        STACK_WIND (frame, barrier_truncate_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->truncate,
+                    loc, offset, xdata);
+        return 0;
+}
+
+int32_t
+barrier_rename (call_frame_t *frame, xlator_t *this, loc_t *oldloc,
+                loc_t *newloc, dict_t *xdata)
+{
+        STACK_WIND (frame, barrier_rename_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->rename,
+                    oldloc, newloc, xdata);
+        return 0;
+}
+
+int
+barrier_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+               dict_t *xdata)
+{
+        STACK_WIND (frame, barrier_rmdir_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->rmdir,
+                    loc, flags, xdata);
+        return 0;
+}
+
+int32_t
+barrier_unlink (call_frame_t *frame, xlator_t *this,
+                loc_t *loc, int xflag, dict_t *xdata)
+{
+        STACK_WIND (frame, barrier_unlink_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->unlink,
+                    loc, xflag, xdata);
+        return 0;
+}
+
+int32_t
+barrier_ftruncate (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   off_t offset, dict_t *xdata)
+{
+        STACK_WIND (frame, barrier_ftruncate_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->ftruncate,
+                    fd, offset, xdata);
+        return 0;
+}
+
+int32_t
+barrier_fsync (call_frame_t *frame, xlator_t *this, fd_t *fd,
+               int32_t flags, dict_t *xdata)
+{
+        STACK_WIND (frame, barrier_fsync_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->fsync,
+                    fd, flags, xdata);
+        return 0;
+}
+
+call_stub_t *
+__barrier_dequeue (xlator_t *this, struct list_head *queue)
+{
+        call_stub_t    *stub            = NULL;
+        barrier_priv_t *priv            = NULL;
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (list_empty (queue))
+                goto out;
+
+        stub = list_entry (queue->next, call_stub_t, list);
+        list_del_init (&stub->list);
+
+out:
+        return stub;
+}
+
+void
+barrier_dequeue_all (xlator_t *this, struct list_head *queue)
+{
+        call_stub_t            *stub    = NULL;
+
+        gf_log (this->name, GF_LOG_INFO, "Dequeuing all the barriered fops");
+
+        /* TODO: Start the below task in a new thread */
+        while ((stub = __barrier_dequeue (this, queue)))
+                call_resume (stub);
+
+        gf_log (this->name, GF_LOG_INFO, "Dequeuing the barriered fops is "
+                                         "finished");
+        return;
+}
+
+void
+barrier_timeout (void *data)
+{
+        xlator_t               *this    = NULL;
+        barrier_priv_t         *priv    = NULL;
+        struct list_head        queue   = {0,};
+
+        this = data;
+        THIS = this;
+        priv = this->private;
+
+        INIT_LIST_HEAD (&queue);
+
+        gf_log (this->name, GF_LOG_CRITICAL, "Disabling barrier because of "
+                                             "the barrier timeout.");
+
+        LOCK (&priv->lock);
+        {
+                __barrier_disable (this, &queue);
+        }
+        UNLOCK (&priv->lock);
+
+        barrier_dequeue_all (this, &queue);
+
+        return;
+}
+
+void
+__barrier_enqueue (xlator_t *this, call_stub_t *stub)
+{
+        barrier_priv_t *priv    = NULL;
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        list_add_tail (&stub->list, &priv->queue);
+        priv->queue_size++;
+
+        return;
+}
+
+void
+__barrier_disable (xlator_t *this, struct list_head *queue)
+{
+        GF_UNUSED int   ret     = 0;
+        barrier_priv_t *priv    = NULL;
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        if (priv->timer) {
+                ret = gf_timer_call_cancel (this->ctx, priv->timer);
+                priv->timer = NULL;
+        }
+
+        list_splice_init (&priv->queue, queue);
+        priv->queue_size = 0;
+        priv->barrier_enabled = _gf_false;
+}
+
+int
+__barrier_enable (xlator_t *this, barrier_priv_t *priv)
+{
+        int             ret     = -1;
+
+        priv->timer = gf_timer_call_after (this->ctx, priv->timeout,
+                                           barrier_timeout, (void *) this);
+        if (!priv->timer) {
+                gf_log (this->name, GF_LOG_CRITICAL, "Couldn't add barrier "
+                                                     "timeout event.");
+                goto out;
+        }
+
+        priv->barrier_enabled = _gf_true;
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+notify (xlator_t *this, int event, void *data, ...)
+{
+        barrier_priv_t  *priv                   = NULL;
+        dict_t          *dict                   = NULL;
+        gf_boolean_t     past                   = _gf_false;
+        int              ret                    = -1;
+        gf_boolean_t     barrier_enabled        = _gf_false;
+        struct list_head queue                  = {0,};
+
+        priv = this->private;
+        GF_ASSERT (priv);
+        INIT_LIST_HEAD (&queue);
+
+        switch (event) {
+        case GF_EVENT_TRANSLATOR_OP:
+        {
+                dict = data;
+                GF_OPTION_RECONF ("barrier", barrier_enabled, dict,
+                                  bool, out);
+
+                LOCK (&priv->lock);
+                {
+                        past = priv->barrier_enabled;
+
+                        switch (past) {
+                        case _gf_false:
+                                if (barrier_enabled) {
+                                        ret = __barrier_enable (this,priv);
+                                        if (ret)
+                                                goto unlock;
+                                } else {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Already disabled.");
+                                        goto unlock;
+                                }
+                                break;
+
+                        case _gf_true:
+                                if (!barrier_enabled) {
+                                        __barrier_disable(this, &queue);
+                                } else {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "Already enabled");
+                                        goto unlock;
+                                }
+                                break;
+                        }
+                        ret = 0;
+                }
+unlock:
+                UNLOCK (&priv->lock);
+
+                if (!list_empty (&queue))
+                        barrier_dequeue_all (this, &queue);
+                // missing break is intentional
+        }
+        default:
+        {
+                default_notify (this, event, data);
+                ret = 0;
+                goto out;
+        }
+        }
+out:
+        return ret;
+}
+
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        barrier_priv_t  *priv                   = NULL;
+        gf_boolean_t     past                   = _gf_false;
+        int              ret                    = -1;
+        gf_boolean_t     barrier_enabled        = _gf_false;
+        uint32_t         timeout                = {0,};
+        struct list_head queue                  = {0,};
+
+        priv = this->private;
+        GF_ASSERT (priv);
+
+        GF_OPTION_RECONF ("barrier", barrier_enabled, options, bool, out);
+        GF_OPTION_RECONF ("barrier-timeout", timeout, options, time, out);
+
+        INIT_LIST_HEAD (&queue);
+
+        LOCK (&priv->lock);
+        {
+                past = priv->barrier_enabled;
+
+                switch (past) {
+                case _gf_false:
+                        if (barrier_enabled) {
+                                ret = __barrier_enable (this, priv);
+                                if (ret) {
+                                        goto unlock;
+                                }
+                        }
+                        break;
+
+                case _gf_true:
+                        if (!barrier_enabled) {
+                                __barrier_disable (this, &queue);
+
+                        }
+                        break;
+                }
+                priv->timeout.tv_sec = timeout;
+                ret = 0;
+        }
+unlock:
+        UNLOCK (&priv->lock);
+
+        if (!list_empty (&queue))
+                barrier_dequeue_all (this, &queue);
+
+out:
+        return ret;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        ret = xlator_mem_acct_init (this, gf_barrier_mt_end + 1);
+        if (ret)
+                gf_log (this->name, GF_LOG_ERROR, "Memory accounting "
+                        "initialization failed.");
+
+        return ret;
+}
+
+int
+init (xlator_t *this)
+{
+        int                     ret     = -1;
+        barrier_priv_t         *priv    = NULL;
+        uint32_t                timeout = {0,};
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "'barrier' not configured with exactly one child");
+                goto out;
+        }
+
+        if (!this->parents)
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dangling volume. check volfile ");
+
+        priv = GF_CALLOC (1, sizeof (*priv), gf_barrier_mt_priv_t);
+        if (!priv)
+                goto out;
+
+        LOCK_INIT (&priv->lock);
+
+        GF_OPTION_INIT ("barrier", priv->barrier_enabled, bool, out);
+        GF_OPTION_INIT ("barrier-timeout", timeout, time, out);
+        priv->timeout.tv_sec = timeout;
+
+        INIT_LIST_HEAD (&priv->queue);
+
+        if (priv->barrier_enabled) {
+                ret = __barrier_enable (this, priv);
+                if (ret == -1)
+                        goto out;
+        }
+
+        this->private = priv;
+        ret = 0;
+out:
+        return ret;
+}
+
+void
+fini (xlator_t *this)
+{
+        barrier_priv_t         *priv    = NULL;
+        struct list_head        queue   = {0,};
+
+        priv = this->private;
+        if (!priv)
+                goto out;
+
+        INIT_LIST_HEAD (&queue);
+
+        gf_log (this->name, GF_LOG_INFO, "Disabling barriering and dequeuing "
+                                         "all the queued fops");
+        LOCK (&priv->lock);
+        {
+                __barrier_disable (this, &queue);
+        }
+        UNLOCK (&priv->lock);
+
+        if (!list_empty (&queue))
+                barrier_dequeue_all (this, &queue);
+
+        this->private = NULL;
+
+        LOCK_DESTROY (&priv->lock);
+        GF_FREE (priv);
+out:
+        return;
+}
+
+static void
+barrier_dump_stub (call_stub_t *stub, char *prefix)
+{
+        char key[GF_DUMP_MAX_BUF_LEN] = {0,};
+
+        gf_proc_dump_build_key (key, prefix, "fop");
+        gf_proc_dump_write (key, "%s", gf_fop_list[stub->fop]);
+
+        gf_proc_dump_build_key (key, prefix, "gfid");
+        gf_proc_dump_write (key, "%s", uuid_utoa (stub->args.loc.gfid));
+
+        if (stub->args.loc.path) {
+                gf_proc_dump_build_key (key, prefix, "path");
+                gf_proc_dump_write (key, "%s", stub->args.loc.path);
+        }
+        if (stub->args.loc.name) {
+                gf_proc_dump_build_key (key, prefix, "name");
+                gf_proc_dump_write (key, "%s", stub->args.loc.name);
+        }
+
+        return;
+}
+
+static void
+__barrier_dump_queue (barrier_priv_t *priv)
+{
+        call_stub_t *stub = NULL;
+        char key[GF_DUMP_MAX_BUF_LEN] = {0,};
+        int i = 0;
+
+        GF_VALIDATE_OR_GOTO ("barrier", priv, out);
+
+        list_for_each_entry (stub, &priv->queue, list) {
+                snprintf (key, sizeof (key), "stub.%d", i++);
+                gf_proc_dump_add_section (key);
+                barrier_dump_stub(stub, key);
+        }
+
+out:
+        return;
+}
+
+int
+barrier_dump_priv (xlator_t *this)
+{
+        int ret = -1;
+        char key[GF_DUMP_MAX_BUF_LEN] = {0,};
+        barrier_priv_t *priv = NULL;
+
+        GF_VALIDATE_OR_GOTO ("barrier", this, out);
+
+        priv = this->private;
+        if (!priv)
+                return 0;
+
+        gf_proc_dump_build_key (key, "xlator.features.barrier", "priv");
+        gf_proc_dump_add_section (key);
+
+        LOCK (&priv->lock);
+        {
+                gf_proc_dump_build_key (key, "barrier", "enabled");
+                gf_proc_dump_write (key, "%d", priv->barrier_enabled);
+                gf_proc_dump_build_key (key, "barrier", "timeout");
+                gf_proc_dump_write (key, "%"PRId64, priv->timeout.tv_sec);
+                if (priv->barrier_enabled) {
+                        gf_proc_dump_build_key (key, "barrier", "queue_size");
+                        gf_proc_dump_write (key, "%d", priv->queue_size);
+                        __barrier_dump_queue (priv);
+                }
+        }
+        UNLOCK (&priv->lock);
+
+out:
+        return ret;
+}
+
+struct xlator_fops fops = {
+
+        /* Barrier Class fops */
+        .rmdir          = barrier_rmdir,
+        .unlink         = barrier_unlink,
+        .rename         = barrier_rename,
+        .removexattr    = barrier_removexattr,
+        .fremovexattr   = barrier_fremovexattr,
+        .truncate       = barrier_truncate,
+        .ftruncate      = barrier_ftruncate,
+        .fsync          = barrier_fsync,
+
+        /* Writes with only O_SYNC flag */
+        .writev         = barrier_writev,
+};
+
+struct xlator_dumpops dumpops = {
+        .priv = barrier_dump_priv,
+};
+
+struct xlator_cbks cbks;
+
+struct volume_options options[] = {
+        { .key  = {"barrier"},
+          .type = GF_OPTION_TYPE_BOOL,
+          .default_value = "off",
+          .description = "When \"on\", blocks acknowledgements to application "
+                         "for file operations such as rmdir, rename, unlink, "
+                         "removexattr, fremovexattr, truncate, ftruncate, "
+                         "write (with O_SYNC), fsync. It is turned \"off\" by "
+                         "default."
+        },
+        { .key = {"barrier-timeout"},
+          .type = GF_OPTION_TYPE_TIME,
+          .default_value = "120",
+          .description = "After 'timeout' seconds since the time 'barrier' "
+                         "option was set to \"on\", acknowledgements to file "
+                         "operations are no longer blocked and previously "
+                         "blocked acknowledgements are sent to the application"
+        },
+        { .key  = {NULL} },
+};
diff --git a/xlators/features/barrier/src/barrier.h b/xlators/features/barrier/src/barrier.h
new file mode 100644
index 000000000..8face9f65
--- /dev/null
+++ b/xlators/features/barrier/src/barrier.h
@@ -0,0 +1,91 @@
+/*
+   Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __BARRIER_H__
+#define __BARRIER_H__
+
+#include "barrier-mem-types.h"
+#include "xlator.h"
+#include "timer.h"
+#include "call-stub.h"
+
+#define BARRIER_SAFE_ASSIGN(lock, to, value)                    \
+        do {                                                    \
+                LOCK (&(lock));                                 \
+                {                                               \
+                        to = value;                             \
+                }                                               \
+                UNLOCK (&(lock));                               \
+        } while (0)
+
+#define BARRIER_FOP_CBK(fop_name, label, frame, this, params ...)       \
+        do {                                                            \
+                barrier_priv_t         *_priv           = NULL;         \
+                call_stub_t            *_stub           = NULL;         \
+                gf_boolean_t            _barrier_enabled= _gf_false;    \
+                struct list_head        queue           = {0, };        \
+                                                                        \
+                INIT_LIST_HEAD (&queue);                                \
+                                                                        \
+                _priv = this->private;                                  \
+                GF_ASSERT (_priv);                                      \
+                                                                        \
+                LOCK (&_priv->lock);                                    \
+                {                                                       \
+                        if (_priv->barrier_enabled) {                   \
+                                _barrier_enabled = _priv->barrier_enabled;\
+                                                                        \
+                                _stub = fop_##fop_name##_cbk_stub       \
+                                        (frame,                         \
+                                         default_##fop_name##_cbk_resume,\
+                                         params);                       \
+                                if (!_stub) {                           \
+                                        __barrier_disable (this, &queue);\
+                                        goto unlock;                    \
+                                }                                       \
+                                                                        \
+                                __barrier_enqueue (this, _stub);        \
+                        }                                               \
+                }                                                       \
+unlock:                                                                 \
+                UNLOCK (&_priv->lock);                                  \
+                                                                        \
+                if (_stub)                                              \
+                        goto label;                                     \
+                                                                        \
+                if (_barrier_enabled && !_stub) {                       \
+                        gf_log (this->name, GF_LOG_CRITICAL,            \
+                                "Failed to barrier FOPs, disabling "    \
+                                "barrier. FOP: %s, ERROR: %s",          \
+                                #fop_name, strerror (ENOMEM));          \
+                        barrier_dequeue_all (this, &queue);             \
+                }                                                       \
+                                                                        \
+                STACK_UNWIND_STRICT (fop_name, frame, params);          \
+                goto label;                                             \
+        } while (0)
+
+typedef struct {
+        gf_timer_t       *timer;
+        gf_boolean_t      barrier_enabled;
+        gf_lock_t         lock;
+        struct list_head  queue;
+        struct timespec   timeout;
+        uint32_t          queue_size;
+} barrier_priv_t;
+
+int __barrier_enable (xlator_t *this, barrier_priv_t *priv);
+void __barrier_enqueue (xlator_t *this, call_stub_t *stub);
+void __barrier_disable (xlator_t *this, struct list_head *queue);
+void barrier_timeout (void *data);
+void barrier_dequeue_all (xlator_t *this, struct list_head *queue);
+call_stub_t *__barrier_dequeue (xlator_t *this, struct list_head *queue);
+
+#endif
diff --git a/xlators/features/changelog/Makefile.am b/xlators/features/changelog/Makefile.am
new file mode 100644
index 000000000..153bb6850
--- /dev/null
+++ b/xlators/features/changelog/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src lib
+
+CLEANFILES =
diff --git a/xlators/features/changelog/lib/Makefile.am b/xlators/features/changelog/lib/Makefile.am
new file mode 100644
index 000000000..a985f42a8
--- /dev/null
+++ b/xlators/features/changelog/lib/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/features/changelog/lib/examples/c/get-changes.c b/xlators/features/changelog/lib/examples/c/get-changes.c
new file mode 100644
index 000000000..14562585a
--- /dev/null
+++ b/xlators/features/changelog/lib/examples/c/get-changes.c
@@ -0,0 +1,87 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+/**
+ * get set of new changes every 10 seconds (just print the file names)
+ *
+ * Compile it using:
+ *  gcc -o getchanges `pkg-config --cflags libgfchangelog` get-changes.c \
+ *  `pkg-config --libs libgfchangelog`
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/un.h>
+#include <limits.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <errno.h>
+
+#include "changelog.h"
+
+#define handle_error(fn)                                \
+        printf ("%s (reason: %s)\n", fn, strerror (errno))
+
+int
+main (int argc, char ** argv)
+{
+        int     i           = 0;
+        int     ret         = 0;
+        ssize_t nr_changes  = 0;
+        ssize_t changes     = 0;
+        char fbuf[PATH_MAX] = {0,};
+
+        /* get changes for brick "/home/vshankar/export/yow/yow-1" */
+        ret = gf_changelog_register ("/home/vshankar/export/yow/yow-1",
+                                     "/tmp/scratch", "/tmp/change.log", 9, 5);
+        if (ret) {
+                handle_error ("register failed");
+                goto out;
+        }
+
+        while (1) {
+                i = 0;
+                nr_changes = gf_changelog_scan ();
+                if (nr_changes < 0) {
+                        handle_error ("scan(): ");
+                        break;
+                }
+
+                if (nr_changes == 0)
+                        goto next;
+
+                printf ("Got %ld changelog files\n", nr_changes);
+
+                while ( (changes =
+                         gf_changelog_next_change (fbuf, PATH_MAX)) > 0) {
+                        printf ("changelog file [%d]: %s\n", ++i, fbuf);
+
+                        /* process changelog */
+                        /* ... */
+                        /* ... */
+                        /* ... */
+                        /* done processing */
+
+                        ret = gf_changelog_done (fbuf);
+                        if (ret)
+                                handle_error ("gf_changelog_done");
+                }
+
+                if (changes == -1)
+                        handle_error ("gf_changelog_next_change");
+
+        next:
+                sleep (10);
+        }
+
+ out:
+        return ret;
+}
diff --git a/xlators/features/changelog/lib/examples/python/changes.py b/xlators/features/changelog/lib/examples/python/changes.py
new file mode 100644
index 000000000..d21db8eab
--- /dev/null
+++ b/xlators/features/changelog/lib/examples/python/changes.py
@@ -0,0 +1,32 @@
+#!/usr/bin/python
+
+import os
+import sys
+import time
+import libgfchangelog
+
+cl = libgfchangelog.Changes()
+
+def get_changes(brick, scratch_dir, log_file, log_level, interval):
+    change_list = []
+    try:
+        cl.cl_register(brick, scratch_dir, log_file, log_level)
+        while True:
+            cl.cl_scan()
+            change_list = cl.cl_getchanges()
+            if change_list:
+                print change_list
+            for change in change_list:
+                print('done with %s' % (change))
+                cl.cl_done(change)
+            time.sleep(interval)
+    except OSError:
+        ex = sys.exc_info()[1]
+        print ex
+
+if __name__ == '__main__':
+    if len(sys.argv) != 5:
+        print("usage: %s <brick> <scratch-dir> <log-file> <fetch-interval>"
+              % (sys.argv[0]))
+        sys.exit(1)
+    get_changes(sys.argv[1], sys.argv[2], sys.argv[3], 9, int(sys.argv[4]))
diff --git a/xlators/features/changelog/lib/examples/python/libgfchangelog.py b/xlators/features/changelog/lib/examples/python/libgfchangelog.py
new file mode 100644
index 000000000..68ec3baf1
--- /dev/null
+++ b/xlators/features/changelog/lib/examples/python/libgfchangelog.py
@@ -0,0 +1,64 @@
+import os
+from ctypes import *
+from ctypes.util import find_library
+
+class Changes(object):
+    libgfc = CDLL(find_library("gfchangelog"), use_errno=True)
+
+    @classmethod
+    def geterrno(cls):
+        return get_errno()
+
+    @classmethod
+    def raise_oserr(cls):
+        errn = cls.geterrno()
+        raise OSError(errn, os.strerror(errn))
+
+    @classmethod
+    def _get_api(cls, call):
+        return getattr(cls.libgfc, call)
+
+    @classmethod
+    def cl_register(cls, brick, path, log_file, log_level, retries = 0):
+        ret = cls._get_api('gf_changelog_register')(brick, path,
+                                                    log_file, log_level, retries)
+        if ret == -1:
+            cls.raise_oserr()
+
+    @classmethod
+    def cl_scan(cls):
+        ret = cls._get_api('gf_changelog_scan')()
+        if ret == -1:
+            cls.raise_oserr()
+
+    @classmethod
+    def cl_startfresh(cls):
+        ret = cls._get_api('gf_changelog_start_fresh')()
+        if ret == -1:
+            cls.raise_oserr()
+
+    @classmethod
+    def cl_getchanges(cls):
+        """ remove hardcoding for path name length """
+        def clsort(f):
+            return f.split('.')[-1]
+        changes = []
+        buf = create_string_buffer('\0', 4096)
+        call = cls._get_api('gf_changelog_next_change')
+
+        while True:
+            ret = call(buf, 4096)
+            if ret in (0, -1):
+                break;
+            changes.append(buf.raw[:ret-1])
+        if ret == -1:
+            cls.raise_oserr()
+        # cleanup tracker
+        cls.cl_startfresh()
+        return sorted(changes, key=clsort)
+
+    @classmethod
+    def cl_done(cls, clfile):
+        ret = cls._get_api('gf_changelog_done')(clfile)
+        if ret == -1:
+            cls.raise_oserr()
diff --git a/xlators/features/changelog/lib/src/Makefile.am b/xlators/features/changelog/lib/src/Makefile.am
new file mode 100644
index 000000000..28d5a70aa
--- /dev/null
+++ b/xlators/features/changelog/lib/src/Makefile.am
@@ -0,0 +1,38 @@
+libgfchangelog_la_CFLAGS = -Wall $(GF_CFLAGS) $(GF_DARWIN_LIBGLUSTERFS_CFLAGS) \
+			-DDATADIR=\"$(localstatedir)\"
+
+libgfchangelog_la_CPPFLAGS = $(GF_CPPFLAGS) -D__USE_FILE_OFFSET64 -fpic \
+				-I../../../src/ -I$(top_srcdir)/libglusterfs/src \
+				-I$(top_srcdir)/xlators/features/changelog/src \
+				-DDATADIR=\"$(localstatedir)\"
+
+libgfchangelog_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+			$(GF_GLUSTERFS_LIBS)
+
+libgfchangelog_la_LDFLAGS = $(GF_LDFLAGS) -version-info $(LIBGFCHANGELOG_LT_VERSION)
+
+libgfchangelogdir = $(includedir)/glusterfs/gfchangelog
+lib_LTLIBRARIES = libgfchangelog.la
+
+CONTRIB_BUILDDIR = $(top_builddir)/contrib
+
+libgfchangelog_la_SOURCES = gf-changelog.c gf-changelog-process.c \
+		gf-changelog-helpers.c gf-history-changelog.c \
+		$(CONTRIBDIR)/uuid/clear.c \
+		$(CONTRIBDIR)/uuid/copy.c $(CONTRIBDIR)/uuid/gen_uuid.c \
+		$(CONTRIBDIR)/uuid/pack.c $(CONTRIBDIR)/uuid/parse.c \
+		$(CONTRIBDIR)/uuid/unparse.c $(CONTRIBDIR)/uuid/uuid_time.c \
+		$(CONTRIBDIR)/uuid/compare.c $(CONTRIBDIR)/uuid/isnull.c \
+		$(CONTRIBDIR)/uuid/unpack.c
+
+noinst_HEADERS = gf-changelog-helpers.h $(CONTRIBDIR)/uuid/uuidd.h \
+		$(CONTRIBDIR)/uuid/uuid.h $(CONTRIBDIR)/uuid/uuidP.h \
+		$(CONTRIB_BUILDDIR)/uuid/uuid_types.h
+
+libgfchangelog_HEADERS = changelog.h
+
+CLEANFILES =
+CONFIG_CLEAN_FILES = $(CONTRIB_BUILDDIR)/uuid/uuid_types.h
+
+$(top_builddir)/libglusterfs/src/libglusterfs.la:
+	$(MAKE) -C $(top_builddir)/libglusterfs/src/ all
diff --git a/xlators/features/changelog/lib/src/changelog.h b/xlators/features/changelog/lib/src/changelog.h
new file mode 100644
index 000000000..5cddfb583
--- /dev/null
+++ b/xlators/features/changelog/lib/src/changelog.h
@@ -0,0 +1,31 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GF_CHANGELOG_H
+#define _GF_CHANGELOG_H
+
+/* API set */
+
+int
+gf_changelog_register (char *brick_path, char *scratch_dir,
+                       char *log_file, int log_levl, int max_reconnects);
+ssize_t
+gf_changelog_scan ();
+
+int
+gf_changelog_start_fresh ();
+
+ssize_t
+gf_changelog_next_change (char *bufptr, size_t maxlen);
+
+int
+gf_changelog_done (char *file);
+
+#endif
diff --git a/xlators/features/changelog/lib/src/gf-changelog-helpers.c b/xlators/features/changelog/lib/src/gf-changelog-helpers.c
new file mode 100644
index 000000000..1eef8bf04
--- /dev/null
+++ b/xlators/features/changelog/lib/src/gf-changelog-helpers.c
@@ -0,0 +1,180 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "changelog-mem-types.h"
+#include "gf-changelog-helpers.h"
+
+ssize_t gf_changelog_read_path (int fd, char *buffer, size_t bufsize)
+{
+        return read (fd, buffer, bufsize);
+}
+
+size_t
+gf_changelog_write (int fd, char *buffer, size_t len)
+{
+        ssize_t size = 0;
+        size_t writen = 0;
+
+        while (writen < len) {
+                size = write (fd,
+                              buffer + writen, len - writen);
+                if (size <= 0)
+                        break;
+
+                writen += size;
+        }
+
+        return writen;
+}
+
+void
+gf_rfc3986_encode (unsigned char *s, char *enc, char *estr)
+{
+        for (; *s; s++) {
+                if (estr[*s])
+                        sprintf(enc, "%c", estr[*s]);
+                else
+                        sprintf(enc, "%%%02X", *s);
+                while (*++enc);
+        }
+}
+
+/**
+ * thread safe version of readline with buffering
+ * (taken from Unix Network Programming Volume I, W.R. Stevens)
+ *
+ * This is favoured over fgets() as we'd need to ftruncate()
+ * (see gf_changelog_scan() API) to record new changelog files.
+ * stream open functions does have a truncate like api (although
+ * that can be done via @fflush(fp), @ftruncate(fd) and @fseek(fp),
+ * but this involves mixing POSIX file descriptors and stream FILE *).
+ *
+ * NOTE: This implmentation still does work with more than one fd's
+ *       used to perform gf_readline(). For this very reason it's not
+ *       made a part of libglusterfs.
+ */
+
+static pthread_key_t rl_key;
+static pthread_once_t rl_once = PTHREAD_ONCE_INIT;
+
+static void
+readline_destructor (void *ptr)
+{
+        GF_FREE (ptr);
+}
+
+static void
+readline_once (void)
+{
+        pthread_key_create (&rl_key, readline_destructor);
+}
+
+static ssize_t
+my_read (read_line_t *tsd, int fd, char *ptr)
+{
+        if (tsd->rl_cnt <= 0) {
+                if ( (tsd->rl_cnt = read (fd, tsd->rl_buf, MAXLINE)) < 0 )
+                        return -1;
+                else if (tsd->rl_cnt == 0)
+                        return 0;
+                tsd->rl_bufptr = tsd->rl_buf;
+        }
+
+        tsd->rl_cnt--;
+        *ptr = *tsd->rl_bufptr++;
+        return 1;
+}
+
+static int
+gf_readline_init_once (read_line_t **tsd)
+{
+        if (pthread_once (&rl_once, readline_once) != 0)
+                return -1;
+
+        *tsd = pthread_getspecific (rl_key);
+        if (*tsd)
+                goto out;
+
+        *tsd = GF_CALLOC (1, sizeof (**tsd),
+                          gf_changelog_mt_libgfchangelog_rl_t);
+        if (!*tsd)
+                return -1;
+
+        if (pthread_setspecific (rl_key, *tsd) != 0)
+                return -1;
+
+ out:
+        return 0;
+}
+
+ssize_t
+gf_readline (int fd, void *vptr, size_t maxlen)
+{
+        size_t       n   = 0;
+        size_t       rc  = 0;
+        char         c   = ' ';
+        char        *ptr = NULL;
+        read_line_t *tsd = NULL;
+
+        if (gf_readline_init_once (&tsd))
+                return -1;
+
+        ptr = vptr;
+        for (n = 1; n < maxlen; n++) {
+                if ( (rc = my_read (tsd, fd, &c)) == 1 ) {
+                        *ptr++ = c;
+                        if (c == '\n')
+                                break;
+                } else if (rc == 0) {
+                        *ptr = '\0';
+                        return (n - 1);
+                } else
+                        return -1;
+        }
+
+        *ptr = '\0';
+        return n;
+
+}
+
+off_t
+gf_lseek (int fd, off_t offset, int whence)
+{
+        off_t        off = 0;
+        read_line_t *tsd = NULL;
+
+        if (gf_readline_init_once (&tsd))
+                return -1;
+
+        if ( (off = lseek (fd, offset, whence)) == -1)
+                return -1;
+
+        tsd->rl_cnt = 0;
+        tsd->rl_bufptr = tsd->rl_buf;
+
+        return off;
+}
+
+int
+gf_ftruncate (int fd, off_t length)
+{
+        read_line_t *tsd = NULL;
+
+        if (gf_readline_init_once (&tsd))
+                return -1;
+
+        if (ftruncate (fd, 0))
+                return -1;
+
+        tsd->rl_cnt = 0;
+        tsd->rl_bufptr = tsd->rl_buf;
+
+        return 0;
+}
diff --git a/xlators/features/changelog/lib/src/gf-changelog-helpers.h b/xlators/features/changelog/lib/src/gf-changelog-helpers.h
new file mode 100644
index 000000000..fa0edabf0
--- /dev/null
+++ b/xlators/features/changelog/lib/src/gf-changelog-helpers.h
@@ -0,0 +1,102 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GF_CHANGELOG_HELPERS_H
+#define _GF_CHANGELOG_HELPERS_H
+
+#include <unistd.h>
+#include <dirent.h>
+#include <limits.h>
+#include <pthread.h>
+
+#include <xlator.h>
+
+#define GF_CHANGELOG_TRACKER  "tracker"
+
+#define GF_CHANGELOG_CURRENT_DIR    ".current"
+#define GF_CHANGELOG_PROCESSED_DIR  ".processed"
+#define GF_CHANGELOG_PROCESSING_DIR ".processing"
+#define GF_CHANGELOG_HISTORY_DIR    ".history"
+
+#ifndef MAXLINE
+#define MAXLINE 4096
+#endif
+
+#define GF_CHANGELOG_FILL_BUFFER(ptr, ascii, off, len) do {     \
+                memcpy (ascii + off, ptr, len);                 \
+                off += len;                                     \
+        } while (0)
+
+typedef struct read_line {
+        int rl_cnt;
+        char *rl_bufptr;
+        char rl_buf[MAXLINE];
+} read_line_t;
+
+typedef struct gf_changelog {
+        xlator_t *this;
+
+        /* 'processing' directory stream */
+        DIR *gfc_dir;
+
+        /* fd to the tracker file */
+        int gfc_fd;
+
+        /* connection retries */
+        int gfc_connretries;
+
+        char gfc_sockpath[UNIX_PATH_MAX];
+
+        char gfc_brickpath[PATH_MAX];
+
+        /* socket for recieving notifications */
+        int gfc_sockfd;
+
+        char *gfc_working_dir;
+
+        /* RFC 3986 string encoding */
+        char rfc3986[256];
+
+        char gfc_current_dir[PATH_MAX];
+        char gfc_processed_dir[PATH_MAX];
+        char gfc_processing_dir[PATH_MAX];
+
+        pthread_t gfc_changelog_processor;
+
+        /* Holds gfc for History API */
+        struct gf_changelog *hist_gfc;
+} gf_changelog_t;
+
+int
+gf_changelog_notification_init (xlator_t *this, gf_changelog_t *gfc);
+
+void *
+gf_changelog_process (void *data);
+
+ssize_t
+gf_changelog_read_path (int fd, char *buffer, size_t bufsize);
+
+void
+gf_rfc3986_encode (unsigned char *s, char *enc, char *estr);
+
+size_t
+gf_changelog_write (int fd, char *buffer, size_t len);
+
+ssize_t
+gf_readline (int fd, void *vptr, size_t maxlen);
+
+int
+gf_ftruncate (int fd, off_t length);
+
+off_t
+gf_lseek (int fd, off_t offset, int whence);
+
+
+#endif
diff --git a/xlators/features/changelog/lib/src/gf-changelog-process.c b/xlators/features/changelog/lib/src/gf-changelog-process.c
new file mode 100644
index 000000000..df7204931
--- /dev/null
+++ b/xlators/features/changelog/lib/src/gf-changelog-process.c
@@ -0,0 +1,571 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <unistd.h>
+#include <pthread.h>
+
+#include "uuid.h"
+#include "globals.h"
+#include "glusterfs.h"
+
+#include "gf-changelog-helpers.h"
+
+/* from the changelog translator */
+#include "changelog-misc.h"
+
+extern int byebye;
+
+/**
+ * number of gfid records after fop number
+ */
+int nr_gfids[] = {
+        [GF_FOP_MKNOD]   = 1,
+        [GF_FOP_MKDIR]   = 1,
+        [GF_FOP_UNLINK]  = 1,
+        [GF_FOP_RMDIR]   = 1,
+        [GF_FOP_SYMLINK] = 1,
+        [GF_FOP_RENAME]  = 2,
+        [GF_FOP_LINK]    = 1,
+        [GF_FOP_CREATE]  = 1,
+};
+
+static char *
+binary_to_ascii (uuid_t uuid)
+{
+        return uuid_utoa (uuid);
+}
+
+static char *
+conv_noop (char *ptr) { return ptr; }
+
+#define VERIFY_SEPARATOR(ptr, plen, perr)       \
+        {                                       \
+                if (*(ptr + plen) != '\0') {    \
+                        perr = 1;               \
+                        break;                  \
+                }                               \
+        }
+
+#define MOVER_MOVE(mover, nleft, bytes)         \
+        {                                       \
+                mover += bytes;                 \
+                nleft -= bytes;                 \
+        }                                       \
+
+#define PARSE_GFID(mov, ptr, le, fn, perr)                      \
+        {                                                       \
+                VERIFY_SEPARATOR (mov, le, perr);               \
+                ptr = fn (mov);                                 \
+                if (!ptr) {                                     \
+                        perr = 1;                               \
+                        break;                                  \
+                }                                               \
+        }
+
+#define FILL_AND_MOVE(pt, buf, of, mo, nl, le)                          \
+        {                                                               \
+                GF_CHANGELOG_FILL_BUFFER (pt, buf, of, strlen (pt));    \
+                MOVER_MOVE (mo, nl, le);                                \
+        }
+
+
+#define PARSE_GFID_MOVE(ptr, uuid, mover, nleft, perr)          \
+        {                                                       \
+                memcpy (uuid, mover, sizeof (uuid_t));          \
+                ptr = binary_to_ascii (uuid);                   \
+                if (!ptr) {                                     \
+                        perr = 1;                               \
+                        break;                                  \
+                }                                               \
+                MOVER_MOVE (mover, nleft, sizeof (uuid_t));     \
+        }                                                       \
+
+#define LINE_BUFSIZE  3*PATH_MAX /* enough buffer for extra chars too */
+
+/**
+ * using mmap() makes parsing easy. fgets() cannot be used here as
+ * the binary gfid could contain a line-feed (0x0A), in that case fgets()
+ * would read an incomplete line and parsing would fail. using POSIX fds
+ * would result is additional code to maintain state in case of partial
+ * reads of data (where multiple entries do not fit extirely in the buffer).
+ *
+ * mmap() gives the flexibility of pointing to an offset in the file
+ * without us worrying about reading it in memory (VM does that for us for
+ * free).
+ */
+
+static int
+gf_changelog_parse_binary (xlator_t *this,
+                           gf_changelog_t *gfc, int from_fd, int to_fd,
+                           size_t start_offset, struct stat *stbuf)
+
+{
+        int     ret              = -1;
+        off_t   off              = 0;
+        off_t   nleft            = 0;
+        uuid_t  uuid             = {0,};
+        char   *ptr              = NULL;
+        char   *bname_start      = NULL;
+        char   *bname_end        = NULL;
+        char   *mover            = NULL;
+        char   *start            = NULL;
+        char    current_mover    = ' ';
+        size_t  blen             = 0;
+        int     parse_err        = 0;
+        char ascii[LINE_BUFSIZE] = {0,};
+
+        nleft = stbuf->st_size;
+
+        start = (char *) mmap (NULL, nleft,
+                               PROT_READ, MAP_PRIVATE, from_fd, 0);
+        if (!start) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "mmap() error (reason: %s)", strerror (errno));
+                goto out;
+        }
+
+        mover = start;
+
+        MOVER_MOVE (mover, nleft, start_offset);
+
+        while (nleft > 0) {
+
+                off = blen = 0;
+                ptr = bname_start = bname_end = NULL;
+
+                current_mover = *mover;
+
+                switch (current_mover) {
+                case 'D':
+                case 'M':
+                        MOVER_MOVE (mover, nleft, 1);
+                        PARSE_GFID_MOVE (ptr, uuid, mover, nleft, parse_err);
+
+                        break;
+
+                case 'E':
+                        MOVER_MOVE (mover, nleft, 1);
+                        PARSE_GFID_MOVE (ptr, uuid, mover, nleft, parse_err);
+
+                        bname_start = mover;
+                        if ( (bname_end = strchr (mover, '\n')) == NULL ) {
+                                parse_err = 1;
+                                break;
+                        }
+
+                        blen = bname_end - bname_start;
+                        MOVER_MOVE (mover, nleft, blen);
+
+                        break;
+
+                default:
+                        parse_err = 1;
+                }
+
+                if (parse_err)
+                        break;
+
+                GF_CHANGELOG_FILL_BUFFER (&current_mover, ascii, off, 1);
+                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
+                GF_CHANGELOG_FILL_BUFFER (ptr, ascii, off, strlen (ptr));
+                if (blen)
+                        GF_CHANGELOG_FILL_BUFFER (bname_start,
+                                                  ascii, off, blen);
+                GF_CHANGELOG_FILL_BUFFER ("\n", ascii, off, 1);
+
+                if (gf_changelog_write (to_fd, ascii, off) != off) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "processing binary changelog failed due to "
+                                " error in writing ascii change (reason: %s)",
+                                strerror (errno));
+                        break;
+                }
+
+                MOVER_MOVE (mover, nleft, 1);
+        }
+
+        if ( (nleft == 0) && (!parse_err))
+                ret = 0;
+
+        if (munmap (start, stbuf->st_size))
+                gf_log (this->name, GF_LOG_ERROR,
+                        "munmap() error (reason: %s)", strerror (errno));
+ out:
+        return ret;
+}
+
+/**
+ * ascii decoder:
+ *  - separate out one entry from another
+ *  - use fop name rather than fop number
+ */
+static int
+gf_changelog_parse_ascii (xlator_t *this,
+                          gf_changelog_t *gfc, int from_fd, int to_fd,
+                          size_t start_offset, struct stat *stbuf)
+{
+        int         ng            = 0;
+        int         ret           = -1;
+        int         fop           = 0;
+        int         len           = 0;
+        off_t       off           = 0;
+        off_t       nleft         = 0;
+        char       *ptr           = NULL;
+        char       *eptr          = NULL;
+        char       *start         = NULL;
+        char       *mover         = NULL;
+        int         parse_err     = 0;
+        char        current_mover = ' ';
+        char ascii[LINE_BUFSIZE]  = {0,};
+        const char *fopname       = NULL;
+
+        nleft = stbuf->st_size;
+
+        start = (char *) mmap (NULL, nleft,
+                               PROT_READ, MAP_PRIVATE, from_fd, 0);
+        if (!start) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "mmap() error (reason: %s)", strerror (errno));
+                goto out;
+        }
+
+        mover = start;
+
+        MOVER_MOVE (mover, nleft, start_offset);
+
+        while (nleft > 0) {
+                off = 0;
+                current_mover = *mover;
+
+                GF_CHANGELOG_FILL_BUFFER (&current_mover, ascii, off, 1);
+                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
+
+                switch (current_mover) {
+                case 'D':
+                case 'M':
+                        MOVER_MOVE (mover, nleft, 1);
+
+                        /* target gfid */
+                        PARSE_GFID (mover, ptr, UUID_CANONICAL_FORM_LEN,
+                                    conv_noop, parse_err);
+                        FILL_AND_MOVE(ptr, ascii, off,
+                                      mover, nleft, UUID_CANONICAL_FORM_LEN);
+                        break;
+
+                case 'E':
+                        MOVER_MOVE (mover, nleft, 1);
+
+                        /* target gfid */
+                        PARSE_GFID (mover, ptr, UUID_CANONICAL_FORM_LEN,
+                                    conv_noop, parse_err);
+                        FILL_AND_MOVE (ptr, ascii, off,
+                                       mover, nleft, UUID_CANONICAL_FORM_LEN);
+                        FILL_AND_MOVE (" ", ascii, off,
+                                       mover, nleft, 1);
+
+                        /* fop */
+                        len = strlen (mover);
+                        VERIFY_SEPARATOR (mover, len, parse_err);
+
+                        fop = atoi (mover);
+                        if ( (fopname = gf_fop_list[fop]) == NULL) {
+                                parse_err = 1;
+                                break;
+                        }
+
+                        MOVER_MOVE (mover, nleft, len);
+
+                        len = strlen (fopname);
+                        GF_CHANGELOG_FILL_BUFFER (fopname, ascii, off, len);
+
+                        /* pargfid + bname */
+                        ng = nr_gfids[fop];
+                        while (ng-- > 0) {
+                                MOVER_MOVE (mover, nleft, 1);
+                                len = strlen (mover);
+                                GF_CHANGELOG_FILL_BUFFER (" ", ascii, off, 1);
+
+                                PARSE_GFID (mover, ptr, len,
+                                            conv_noop, parse_err);
+                                eptr = calloc (3, strlen (ptr));
+                                if (!eptr) {
+                                        parse_err = 1;
+                                        break;
+                                }
+
+                                gf_rfc3986_encode ((unsigned char *) ptr,
+                                                   eptr, gfc->rfc3986);
+                                FILL_AND_MOVE (eptr, ascii, off,
+                                               mover, nleft, len);
+                                free (eptr);
+                        }
+
+                        break;
+                default:
+                        parse_err = 1;
+                }
+
+                if (parse_err)
+                        break;
+
+                GF_CHANGELOG_FILL_BUFFER ("\n", ascii, off, 1);
+
+                if (gf_changelog_write (to_fd, ascii, off) != off) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "processing ascii changelog failed due to "
+                                " wrror in writing change (reason: %s)",
+                                strerror (errno));
+                        break;
+                }
+
+                MOVER_MOVE (mover, nleft, 1);
+
+        }
+
+        if ( (nleft == 0) && (!parse_err))
+                ret = 0;
+
+        if (munmap (start, stbuf->st_size))
+                gf_log (this->name, GF_LOG_ERROR,
+                        "munmap() error (reason: %s)", strerror (errno));
+
+ out:
+        return ret;
+}
+
+#define COPY_BUFSIZE  8192
+static int
+gf_changelog_copy (xlator_t *this, int from_fd, int to_fd)
+{
+        ssize_t size                  = 0;
+        char   buffer[COPY_BUFSIZE+1] = {0,};
+
+        while (1) {
+                size = read (from_fd, buffer, COPY_BUFSIZE);
+                if (size <= 0)
+                        break;
+
+                if (gf_changelog_write (to_fd,
+                                        buffer, size) != size) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error processing ascii changlog");
+                        size = -1;
+                        break;
+                }
+        }
+
+        return (size < 0 ? -1 : 0);
+}
+
+static int
+gf_changelog_decode (xlator_t *this, gf_changelog_t *gfc, int from_fd,
+                     int to_fd, struct stat *stbuf, int *zerob)
+{
+        int    ret        = -1;
+        int    encoding   = -1;
+        size_t elen       = 0;
+        char buffer[1024] = {0,};
+
+        CHANGELOG_GET_ENCODING (from_fd, buffer, 1024, encoding, elen);
+        if (encoding == -1) /* unknown encoding */
+                goto out;
+
+        if (!CHANGELOG_VALID_ENCODING (encoding))
+                goto out;
+
+        if (elen == stbuf->st_size) {
+                *zerob = 1;
+                goto out;
+        }
+
+        /**
+         * start processing after the header
+         */
+        lseek (from_fd, elen, SEEK_SET);
+
+        switch (encoding) {
+        case CHANGELOG_ENCODE_BINARY:
+                /**
+                 * this ideally should have been a part of changelog-encoders.c
+                 * (ie. part of the changelog translator).
+                 */
+                ret = gf_changelog_parse_binary (this, gfc, from_fd,
+                                                 to_fd, elen, stbuf);
+                break;
+
+        case CHANGELOG_ENCODE_ASCII:
+                ret = gf_changelog_parse_ascii (this, gfc, from_fd,
+                                                to_fd, elen, stbuf);
+                break;
+        default:
+                ret = gf_changelog_copy (this, from_fd, to_fd);
+        }
+
+ out:
+        return ret;
+}
+
+static int
+gf_changelog_consume (xlator_t *this, gf_changelog_t *gfc, char *from_path)
+{
+        int         ret        = -1;
+        int         fd1        = 0;
+        int         fd2        = 0;
+        int         zerob      = 0;
+        struct stat stbuf      = {0,};
+        char dest[PATH_MAX]    = {0,};
+        char to_path[PATH_MAX] = {0,};
+
+        ret = stat (from_path, &stbuf);
+        if (ret || !S_ISREG(stbuf.st_mode)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "stat failed on changelog file: %s", from_path);
+                goto out;
+        }
+
+        fd1 = open (from_path, O_RDONLY);
+        if (fd1 < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cannot open changelog file: %s (reason: %s)",
+                        from_path, strerror (errno));
+                goto out;
+        }
+
+        (void) snprintf (to_path, PATH_MAX, "%s%s",
+                         gfc->gfc_current_dir, basename (from_path));
+        (void) snprintf (dest, PATH_MAX, "%s%s",
+                         gfc->gfc_processing_dir, basename (from_path));
+
+        fd2 = open (to_path, O_CREAT | O_TRUNC | O_RDWR,
+                    S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+        if (fd2 < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cannot create ascii changelog file %s (reason %s)",
+                        to_path, strerror (errno));
+                goto close_fd;
+        } else {
+                ret = gf_changelog_decode (this, gfc, fd1,
+                                           fd2, &stbuf, &zerob);
+
+                close (fd2);
+
+                if (!ret) {
+                        /* move it to processing on a successfull
+                           decode */
+                        ret = rename (to_path, dest);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "error moving %s to processing dir"
+                                        " (reason: %s)", to_path,
+                                        strerror (errno));
+                }
+
+                /* remove it from .current if it's an empty file */
+                if (zerob) {
+                        ret = unlink (to_path);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "could not unlink %s (reason: %s",
+                                        to_path, strerror (errno));
+                }
+        }
+
+ close_fd:
+        close (fd1);
+
+ out:
+        return ret;
+}
+
+static char *
+gf_changelog_ext_change (xlator_t *this,
+                         gf_changelog_t *gfc, char *path, size_t readlen)
+{
+        int     alo = 0;
+        int     ret = 0;
+        size_t  len = 0;
+        char   *buf = NULL;
+
+        buf = path;
+        while (len < readlen) {
+                if (*buf == '\0') {
+                        alo = 1;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "processing changelog: %s", path);
+                        ret = gf_changelog_consume (this, gfc, path);
+                }
+
+                if (ret)
+                        break;
+
+                len++; buf++;
+                if (alo) {
+                        alo = 0;
+                        path = buf;
+                }
+        }
+
+        return (ret) ? NULL : path;
+}
+
+void *
+gf_changelog_process (void *data)
+{
+        ssize_t         len      = 0;
+        ssize_t         offlen   = 0;
+        xlator_t       *this     = NULL;
+        char           *sbuf     = NULL;
+        gf_changelog_t *gfc      = NULL;
+        char from_path[PATH_MAX] = {0,};
+
+        gfc = (gf_changelog_t *) data;
+        this = gfc->this;
+
+        pthread_detach (pthread_self());
+
+        for (;;) {
+                len = gf_changelog_read_path (gfc->gfc_sockfd,
+                                              from_path + offlen,
+                                              PATH_MAX - offlen);
+                if (len < 0)
+                        continue; /* ignore it for now */
+
+                if (len == 0) { /* close() from the changelog translator */
+                        gf_log (this->name, GF_LOG_INFO, "close from changelog"
+                                " notification translator.");
+
+                        if (gfc->gfc_connretries != 1) {
+                                if (!gf_changelog_notification_init(this, gfc))
+                                        continue;
+                        }
+
+                        byebye = 1;
+                        break;
+                }
+
+                len += offlen;
+                sbuf = gf_changelog_ext_change (this, gfc, from_path, len);
+                if (!sbuf) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "could not extract changelog filename");
+                        continue;
+                }
+
+                offlen = 0;
+                if (sbuf != (from_path + len)) {
+                        offlen = from_path + len - sbuf;
+                        memmove (from_path, sbuf, offlen);
+                }
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "byebye (%d) from processing thread...", byebye);
+        return NULL;
+}
diff --git a/xlators/features/changelog/lib/src/gf-changelog.c b/xlators/features/changelog/lib/src/gf-changelog.c
new file mode 100644
index 000000000..0827f2cac
--- /dev/null
+++ b/xlators/features/changelog/lib/src/gf-changelog.c
@@ -0,0 +1,571 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <errno.h>
+#include <dirent.h>
+#include <stddef.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <string.h>
+
+#include "globals.h"
+#include "glusterfs.h"
+#include "logging.h"
+
+#include "gf-changelog-helpers.h"
+
+/* from the changelog translator */
+#include "changelog-misc.h"
+#include "changelog-mem-types.h"
+
+int byebye = 0;
+
+static void
+gf_changelog_cleanup (gf_changelog_t *gfc)
+{
+        /* socket */
+        if (gfc->gfc_sockfd != -1)
+                close (gfc->gfc_sockfd);
+        /* tracker fd */
+        if (gfc->gfc_fd != -1)
+                close (gfc->gfc_fd);
+        /* processing dir */
+        if (gfc->gfc_dir)
+                closedir (gfc->gfc_dir);
+
+        if (gfc->gfc_working_dir)
+                free (gfc->gfc_working_dir); /* allocated by realpath */
+}
+
+void
+__attribute__ ((constructor)) gf_changelog_ctor (void)
+{
+        glusterfs_ctx_t *ctx = NULL;
+
+        ctx = glusterfs_ctx_new ();
+        if (!ctx)
+                return;
+
+        if (glusterfs_globals_init (ctx)) {
+                free (ctx);
+                ctx = NULL;
+                return;
+        }
+
+        THIS->ctx = ctx;
+}
+
+void
+__attribute__ ((destructor)) gf_changelog_dtor (void)
+{
+        xlator_t        *this = NULL;
+        glusterfs_ctx_t *ctx  = NULL;
+        gf_changelog_t  *gfc  = NULL;
+
+        this = THIS;
+        if (!this)
+                return;
+
+        ctx = this->ctx;
+        gfc = this->private;
+
+        if (gfc) {
+                if (gfc->hist_gfc) {
+                        gf_changelog_cleanup(gfc->hist_gfc);
+                        GF_FREE (gfc->hist_gfc);
+                }
+                gf_changelog_cleanup (gfc);
+                GF_FREE (gfc);
+        }
+
+        if (ctx) {
+                pthread_mutex_destroy (&ctx->lock);
+                free (ctx);
+                ctx = NULL;
+        }
+}
+
+
+static int
+gf_changelog_open_dirs (gf_changelog_t *gfc)
+{
+        int  ret                    = -1;
+        DIR *dir                    = NULL;
+        int  tracker_fd             = 0;
+        char tracker_path[PATH_MAX] = {0,};
+
+        (void) snprintf (gfc->gfc_current_dir, PATH_MAX,
+                         "%s/"GF_CHANGELOG_CURRENT_DIR"/",
+                         gfc->gfc_working_dir);
+        ret = mkdir_p (gfc->gfc_current_dir, 0600, _gf_false);
+        if (ret)
+                goto out;
+
+        (void) snprintf (gfc->gfc_processed_dir, PATH_MAX,
+                         "%s/"GF_CHANGELOG_PROCESSED_DIR"/",
+                         gfc->gfc_working_dir);
+        ret = mkdir_p (gfc->gfc_processed_dir, 0600, _gf_false);
+        if (ret)
+                goto out;
+
+        (void) snprintf (gfc->gfc_processing_dir, PATH_MAX,
+                         "%s/"GF_CHANGELOG_PROCESSING_DIR"/",
+                         gfc->gfc_working_dir);
+        ret = mkdir_p (gfc->gfc_processing_dir, 0600, _gf_false);
+        if (ret)
+                goto out;
+
+        dir = opendir (gfc->gfc_processing_dir);
+        if (!dir) {
+                gf_log ("", GF_LOG_ERROR,
+                        "opendir() error [reason: %s]", strerror (errno));
+                goto out;
+        }
+
+        gfc->gfc_dir = dir;
+
+        (void) snprintf (tracker_path, PATH_MAX,
+                         "%s/"GF_CHANGELOG_TRACKER, gfc->gfc_working_dir);
+
+        tracker_fd = open (tracker_path, O_CREAT | O_APPEND | O_RDWR,
+                           S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+        if (tracker_fd < 0) {
+                closedir (gfc->gfc_dir);
+                ret = -1;
+                goto out;
+        }
+
+        gfc->gfc_fd = tracker_fd;
+        ret = 0;
+ out:
+        return ret;
+}
+
+int
+gf_changelog_notification_init (xlator_t *this, gf_changelog_t *gfc)
+{
+        int                ret    = 0;
+        int                len    = 0;
+        int                tries  = 0;
+        int                sockfd = 0;
+        struct sockaddr_un remote;
+
+        this = gfc->this;
+
+        if (gfc->gfc_sockfd != -1) {
+                gf_log (this->name, GF_LOG_INFO,
+                        "Reconnecting...");
+                close (gfc->gfc_sockfd);
+        }
+
+        sockfd = socket (AF_UNIX, SOCK_STREAM, 0);
+        if (sockfd < 0) {
+                ret = -1;
+                goto out;
+        }
+
+        CHANGELOG_MAKE_SOCKET_PATH (gfc->gfc_brickpath,
+                                    gfc->gfc_sockpath, UNIX_PATH_MAX);
+        gf_log (this->name, GF_LOG_INFO,
+                "connecting to changelog socket: %s (brick: %s)",
+                gfc->gfc_sockpath, gfc->gfc_brickpath);
+
+        remote.sun_family = AF_UNIX;
+        strcpy (remote.sun_path, gfc->gfc_sockpath);
+
+        len = strlen (remote.sun_path) + sizeof (remote.sun_family);
+
+        while (tries < gfc->gfc_connretries) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "connection attempt %d/%d...",
+                        tries + 1, gfc->gfc_connretries);
+
+                /* initiate a connect */
+                if (connect (sockfd, (struct sockaddr *) &remote, len) == 0) {
+                        gfc->gfc_sockfd = sockfd;
+                        break;
+                }
+
+                tries++;
+                sleep (2);
+        }
+
+        if (tries == gfc->gfc_connretries) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not connect to changelog socket!"
+                        " bailing out...");
+                close (sockfd);
+                ret = -1;
+        } else
+                gf_log (this->name, GF_LOG_INFO,
+                        "connection successful");
+
+ out:
+        return ret;
+}
+
+int
+gf_changelog_done (char *file)
+{
+        int             ret    = -1;
+        char           *buffer = NULL;
+        xlator_t       *this   = NULL;
+        gf_changelog_t *gfc    = NULL;
+        char to_path[PATH_MAX] = {0,};
+
+        errno = EINVAL;
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        if (!file || !strlen (file))
+                goto out;
+
+        /* make sure 'file' is inside ->gfc_working_dir */
+        buffer = realpath (file, NULL);
+        if (!buffer)
+                goto out;
+
+        if (strncmp (gfc->gfc_working_dir,
+                     buffer, strlen (gfc->gfc_working_dir)))
+                goto out;
+
+        (void) snprintf (to_path, PATH_MAX, "%s%s",
+                         gfc->gfc_processed_dir, basename (buffer));
+        gf_log (this->name, GF_LOG_DEBUG,
+                "moving %s to processed directory", file);
+        ret = rename (buffer, to_path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cannot move %s to %s (reason: %s)",
+                        file, to_path, strerror (errno));
+                goto out;
+        }
+
+        ret = 0;
+
+ out:
+        if (buffer)
+                free (buffer); /* allocated by realpath() */
+        return ret;
+}
+
+/**
+ * @API
+ *  for a set of changelogs, start from the begining
+ */
+int
+gf_changelog_start_fresh ()
+{
+        xlator_t *this = NULL;
+        gf_changelog_t *gfc = NULL;
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        errno = EINVAL;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        if (gf_ftruncate (gfc->gfc_fd, 0))
+                goto out;
+
+        return 0;
+
+ out:
+        return -1;
+}
+
+/**
+ * @API
+ * return the next changelog file entry. zero means all chanelogs
+ * consumed.
+ */
+ssize_t
+gf_changelog_next_change (char *bufptr, size_t maxlen)
+{
+        ssize_t         size       = 0;
+        int             tracker_fd = 0;
+        xlator_t       *this       = NULL;
+        gf_changelog_t *gfc        = NULL;
+        char buffer[PATH_MAX]      = {0,};
+
+        errno = EINVAL;
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        tracker_fd = gfc->gfc_fd;
+
+        size = gf_readline (tracker_fd, buffer, maxlen);
+        if (size < 0)
+                goto out;
+        if (size == 0)
+                return 0;
+
+        memcpy (bufptr, buffer, size - 1);
+        *(buffer + size) = '\0';
+
+        return size;
+
+ out:
+        return -1;
+}
+
+/**
+ * @API
+ *  gf_changelog_scan() - scan and generate a list of change entries
+ *
+ * calling this api multiple times (without calling gf_changlog_done())
+ * would result new changelogs(s) being refreshed in the tracker file.
+ * This call also acts as a cancellation point for the consumer.
+ */
+ssize_t
+gf_changelog_scan ()
+{
+        int             ret        = 0;
+        int             tracker_fd = 0;
+        size_t          len        = 0;
+        size_t          off        = 0;
+        xlator_t       *this       = NULL;
+        size_t          nr_entries = 0;
+        gf_changelog_t *gfc        = NULL;
+        struct dirent  *entryp     = NULL;
+        struct dirent  *result     = NULL;
+        char buffer[PATH_MAX]      = {0,};
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        /**
+         * do we need to protect 'byebye' with locks? worst, the
+         * consumer would get notified during next scan().
+         */
+        if (byebye) {
+                errno = ECONNREFUSED;
+                goto out;
+        }
+
+        errno = EINVAL;
+
+        tracker_fd = gfc->gfc_fd;
+
+        if (gf_ftruncate (tracker_fd, 0))
+                goto out;
+
+        len = offsetof(struct dirent, d_name)
+                + pathconf(gfc->gfc_processing_dir, _PC_NAME_MAX) + 1;
+        entryp = GF_CALLOC (1, len,
+                            gf_changelog_mt_libgfchangelog_dirent_t);
+        if (!entryp)
+                goto out;
+
+        rewinddir (gfc->gfc_dir);
+        while (1) {
+                ret = readdir_r (gfc->gfc_dir, entryp, &result);
+                if (ret || !result)
+                        break;
+
+                if ( !strcmp (basename (entryp->d_name), ".")
+                     || !strcmp (basename (entryp->d_name), "..") )
+                        continue;
+
+                nr_entries++;
+
+                GF_CHANGELOG_FILL_BUFFER (gfc->gfc_processing_dir,
+                                          buffer, off,
+                                          strlen (gfc->gfc_processing_dir));
+                GF_CHANGELOG_FILL_BUFFER (entryp->d_name, buffer,
+                                          off, strlen (entryp->d_name));
+                GF_CHANGELOG_FILL_BUFFER ("\n", buffer, off, 1);
+
+                if (gf_changelog_write (tracker_fd, buffer, off) != off) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error writing changelog filename"
+                                " to tracker file");
+                        break;
+                }
+                off = 0;
+        }
+
+        GF_FREE (entryp);
+
+        if (!result) {
+                if (gf_lseek (tracker_fd, 0, SEEK_SET) != -1)
+                        return nr_entries;
+        }
+ out:
+        return -1;
+}
+
+/**
+ * @API
+ *  gf_changelog_register() - register a client for updates.
+ */
+int
+gf_changelog_register (char *brick_path, char *scratch_dir,
+                       char *log_file, int log_level, int max_reconnects)
+{
+        int             i    = 0;
+        int             ret  = -1;
+        int             errn = 0;
+        xlator_t       *this = NULL;
+        gf_changelog_t *gfc  = NULL;
+        char hist_scratch_dir[PATH_MAX] = {0,};
+
+        this = THIS;
+        if (!this->ctx)
+                goto out;
+
+        errno = ENOMEM;
+
+        gfc = GF_CALLOC (1, sizeof (*gfc),
+                         gf_changelog_mt_libgfchangelog_t);
+        if (!gfc)
+                goto out;
+
+        gfc->this = this;
+
+        gfc->gfc_dir = NULL;
+        gfc->gfc_fd = gfc->gfc_sockfd = -1;
+
+        gfc->gfc_working_dir = realpath (scratch_dir, NULL);
+        if (!gfc->gfc_working_dir) {
+                errn = errno;
+                goto cleanup;
+        }
+
+        /* Begin: Changes for History API */
+        gfc->hist_gfc = NULL;
+
+        gfc->hist_gfc = GF_CALLOC (1, sizeof (*gfc),
+                         gf_changelog_mt_libgfchangelog_t);
+        if (!gfc->hist_gfc)
+                goto cleanup;
+
+        gfc->hist_gfc->gfc_dir = NULL;
+        gfc->hist_gfc->gfc_fd = gfc->hist_gfc->gfc_sockfd = -1;
+        gfc->hist_gfc->this = NULL;
+
+        (void) strncpy (hist_scratch_dir, scratch_dir, PATH_MAX);
+        (void) snprintf (hist_scratch_dir, PATH_MAX,
+                         "%s/"GF_CHANGELOG_HISTORY_DIR"/",
+                         gfc->gfc_working_dir);
+
+        ret = mkdir_p (hist_scratch_dir, 0600, _gf_false);
+        if (ret) {
+                errn = errno;
+                goto cleanup;
+        }
+
+        gfc->hist_gfc->gfc_working_dir = realpath (hist_scratch_dir, NULL);
+        if (!gfc->hist_gfc->gfc_working_dir) {
+                errn = errno;
+                goto cleanup;
+        }
+
+        ret = gf_changelog_open_dirs (gfc->hist_gfc);
+        if (ret) {
+                errn = errno;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not create entries in history scratch dir");
+                goto cleanup;
+        }
+
+        (void) strncpy (gfc->hist_gfc->gfc_brickpath, brick_path, PATH_MAX);
+
+        for (i=0; i < 256; i++) {
+                gfc->hist_gfc->rfc3986[i] =
+                        (isalnum(i) || i == '~' ||
+                        i == '-' || i == '.' || i == '_') ? i : 0;
+        }
+        /* End: Changes for History API*/
+
+        ret = gf_changelog_open_dirs (gfc);
+        if (ret) {
+                errn = errno;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not create entries in scratch dir");
+                goto cleanup;
+        }
+
+        /* passing ident as NULL means to use default ident for syslog */
+        if (gf_log_init (this->ctx, log_file, NULL))
+                goto cleanup;
+
+        gf_log_set_loglevel ((log_level == -1) ? GF_LOG_INFO :
+                             log_level);
+
+        gfc->gfc_connretries = (max_reconnects <= 0) ? 1 : max_reconnects;
+        (void) strncpy (gfc->gfc_brickpath, brick_path, PATH_MAX);
+
+        ret = gf_changelog_notification_init (this, gfc);
+        if (ret) {
+                errn = errno;
+                goto cleanup;
+        }
+
+        ret = gf_thread_create (&gfc->gfc_changelog_processor,
+				NULL, gf_changelog_process, gfc);
+        if (ret) {
+                errn = errno;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error creating changelog processor thread"
+                        " new changes won't be recorded!!!");
+                goto cleanup;
+        }
+
+        for (i=0; i < 256; i++) {
+                gfc->rfc3986[i] =
+                        (isalnum(i) || i == '~' ||
+                        i == '-' || i == '.' || i == '_') ? i : 0;
+        }
+
+        ret = 0;
+        this->private = gfc;
+
+        goto out;
+
+ cleanup:
+        if (gfc->hist_gfc) {
+                gf_changelog_cleanup (gfc->hist_gfc);
+                GF_FREE (gfc->hist_gfc);
+        }
+        gf_changelog_cleanup (gfc);
+        GF_FREE (gfc);
+        this->private = NULL;
+        errno = errn;
+
+ out:
+        return ret;
+}
diff --git a/xlators/features/changelog/lib/src/gf-history-changelog.c b/xlators/features/changelog/lib/src/gf-history-changelog.c
new file mode 100644
index 000000000..bfc4cd37d
--- /dev/null
+++ b/xlators/features/changelog/lib/src/gf-history-changelog.c
@@ -0,0 +1,274 @@
+#include <errno.h>
+#include <dirent.h>
+#include <stddef.h>
+#include <sys/types.h>
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <string.h>
+
+#include "globals.h"
+#include "glusterfs.h"
+#include "logging.h"
+
+#include "gf-changelog-helpers.h"
+
+/* from the changelog translator */
+#include "changelog-misc.h"
+#include "changelog-mem-types.h"
+
+/*@API
+ * gf_history_changelog_done:
+ *    Move processed history changelog file from .processing
+ *    to .processed
+ *
+ * ARGUMENTS:
+ *    file(IN): path to processed history changelog file in
+ *    .processing directory.
+ *
+ * RETURN VALUE:
+ *    0: On success.
+ *   -1: On error.
+ */
+int
+gf_history_changelog_done (char *file)
+{
+        int                     ret                     = -1;
+        char                    *buffer                 = NULL;
+        xlator_t                *this                   = NULL;
+        gf_changelog_t          *gfc                    = NULL;
+        gf_changelog_t          *hist_gfc               = NULL;
+        char                    to_path[PATH_MAX]       = {0,};
+
+        errno = EINVAL;
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        hist_gfc = gfc->hist_gfc;
+        if (!hist_gfc)
+                goto out;
+
+        if (!file || !strlen (file))
+                goto out;
+
+        /* make sure 'file' is inside ->gfc_working_dir */
+        buffer = realpath (file, NULL);
+        if (!buffer)
+                goto out;
+
+        if (strncmp (hist_gfc->gfc_working_dir,
+                     buffer, strlen (hist_gfc->gfc_working_dir)))
+                goto out;
+
+        (void) snprintf (to_path, PATH_MAX, "%s%s",
+                         hist_gfc->gfc_processed_dir, basename (buffer));
+        gf_log (this->name, GF_LOG_DEBUG,
+                "moving %s to processed directory", file);
+        ret = rename (buffer, to_path);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cannot move %s to %s (reason: %s)",
+                        file, to_path, strerror (errno));
+                goto out;
+        }
+
+        ret = 0;
+
+ out:
+        if (buffer)
+                free (buffer); /* allocated by realpath() */
+        return ret;
+}
+/**
+ * @API
+ *  gf_history_changelog_start_fresh:
+ *     For a set of changelogs, start from the begining.
+ *     It will truncates the history tracker fd.
+ *
+ *  RETURN VALUES:
+ *     0: On success.
+ *    -1: On error.
+ */
+int
+gf_history_changelog_start_fresh ()
+{
+        xlator_t                *this                   = NULL;
+        gf_changelog_t          *gfc                    = NULL;
+        gf_changelog_t          *hist_gfc               = NULL;
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        errno = EINVAL;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        hist_gfc = gfc->hist_gfc;
+        if (!hist_gfc)
+                goto out;
+
+        if (gf_ftruncate (hist_gfc->gfc_fd, 0))
+                goto out;
+
+        return 0;
+
+ out:
+        return -1;
+}
+
+/*
+ * @API
+ *  gf_history_changelog_next_change:
+ *     Return the next history changelog file entry. Zero means all
+ *     history chanelogs are consumed.
+ *
+ *  ARGUMENTS:
+ *     bufptr(OUT): Path to unprocessed history changelog file
+ *                  from tracker file.
+ *     maxlen(IN): Usually PATH_MAX.
+ *
+ *  RETURN VALUES:
+ *     size: On success.
+ *     -1  : On error.
+ */
+ssize_t
+gf_history_changelog_next_change (char *bufptr, size_t maxlen)
+{
+        ssize_t                 size                    = 0;
+        int                     tracker_fd              = 0;
+        xlator_t                *this                   = NULL;
+        gf_changelog_t          *gfc                    = NULL;
+        gf_changelog_t          *hist_gfc               = NULL;
+        char                    buffer[PATH_MAX]        = {0,};
+
+        errno = EINVAL;
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        hist_gfc = gfc->hist_gfc;
+        if (!hist_gfc)
+                goto out;
+
+        tracker_fd = hist_gfc->gfc_fd;
+
+        size = gf_readline (tracker_fd, buffer, maxlen);
+        if (size < 0)
+                goto out;
+        if (size == 0)
+                return 0;
+
+        memcpy (bufptr, buffer, size - 1);
+        *(buffer + size) = '\0';
+
+        return size;
+
+ out:
+        return -1;
+}
+
+/*
+ * @API
+ *  gf_history_changelog_scan:
+ *     Scan and generate a list of change entries.
+ *     Calling this api multiple times (without calling gf_changlog_done())
+ *     would result new changelogs(s) being refreshed in the tracker file.
+ *     This call also acts as a cancellation point for the consumer.
+ *
+ *  RETURN VALUES:
+ *     nr_entries: On success.
+ *     -1        : On error.
+ */
+ssize_t
+gf_history_changelog_scan ()
+{
+        int             ret        = 0;
+        int             tracker_fd = 0;
+        size_t          len        = 0;
+        size_t          off        = 0;
+        xlator_t       *this       = NULL;
+        size_t          nr_entries = 0;
+        gf_changelog_t *gfc        = NULL;
+        gf_changelog_t *hist_gfc        = NULL;
+        struct dirent  *entryp     = NULL;
+        struct dirent  *result     = NULL;
+        char buffer[PATH_MAX]      = {0,};
+
+        this = THIS;
+        if (!this)
+                goto out;
+
+        gfc = (gf_changelog_t *) this->private;
+        if (!gfc)
+                goto out;
+
+        hist_gfc = gfc->hist_gfc;
+        if (!hist_gfc)
+                goto out;
+
+        errno = EINVAL;
+
+        tracker_fd = hist_gfc->gfc_fd;
+
+        if (gf_ftruncate (tracker_fd, 0))
+                goto out;
+
+        len = offsetof(struct dirent, d_name)
+                + pathconf(hist_gfc->gfc_processing_dir, _PC_NAME_MAX) + 1;
+        entryp = GF_CALLOC (1, len,
+                            gf_changelog_mt_libgfchangelog_dirent_t);
+        if (!entryp)
+                goto out;
+
+        rewinddir (hist_gfc->gfc_dir);
+        while (1) {
+                ret = readdir_r (hist_gfc->gfc_dir, entryp, &result);
+                if (ret || !result)
+                        break;
+
+                if ( !strcmp (basename (entryp->d_name), ".")
+                     || !strcmp (basename (entryp->d_name), "..") )
+                        continue;
+
+                nr_entries++;
+
+                GF_CHANGELOG_FILL_BUFFER (hist_gfc->gfc_processing_dir,
+                                          buffer, off,
+                                          strlen (hist_gfc->gfc_processing_dir));
+                GF_CHANGELOG_FILL_BUFFER (entryp->d_name, buffer,
+                                          off, strlen (entryp->d_name));
+                GF_CHANGELOG_FILL_BUFFER ("\n", buffer, off, 1);
+
+                if (gf_changelog_write (tracker_fd, buffer, off) != off) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error writing changelog filename"
+                                " to tracker file");
+                        break;
+                }
+                off = 0;
+        }
+
+        GF_FREE (entryp);
+
+        if (!result) {
+                if (gf_lseek (tracker_fd, 0, SEEK_SET) != -1)
+                        return nr_entries;
+        }
+ out:
+        return -1;
+}
diff --git a/xlators/features/changelog/src/Makefile.am b/xlators/features/changelog/src/Makefile.am
new file mode 100644
index 000000000..525ce97dc
--- /dev/null
+++ b/xlators/features/changelog/src/Makefile.am
@@ -0,0 +1,21 @@
+xlator_LTLIBRARIES = changelog.la
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+noinst_HEADERS = changelog-helpers.h changelog-mem-types.h changelog-rt.h \
+	changelog-misc.h changelog-encoders.h changelog-notifier.h \
+	changelog-fops.h policy/changelog-policy.h
+
+changelog_la_LDFLAGS = -module -avoid-version
+
+changelog_la_SOURCES = changelog.c changelog-rt.c changelog-helpers.c \
+	changelog-encoders.c changelog-notifier.c changelog-default-fops.c \
+	policy/changelog-policy-default.c policy/changelog-policy-replication.c
+changelog_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -Ipolicy/ -fPIC -D_FILE_OFFSET_BITS=64 \
+	-D_GNU_SOURCE -D$(GF_HOST_OS) -shared -DDATADIR=\"$(localstatedir)\"
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/changelog/src/changelog-default-fops.c b/xlators/features/changelog/src/changelog-default-fops.c
new file mode 100644
index 000000000..59749905e
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-default-fops.c
@@ -0,0 +1,561 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "xlator.h"
+#include "defaults.h"
+#include "logging.h"
+
+#include "changelog-encoders.h"
+
+/** FOPS */
+
+/* default rmdir */
+int32_t
+changelog_default_rmdir (call_frame_t *frame, xlator_t *this,
+                         loc_t *loc, int xflags, dict_t *xdata)
+{
+        int                ret      = -1;
+        size_t             xtra_len = 0;
+        changelog_opt_t   *co       = NULL;
+        changelog_local_t *local    = NULL;
+
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, loc->inode->gfid, 2);
+        if (!local)
+                goto out;
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, out);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 2);
+
+        frame->local = local;
+        ret = 0;
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+/* default unlink */
+int32_t
+changelog_default_unlink (call_frame_t *frame, xlator_t *this,
+                          loc_t *loc, int xflags, dict_t *xdata)
+{
+        int                ret      = -1;
+        size_t             xtra_len = 0;
+        changelog_opt_t   *co       = NULL;
+        changelog_local_t *local    = NULL;
+
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, loc->inode->gfid, 2);
+        if (!local)
+                goto out;
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, out);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 2);
+
+        frame->local = local;
+        ret = 0;
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+/* default rename */
+int32_t
+changelog_default_rename (call_frame_t *frame, xlator_t *this,
+                          loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        int                ret      = -1;
+        size_t             xtra_len = 0;
+        changelog_opt_t   *co       = NULL;
+        changelog_local_t *local    = NULL;
+
+        /* 3 == fop + oldloc + newloc */
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, oldloc->inode->gfid, 3);
+        if (!local)
+                goto out;
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, oldloc->pargfid, oldloc->name,
+                              entry_fn, entry_free_fn, xtra_len, out);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, newloc->pargfid, newloc->name,
+                              entry_fn, entry_free_fn, xtra_len, out);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 3);
+
+        frame->local = local;
+        ret = 0;
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+/* default link */
+int32_t
+changelog_default_link (call_frame_t *frame,
+                        xlator_t *this, loc_t *oldloc,
+                        loc_t *newloc, dict_t *xdata)
+{
+        int                ret      = 1;
+        size_t             xtra_len = 0;
+        changelog_opt_t   *co       = NULL;
+        changelog_local_t *local    = NULL;
+
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, oldloc->gfid, 2);
+        if (!local)
+                goto out;
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, newloc->pargfid, newloc->name,
+                              entry_fn, entry_free_fn, xtra_len, out);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 2);
+
+        frame->local = local;
+        ret = 0;
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+/* default mknid */
+int32_t
+changelog_default_mkdir (call_frame_t *frame, xlator_t *this,
+                         loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
+{
+        int                ret      = -1;
+        uuid_t             gfid     = {0,};
+        void              *uuid_req = NULL;
+        size_t             xtra_len = 0;
+        changelog_opt_t   *co       = NULL;
+        changelog_local_t *local    = NULL;
+
+        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get gfid from dict");
+                goto out;
+        }
+        uuid_copy (gfid, uuid_req);
+
+        ret = -1;
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, gfid, 2);
+        if (!local)
+                goto out;
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, out);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 2);
+
+        frame->local = local;
+        ret = 0;
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+/* default symlink */
+int32_t
+changelog_default_symlink (call_frame_t *frame, xlator_t *this,
+                           const char *linkname, loc_t *loc,
+                           mode_t umask, dict_t *xdata)
+{
+        int                ret      = -1;
+        size_t             xtra_len = 0;
+        uuid_t             gfid     = {0,};
+        void              *uuid_req = NULL;
+        changelog_opt_t   *co       = NULL;
+        changelog_local_t *local    = NULL;
+
+        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get gfid from dict");
+                goto out;
+        }
+        uuid_copy (gfid, uuid_req);
+
+        ret = -1;
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, gfid, 2);
+        if (!local)
+                goto out;
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, out);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 2);
+
+        frame->local = local;
+        ret = 0;
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+/* default mknod */
+int32_t
+changelog_default_mknod (call_frame_t *frame,
+                         xlator_t *this, loc_t *loc,
+                         mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
+{
+        int                ret      = -1;
+        uuid_t             gfid     = {0,};
+        void              *uuid_req = NULL;
+        size_t             xtra_len = 0;
+        changelog_opt_t   *co       = NULL;
+        changelog_local_t *local    = NULL;
+
+        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get gfid from dict");
+                goto out;
+        }
+        uuid_copy (gfid, uuid_req);
+
+        ret = -1;
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, gfid, 2);
+
+        co = changelog_get_usable_buffer (frame->local);
+        if (!co)
+                goto out;
+
+        CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, out);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 2);
+
+        frame->local = local;
+        ret = 0;
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+/* default create */
+int32_t
+changelog_default_create (call_frame_t *frame, xlator_t *this,
+                          loc_t *loc, int32_t flags, mode_t mode,
+                          mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        int                ret      = -1;
+        uuid_t             gfid     = {0,};
+        void              *uuid_req = NULL;
+        changelog_opt_t   *co       = NULL;
+        size_t             xtra_len = 0;
+        changelog_local_t *local    = NULL;
+
+        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get gfid from dict");
+                goto out;
+        }
+        uuid_copy (gfid, uuid_req);
+
+        /* init with two extra records */
+        ret = -1;
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, gfid, 2);
+        if (!local)
+                goto out;
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+
+        co++;
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, out);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 2);
+
+        frame->local = local;
+        ret = 0;
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+/* default fsetattr */
+int32_t
+changelog_default_fsetattr (call_frame_t *frame,
+                            xlator_t *this, fd_t *fd,
+                            struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        CHANGELOG_INIT (this, frame->local,
+                        fd->inode, fd->inode->gfid, 0);
+        return 0;
+}
+
+/* default setattr */
+int32_t
+changelog_default_setattr (call_frame_t *frame,
+                           xlator_t *this, loc_t *loc,
+                           struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        CHANGELOG_INIT (this, frame->local,
+                        loc->inode, loc->inode->gfid, 0);
+        return 0;
+}
+
+/* default fremovexattr */
+int32_t
+changelog_default_fremovexattr (call_frame_t *frame, xlator_t *this,
+                                fd_t *fd, const char *name, dict_t *xdata)
+{
+        CHANGELOG_INIT (this, frame->local,
+                        fd->inode, fd->inode->gfid, 0);
+        return 0;
+}
+
+/* default removexattr */
+int32_t
+changelog_default_removexattr (call_frame_t *frame, xlator_t *this,
+                               loc_t *loc, const char *name, dict_t *xdata)
+{
+        CHANGELOG_INIT (this, frame->local,
+                        loc->inode, loc->inode->gfid, 0);
+        return 0;
+}
+
+/* default setxattr */
+int32_t
+changelog_default_setxattr (call_frame_t *frame,
+                            xlator_t *this, loc_t *loc,
+                            dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        CHANGELOG_INIT (this, frame->local,
+                        loc->inode, loc->inode->gfid, 0);
+        return 0;
+}
+
+/* default fsetxattr */
+int32_t
+changelog_default_fsetxattr (call_frame_t *frame,
+                             xlator_t *this, fd_t *fd, dict_t *dict,
+                             int32_t flags, dict_t *xdata)
+{
+        CHANGELOG_INIT (this, frame->local,
+                        fd->inode, fd->inode->gfid, 0);
+        return 0;
+}
+
+/* default truncate */
+int32_t
+changelog_default_truncate (call_frame_t *frame,
+                            xlator_t *this, loc_t *loc,
+                            off_t offset, dict_t *xdata)
+{
+        CHANGELOG_INIT (this, frame->local,
+                        loc->inode, loc->inode->gfid, 0);
+        return 0;
+}
+
+/* default ftruncate */
+int32_t
+changelog_default_ftruncate (call_frame_t *frame,
+                             xlator_t *this, fd_t *fd,
+                             off_t offset, dict_t *xdata)
+{
+        CHANGELOG_INIT (this, frame->local,
+                        fd->inode, fd->inode->gfid, 0);
+        return 0;
+}
+
+/* default writev */
+int32_t
+changelog_default_writev (call_frame_t *frame,
+                          xlator_t *this, fd_t *fd, struct iovec *vector,
+                          int32_t count, off_t offset, uint32_t flags,
+                          struct iobref *iobref, dict_t *xdata)
+{
+        CHANGELOG_INIT (this, frame->local,
+                        fd->inode, fd->inode->gfid, 0);
+        return 0;
+}
+
+/** COPS */
+
+int
+changelog_default_cops_open (xlator_t *this,
+                             changelog_priv_t *priv,
+                             void *cpriv, char *name, gf_boolean_t last)
+{
+        changelog_log_data_t       cld = {0,};
+        changelog_rollover_data_t *crd = NULL;
+        struct timeval             tv  = {0,};
+
+        crd = &cld.cld_roll;
+
+        cld.cld_type = CHANGELOG_TYPE_ROLLOVER;
+
+        if (gettimeofday (&tv, NULL))
+                return -1;
+
+        crd->crd_prealloc_size = 0; /* no preallocation */
+        crd->crd_finale = last;
+        crd->crd_use_suffix = _gf_true;
+        crd->crd_roll_key = (unsigned long) tv.tv_sec;
+
+        (void) strcpy (crd->crd_changelog_name, name);
+        (void) strcpy (crd->crd_changelog_oname, name);
+
+        /* inject a roll-over event */
+        return changelog_inject_single_event (this, priv, NULL, &cld);
+}
+
+int
+changelog_default_cops_rollover (xlator_t *this,
+                                 changelog_priv_t *priv, void *cpriv,
+                                 char *name, gf_boolean_t last)
+{
+        return changelog_default_cops_open (this, priv, cpriv, name, last);
+}
+
+int
+changelog_default_cops_sync (xlator_t *this,
+                             changelog_priv_t *priv, void *cpriv)
+{
+        changelog_log_data_t cld = {0,};
+
+        cld.cld_type = CHANGELOG_TYPE_FSYNC;
+        return changelog_inject_single_event (this, priv, NULL, &cld);
+}
+
+/**
+ * write to the changelog: @changelog_update() implements inode version
+ * checking and all other stuffs...
+ */
+int
+changelog_default_cops_write (xlator_t *this,
+                              changelog_priv_t *priv, void *cpriv,
+                              changelog_local_t *local, changelog_log_type type)
+{
+        changelog_update (this, priv, local, type);
+        return 0;
+}
+
+off_t
+changelog_default_cops_get_offset (xlator_t *this,
+                                   changelog_priv_t *priv, void *cpriv,
+                                   changelog_local_t *local)
+{
+        return *(off_t *)cpriv;
+}
+
+void
+changelog_default_cops_set_offset (xlator_t *this,
+                                   changelog_priv_t *priv, void *cpriv,
+                                   changelog_local_t *local, off_t bytes)
+{
+        *(off_t *)cpriv += bytes;
+}
+
+void
+changelog_default_cops_reset_offset (xlator_t *this, changelog_priv_t *priv,
+                                     void *cpriv, changelog_local_t *local)
+{
+        *(off_t *)cpriv = 0;
+}
+
+/**
+ * roll-over takes care of close and open
+ */
+int
+changelog_default_cops_close (xlator_t *this,
+                              changelog_priv_t *priv, void *cpriv)
+{
+        errno = ENOTSUP;
+        return -1;
+}
+
+int
+changelog_default_cops_read (xlator_t *this,
+                             changelog_priv_t *priv, void *cpriv, char *buffer)
+{
+        errno = ENOTSUP;
+        return -1;
+}
+
+/**
+ * no purging of changelogs
+ */
+int
+changelog_default_cops_unlink (xlator_t *this,
+                               changelog_priv_t *priv, void *cpriv, char *name)
+{
+        errno = ENOTSUP;
+        return -1;
+}
diff --git a/xlators/features/changelog/src/changelog-encoders.c b/xlators/features/changelog/src/changelog-encoders.c
new file mode 100644
index 000000000..ecd598e4d
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-encoders.c
@@ -0,0 +1,182 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "changelog-encoders.h"
+
+size_t
+entry_fn (void *data, char *buffer, gf_boolean_t encode)
+{
+        char    *tmpbuf = NULL;
+        size_t  bufsz  = 0;
+        struct changelog_entry_fields *ce = NULL;
+
+        ce = (struct changelog_entry_fields *) data;
+
+        if (encode) {
+                tmpbuf = uuid_utoa (ce->cef_uuid);
+                CHANGELOG_FILL_BUFFER (buffer, bufsz, tmpbuf, strlen (tmpbuf));
+        } else {
+                CHANGELOG_FILL_BUFFER (buffer, bufsz,
+                                       ce->cef_uuid, sizeof (uuid_t));
+        }
+
+        CHANGELOG_FILL_BUFFER (buffer, bufsz, "/", 1);
+        CHANGELOG_FILL_BUFFER (buffer, bufsz,
+                               ce->cef_bname, strlen (ce->cef_bname));
+        return bufsz;
+}
+
+size_t
+fop_fn (void *data, char *buffer, gf_boolean_t encode)
+{
+        char buf[10]          = {0,};
+        size_t         bufsz = 0;
+        glusterfs_fop_t fop   = 0;
+
+        fop = *(glusterfs_fop_t *) data;
+
+        if (encode) {
+                (void) snprintf (buf, sizeof (buf), "%d", fop);
+                CHANGELOG_FILL_BUFFER (buffer, bufsz, buf, strlen (buf));
+        } else
+                CHANGELOG_FILL_BUFFER (buffer, bufsz, &fop, sizeof (fop));
+
+        return bufsz;
+}
+
+void
+entry_free_fn (void *data)
+{
+        changelog_opt_t *co = data;
+
+        if (!co)
+                return;
+
+        GF_FREE (co->co_entry.cef_bname);
+}
+
+/**
+ * try to write all data in one shot
+ */
+
+static inline void
+changelog_encode_write_xtra (changelog_write_data_t *cwd,
+                             char *buffer, size_t *off, gf_boolean_t encode)
+{
+        int              i      = 0;
+        size_t           offset = 0;
+        void            *data   = NULL;
+        changelog_opt_t *co     = NULL;
+
+        offset = *off;
+
+        co = (changelog_opt_t *) cwd->cwd_ptr;
+
+        for (; i < cwd->cwd_xtra_records; i++, co++) {
+                if (i)
+                        CHANGELOG_FILL_BUFFER (buffer, offset, "\0", 1);
+
+                switch (co->co_type) {
+                case CHANGELOG_OPT_REC_FOP:
+                        data = &co->co_fop;
+                        break;
+                case CHANGELOG_OPT_REC_ENTRY:
+                        data = &co->co_entry;
+                        break;
+                case CHANGELOG_OPT_REC_NAME:
+                        data = co->co_entry.cef_bname;
+                        break;
+                case CHANGELOG_OPT_REC_ULL:
+                        data = &co->co_number;
+                        break;
+                case CHANGELOG_OPT_REC_UUID:
+                        data = &co->co_uuid;
+                        break;
+                case CHANGELOG_OPT_REC_INT32:
+                        data = &co->co_int32;
+                        break;
+                case CHANGELOG_OPT_REC_UINT32:
+                        data = &co->co_uint32;
+                        break;
+                }
+
+                if (co->co_convert)
+                        offset += co->co_convert (data,
+                                                  buffer + offset, encode);
+                else /* no coversion: write it out as it is */
+                        CHANGELOG_FILL_BUFFER (buffer, offset,
+                                               data, co->co_len);
+        }
+
+        *off = offset;
+}
+
+int
+changelog_encode_ascii (xlator_t *this,
+                        changelog_local_t *local, changelog_log_data_t *cld)
+{
+        size_t                  off      = 0;
+        size_t                  gfid_len = 0;
+        char                   *gfid_str = NULL;
+        char                   *buffer   = NULL;
+        changelog_priv_t       *priv     = NULL;
+        changelog_write_data_t *cwd      = NULL;
+
+        priv = this->private;
+        cwd = &cld->cld_wdata;
+
+        gfid_str = uuid_utoa (cwd->cwd_gfid);
+        gfid_len = strlen (gfid_str);
+
+        /* extra bytes for decorations */
+        buffer = alloca (gfid_len + cwd->cwd_ptr_len + 100);
+        if (!priv->no_gfid_hdr)
+                CHANGELOG_STORE_ASCII (priv, buffer,
+                                       off, gfid_str, gfid_len, cld);
+
+        if (cwd->cwd_xtra_records) {
+                changelog_encode_write_xtra (cwd, buffer, &off, _gf_true);
+                CHANGELOG_FILL_BUFFER (buffer, off, "\0", 1);
+        }
+
+        return changelog_write_change (this, priv,
+                                       local, buffer, off);
+}
+
+int
+changelog_encode_binary (xlator_t *this,
+                         changelog_local_t *local, changelog_log_data_t *cld)
+{
+        size_t                  off    = 0;
+        char                   *buffer = NULL;
+        changelog_priv_t       *priv   = NULL;
+        changelog_write_data_t *cwd    = NULL;
+
+        priv = this->private;
+        cwd = &cld->cld_wdata;
+
+        /* extra bytes for decorations */
+        buffer = alloca (sizeof (uuid_t) + cwd->cwd_ptr_len + 100);
+        if (!priv->no_gfid_hdr)
+                CHANGELOG_STORE_BINARY (priv, buffer, off, cwd->cwd_gfid, cld);
+
+        if (cwd->cwd_xtra_records) {
+                changelog_encode_write_xtra (cwd, buffer, &off, _gf_false);
+                CHANGELOG_FILL_BUFFER (buffer, off, "\0", 1);
+        }
+
+        return changelog_write_change (this, priv,
+                                       local, buffer, off);
+}
diff --git a/xlators/features/changelog/src/changelog-encoders.h b/xlators/features/changelog/src/changelog-encoders.h
new file mode 100644
index 000000000..2a96ba4dd
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-encoders.h
@@ -0,0 +1,48 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_ENCODERS_H
+#define _CHANGELOG_ENCODERS_H
+
+#include "xlator.h"
+#include "defaults.h"
+
+#include "changelog-helpers.h"
+
+#define CHANGELOG_STORE_ASCII(priv, buf, off, gfid, gfid_len, cld) do { \
+                CHANGELOG_FILL_BUFFER (buffer, off,                     \
+                                       priv->maps[cld->cld_type], 1);   \
+                CHANGELOG_FILL_BUFFER (buffer,                          \
+                                       off, gfid, gfid_len);            \
+                CHANGELOG_FILL_BUFFER (buffer, off, "\0", 1);           \
+        } while (0)
+
+#define CHANGELOG_STORE_BINARY(priv, buf, off, gfid, cld) do {          \
+                CHANGELOG_FILL_BUFFER (buffer, off,                     \
+                                       priv->maps[cld->cld_type], 1);   \
+                CHANGELOG_FILL_BUFFER (buffer,                          \
+                                       off, gfid, sizeof (uuid_t));     \
+                CHANGELOG_FILL_BUFFER (buffer, off, "\0", 1);           \
+        } while (0)
+
+size_t
+entry_fn (void *data, char *buffer, gf_boolean_t encode);
+size_t
+fop_fn (void *data, char *buffer, gf_boolean_t encode);
+void
+entry_free_fn (void *data);
+int
+changelog_encode_binary (xlator_t *,
+                         changelog_local_t *, changelog_log_data_t *);
+int
+changelog_encode_ascii (xlator_t *,
+                        changelog_local_t *, changelog_log_data_t *);
+
+#endif /* _CHANGELOG_ENCODERS_H */
diff --git a/xlators/features/changelog/src/changelog-fops.h b/xlators/features/changelog/src/changelog-fops.h
new file mode 100644
index 000000000..597327be3
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-fops.h
@@ -0,0 +1,157 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_FOPS_H
+#define _CHANGELOG_FOPS_H
+
+/* FOPS */
+
+int32_t
+changelog_default_rmdir (call_frame_t *frame, xlator_t *this,
+                         loc_t *loc, int xflags, dict_t *xdata);
+int32_t
+changelog_default_unlink (call_frame_t *frame, xlator_t *this,
+                          loc_t *loc, int xflags, dict_t *xdata);
+int32_t
+changelog_default_rename (call_frame_t *frame, xlator_t *this,
+                          loc_t *oldloc, loc_t *newloc, dict_t *xdata);
+int32_t
+changelog_default_link (call_frame_t *frame,
+                        xlator_t *this, loc_t *oldloc,
+                        loc_t *newloc, dict_t *xdata);
+int32_t
+changelog_default_mkdir (call_frame_t *frame, xlator_t *this,
+                         loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata);
+int32_t
+changelog_default_symlink (call_frame_t *frame, xlator_t *this,
+                           const char *linkname, loc_t *loc,
+                           mode_t umask, dict_t *xdata);
+int32_t
+changelog_default_mknod (call_frame_t *frame,
+                         xlator_t *this, loc_t *loc,
+                         mode_t mode, dev_t dev, mode_t umask, dict_t *xdata);
+int32_t
+changelog_default_create (call_frame_t *frame, xlator_t *this,
+                          loc_t *loc, int32_t flags, mode_t mode,
+                          mode_t umask, fd_t *fd, dict_t *xdata);
+int32_t
+changelog_default_fsetattr (call_frame_t *frame,
+                            xlator_t *this, fd_t *fd,
+                            struct iatt *stbuf, int32_t valid, dict_t *xdata);
+int32_t
+changelog_default_setattr (call_frame_t *frame,
+                           xlator_t *this, loc_t *loc,
+                           struct iatt *stbuf, int32_t valid, dict_t *xdata);
+int32_t
+changelog_default_fremovexattr (call_frame_t *frame, xlator_t *this,
+                                fd_t *fd, const char *name, dict_t *xdata);
+int32_t
+changelog_default_removexattr (call_frame_t *frame, xlator_t *this,
+                               loc_t *loc, const char *name, dict_t *xdata);
+int32_t
+changelog_default_setxattr (call_frame_t *frame,
+                            xlator_t *this, loc_t *loc,
+                            dict_t *dict, int32_t flags, dict_t *xdata);
+int32_t
+changelog_default_fsetxattr (call_frame_t *frame,
+                             xlator_t *this, fd_t *fd, dict_t *dict,
+                             int32_t flags, dict_t *xdata);
+int32_t
+changelog_default_truncate (call_frame_t *frame,
+                            xlator_t *this, loc_t *loc,
+                            off_t offset, dict_t *xdata);
+int32_t
+changelog_default_ftruncate (call_frame_t *frame,
+                             xlator_t *this, fd_t *fd,
+                             off_t offset, dict_t *xdata);
+int32_t
+changelog_default_writev (call_frame_t *frame,
+                          xlator_t *this, fd_t *fd, struct iovec *vector,
+                          int32_t count, off_t offset, uint32_t flags,
+                          struct iobref *iobref, dict_t *xdata);
+
+/* COPS */
+int
+changelog_default_cops_open (xlator_t *, changelog_priv_t *,
+                             void *, char*, gf_boolean_t);
+
+int
+changelog_default_cops_close (xlator_t *, changelog_priv_t *, void *);
+
+int
+changelog_default_cops_sync (xlator_t *this,
+                             changelog_priv_t *priv, void *);
+
+int
+changelog_default_cops_rollover (xlator_t *,
+                                 changelog_priv_t *, void *,
+                                 char *, gf_boolean_t);
+int
+changelog_default_cops_write (xlator_t *,
+                              changelog_priv_t *, void *,
+                              changelog_local_t *, changelog_log_type);
+
+int
+changelog_default_cops_read (xlator_t *,
+                             changelog_priv_t *, void *, char *);
+
+int
+changelog_default_cops_unlink (xlator_t *,
+                               changelog_priv_t *, void *, char *);
+
+off_t
+changelog_default_cops_get_offset (xlator_t *,
+                                   changelog_priv_t *, void *,
+                                   changelog_local_t *);
+
+void
+changelog_default_cops_set_offset (xlator_t *,
+                                   changelog_priv_t *, void *,
+                                   changelog_local_t *, off_t );
+
+void
+changelog_default_cops_reset_offset (xlator_t *, changelog_priv_t *,
+                                     void *, changelog_local_t *);
+
+
+GF_UNUSED static struct xlator_fops changelog_default_fops = {
+        .mknod        = changelog_default_mknod,
+        .mkdir        = changelog_default_mkdir,
+        .create       = changelog_default_create,
+        .symlink      = changelog_default_symlink,
+        .writev       = changelog_default_writev,
+        .truncate     = changelog_default_truncate,
+        .ftruncate    = changelog_default_ftruncate,
+        .link         = changelog_default_link,
+        .rename       = changelog_default_rename,
+        .unlink       = changelog_default_unlink,
+        .rmdir        = changelog_default_rmdir,
+        .setattr      = changelog_default_setattr,
+        .fsetattr     = changelog_default_fsetattr,
+        .setxattr     = changelog_default_setxattr,
+        .fsetxattr    = changelog_default_fsetxattr,
+        .removexattr  = changelog_default_removexattr,
+        .fremovexattr = changelog_default_fremovexattr,
+};
+
+GF_UNUSED static struct changelog_ops changelog_default_cops = {
+        .open         = changelog_default_cops_open,
+        .sync         = changelog_default_cops_sync,
+        .read         = changelog_default_cops_read,
+        .close        = changelog_default_cops_close,
+        .write        = changelog_default_cops_write,
+        .unlink       = changelog_default_cops_unlink,
+        .rollover     = changelog_default_cops_rollover,
+        .get_offset   = changelog_default_cops_get_offset,
+        .set_offset   = changelog_default_cops_set_offset,
+        .reset_offset = changelog_default_cops_reset_offset,
+};
+
+#endif /* _CHANGELOG_FOPS_H */
diff --git a/xlators/features/changelog/src/changelog-helpers.c b/xlators/features/changelog/src/changelog-helpers.c
new file mode 100644
index 000000000..ad4fe4013
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-helpers.c
@@ -0,0 +1,719 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "logging.h"
+#include "iobuf.h"
+
+#include "changelog-helpers.h"
+#include "changelog-mem-types.h"
+
+#include <pthread.h>
+
+void
+changelog_thread_cleanup (xlator_t *this, pthread_t thr_id)
+{
+        int   ret    = 0;
+        void *retval = NULL;
+
+        /* send a cancel request to the thread */
+        ret = pthread_cancel (thr_id);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "could not cancel thread (reason: %s)",
+                        strerror (errno));
+                goto out;
+        }
+
+        ret = pthread_join (thr_id, &retval);
+        if (ret || (retval != PTHREAD_CANCELED)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "cancel request not adhered as expected"
+                        " (reason: %s)", strerror (errno));
+        }
+
+ out:
+        return;
+}
+
+inline void *
+changelog_get_usable_buffer (changelog_local_t *local)
+{
+        changelog_write_data_t *cwd = &local->cld.cld_wdata;
+
+        if (!cwd->cwd_iobuf)
+                return NULL;
+
+        return cwd->cwd_ptr;
+}
+
+inline void
+changelog_set_usable_record_and_length (changelog_local_t *local,
+                                        size_t len, int xr)
+{
+        changelog_write_data_t *cwd = &local->cld.cld_wdata;
+
+        cwd->cwd_ptr_len = len;
+        cwd->cwd_xtra_records = xr;
+}
+
+void
+changelog_local_cleanup (xlator_t *xl, changelog_local_t *local)
+{
+        int                     i   = 0;
+        changelog_opt_t        *co  = NULL;
+        changelog_write_data_t *cwd = NULL;
+
+        if (!local)
+                return;
+
+        cwd = &local->cld.cld_wdata;
+
+        /* cleanup dynamic allocation for extra records */
+        if (cwd->cwd_xtra_records) {
+                co = (changelog_opt_t *) cwd->cwd_ptr;
+                for (; i < cwd->cwd_xtra_records; i++, co++)
+                        if (co->co_free)
+                                co->co_free (co);
+        }
+
+        CHANGELOG_IOBUF_UNREF (cwd->cwd_iobuf);
+
+        if (local->inode)
+                inode_unref (local->inode);
+
+        mem_put (local);
+}
+
+inline int
+changelog_write (int fd, char *buffer, size_t len)
+{
+        ssize_t size = 0;
+        size_t writen = 0;
+
+        while (writen < len) {
+                size = write (fd,
+                              buffer + writen, len - writen);
+                if (size <= 0)
+                        break;
+
+                writen += size;
+        }
+
+        return (writen != len);
+}
+
+static int
+changelog_rollover_changelog (xlator_t *this,
+                              changelog_priv_t *priv,
+                              changelog_rollover_data_t *crd)
+{
+        int   ret            = -1;
+        int   notify         = 0;
+        char *bname          = NULL;
+        char ofile[PATH_MAX] = {0,};
+        char nfile[PATH_MAX] = {0,};
+
+        if (priv->changelog_fd != -1) {
+                close (priv->changelog_fd);
+                priv->changelog_fd = -1;
+        }
+
+        /**
+         * no rolling-over of changelogs, policy implementer choose
+         * to do the heavy-lifting of having distinct changelog name.
+         *
+         * NOTE: This implies libgfchangelog would not be notified
+                 (well, we could, but lets not do that now...)
+         */
+        if (!crd->crd_use_suffix)
+                return 0;
+
+        (void) snprintf (ofile, PATH_MAX,
+                         "%s/%s", priv->changelog_dir,
+                         crd->crd_changelog_oname);
+        (void) snprintf (nfile, PATH_MAX, "%s/%s.%lu",
+                         priv->changelog_dir,
+                         crd->crd_changelog_name, crd->crd_roll_key);
+
+        ret = rename (ofile, nfile);
+        if (!ret)
+                notify = 1;
+
+        if (ret && (errno == ENOENT)) {
+                ret = 0;
+        }
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error renaming %s -> %s (reason %s)",
+                        ofile, nfile, strerror (errno));
+        }
+
+        if (notify) {
+                bname = basename (nfile);
+                gf_log (this->name, GF_LOG_DEBUG, "notifying: %s", bname);
+                ret = changelog_write (priv->wfd, bname, strlen (bname) + 1);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Failed to send file name to notify thread"
+                                " (reason: %s)", strerror (errno));
+                }
+        }
+
+        return ret;
+}
+
+int
+changelog_open (xlator_t *this,
+                changelog_priv_t *priv,
+                changelog_local_t *local, changelog_rollover_data_t *crd)
+{
+        int fd                        = 0;
+        int ret                       = -1;
+        int flags                     = 0;
+        char buffer[1024]             = {0,};
+        char changelog_path[PATH_MAX] = {0,};
+
+        (void) snprintf (changelog_path, PATH_MAX,
+                         "%s/%s", priv->changelog_dir,
+                         crd->crd_changelog_name);
+
+        flags |= (O_CREAT | O_RDWR);
+        if (priv->fsync_interval == 0)
+	        flags |= O_SYNC;
+
+        fd = open (changelog_path, flags,
+                   S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
+        if (fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "unable to open/create changelog file %s"
+                        " (reason: %s). change-logging will be"
+                        " inactive", changelog_path, strerror (errno));
+                goto out;
+        }
+
+        priv->changelog_fd = fd;
+        CHANGELOG_INVOKE_CFOP (this, priv, reset_offset, local);
+
+        /* preallocate if required */
+        if (crd->crd_prealloc_size > 0) {
+                ret = posix_fallocate (priv->changelog_fd,
+                                       0, crd->crd_prealloc_size);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to preallocate %llu bytes",
+                                (unsigned long long) crd->crd_prealloc_size);
+                }
+        }
+
+        (void) snprintf (buffer, 1024, CHANGELOG_HEADER,
+                         CHANGELOG_VERSION_MAJOR,
+                         CHANGELOG_VERSION_MINOR,
+                         priv->encode_mode);
+        ret = changelog_write_change (this, priv,
+                                      local, buffer, strlen (buffer));
+        if (ret) {
+                close (priv->changelog_fd);
+                priv->changelog_fd = -1;
+                goto out;
+        }
+
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+static int
+changelog_start_next_change (xlator_t *this,
+                             changelog_priv_t *priv,
+                             changelog_local_t *local,
+                             changelog_log_data_t *cld)
+{
+        int ret = 0;
+        changelog_rollover_data_t *crd = &cld->cld_roll;
+
+        ret = changelog_rollover_changelog (this, priv, crd);
+
+        if (!ret && !crd->crd_finale)
+                ret = changelog_open (this, priv, local, crd);
+        return ret;
+}
+
+/**
+ * return the length of entry
+ */
+inline size_t
+changelog_entry_length ()
+{
+        return sizeof (changelog_log_data_t);
+}
+
+int
+changelog_write_change (xlator_t *this, changelog_priv_t *priv,
+                        changelog_local_t *local, char *buffer, size_t len)
+{
+        int     ret    = -1;
+        off_t   offset = 0;
+        ssize_t size   = 0;
+        size_t  writen = 0;
+
+        offset = CHANGELOG_INVOKE_CFOP (this, priv, get_offset, local);
+
+        while (writen < len) {
+                size = pwrite (priv->changelog_fd,
+                               buffer + writen, len - writen, offset + writen);
+                if (size <= 0)
+                        break;
+
+                writen += size;
+        }
+
+        if (writen == len) {
+                ret = 0;
+                CHANGELOG_INVOKE_CFOP (this, priv, set_offset, local, writen);
+        }
+
+        return ret;
+}
+
+inline int
+changelog_handle_change (xlator_t *this,
+                         changelog_priv_t *priv,
+                         changelog_local_t *local, changelog_log_data_t *cld)
+{
+        int ret = 0;
+
+        if (CHANGELOG_TYPE_IS_ROLLOVER (cld->cld_type)) {
+                ret = changelog_start_next_change (this, priv, local, cld);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Problem rolling over changelog(s)");
+                goto out;
+        }
+
+        /**
+         * case when there is reconfigure done (disabling changelog) and there
+         * are still fops that have updates in prgress.
+         */
+        if (priv->changelog_fd == -1)
+                return 0;
+
+        if (CHANGELOG_TYPE_IS_FSYNC (cld->cld_type)) {
+                ret = fsync (priv->changelog_fd);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "fsync failed (reason: %s)",
+                                strerror (errno));
+                }
+                goto out;
+        }
+
+        ret = priv->ce->encode (this, local, cld);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "error writing changelog to disk");
+        }
+
+ out:
+        return ret;
+}
+
+static inline void
+changelog_local_init_defaults (changelog_local_t *local,
+                               uuid_t gfid, struct iobuf *iobuf)
+{
+        changelog_write_data_t *cwd = &(local->cld.cld_wdata);
+
+        uuid_copy (cwd->cwd_gfid, gfid);
+        cwd->cwd_iobuf        = iobuf;
+        cwd->cwd_xtra_records = 0; /* set by the caller */
+}
+
+changelog_local_t *
+changelog_local_init (xlator_t *this, inode_t *inode,
+                      uuid_t gfid, int xtra_records,
+                      gf_boolean_t update_flag)
+{
+        changelog_local_t *local = NULL;
+        struct iobuf      *iobuf = NULL;
+
+        /**
+         * Relax the presence of inode if @update_flag is true.
+         * The caller (implmentation of the fop) needs to be careful to
+         * not blindly use local->inode.
+         */
+        if (!update_flag && !inode) {
+                gf_log_callingfn (this->name, GF_LOG_WARNING,
+                                  "inode needed for version checking !!!");
+                goto out;
+        }
+
+        if (xtra_records) {
+                iobuf = iobuf_get2 (this->ctx->iobuf_pool,
+                                    xtra_records * CHANGELOG_OPT_RECORD_LEN);
+                if (!iobuf)
+                        goto out;
+        }
+
+        local = mem_get0 (this->local_pool);
+        if (!local) {
+                CHANGELOG_IOBUF_UNREF (iobuf);
+                goto out;
+        }
+
+        local->update_no_check = update_flag;
+
+        (void) changelog_local_init_defaults (local, gfid, iobuf);
+
+        if (inode)
+                local->inode = inode_ref (inode);
+
+ out:
+        return local;
+}
+
+int
+changelog_forget (xlator_t *this, inode_t *inode)
+{
+        uint64_t ctx_addr = 0;
+        changelog_inode_ctx_t *ctx = NULL;
+
+        inode_ctx_del (inode, this, &ctx_addr);
+        if (!ctx_addr)
+                return 0;
+
+        ctx = (changelog_inode_ctx_t *) (long) ctx_addr;
+        GF_FREE (ctx);
+
+        return 0;
+}
+
+int
+changelog_inject_single_event (xlator_t *this,
+                               changelog_priv_t *priv,
+                               changelog_local_t *local,
+                               changelog_log_data_t *cld)
+{
+        return priv->cd.dispatchfn (this, priv,
+                                    priv->cd.cd_data, local, cld);
+}
+
+/**
+ * TODO: these threads have many thing in common (wake up after
+ * a certain time etc..). move them into separate routine.
+ */
+void *
+changelog_rollover (void *data)
+{
+        int                     ret   = 0;
+        char                   *cname = NULL;
+        xlator_t               *this  = NULL;
+        struct timeval          tv    = {0,};
+        changelog_time_slice_t *slice = NULL;
+        changelog_priv_t       *priv  = data;
+
+        this = priv->cr.this;
+        slice = &priv->slice;
+
+        while (1) {
+                tv.tv_sec  = priv->rollover_time;
+                tv.tv_usec = 0;
+
+                ret = select (0, NULL, NULL, NULL, &tv);
+                if (ret)
+                        continue;
+
+                LOCK (&priv->lock);
+                {
+                        cname = CHANGELOG_FNAME_FROM_POLICY (priv->cp);
+                        ret = CHANGELOG_INVOKE_CFOP (this, priv, rollover,
+                                                     cname, _gf_false);
+                        if (!ret)
+                                SLICE_VERSION_UPDATE (slice);
+                }
+                UNLOCK (&priv->lock);
+        }
+
+        return NULL;
+}
+
+void *
+changelog_fsync_thread (void *data)
+{
+        int                   ret  = 0;
+        xlator_t             *this = NULL;
+        struct timeval        tv   = {0,};
+        changelog_priv_t     *priv = data;
+
+        this = priv->cf.this;
+
+        while (1) {
+                tv.tv_sec  = priv->fsync_interval;
+                tv.tv_usec = 0;
+
+                ret = select (0, NULL, NULL, NULL, &tv);
+                if (ret)
+                        continue;
+
+                ret = CHANGELOG_INVOKE_CFOP (this, priv, sync);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "failed to inject fsync event");
+        }
+
+        return NULL;
+}
+
+/* macros for inode/changelog version checks */
+
+#define INODE_VERSION_UPDATE(priv, inode, iver, slice, type) do {       \
+                LOCK (&inode->lock);                                    \
+                {                                                       \
+                        LOCK (&priv->lock);                             \
+                        {                                               \
+                                *iver = slice->changelog_version[type]; \
+                        }                                               \
+                        UNLOCK (&priv->lock);                           \
+                }                                                       \
+                UNLOCK (&inode->lock);                                  \
+        } while (0)
+
+#define INODE_VERSION_EQUALS_SLICE(priv, ver, slice, type, upd) do {    \
+                LOCK (&priv->lock);                                     \
+                {                                                       \
+                        upd = (ver == slice->changelog_version[type])   \
+                                ? _gf_false : _gf_true;                 \
+                }                                                       \
+                UNLOCK (&priv->lock);                                   \
+        } while (0)
+
+static int
+__changelog_inode_ctx_set (xlator_t *this,
+                           inode_t *inode, changelog_inode_ctx_t *ctx)
+{
+        uint64_t ctx_addr = (uint64_t) ctx;
+        return __inode_ctx_set (inode, this, &ctx_addr);
+}
+
+/**
+ * one shot routine to get the address and the value of a inode version
+ * for a particular type.
+ */
+static changelog_inode_ctx_t *
+__changelog_inode_ctx_get (xlator_t *this,
+                           inode_t *inode, unsigned long **iver,
+                           unsigned long *version, changelog_log_type type)
+{
+        int                    ret      = 0;
+        uint64_t               ctx_addr = 0;
+        changelog_inode_ctx_t *ctx      = NULL;
+
+        ret = __inode_ctx_get (inode, this, &ctx_addr);
+        if (ret < 0)
+                ctx_addr = 0;
+        if (ctx_addr != 0) {
+                ctx = (changelog_inode_ctx_t *) (long)ctx_addr;
+                goto out;
+        }
+
+        ctx = GF_CALLOC (1, sizeof (*ctx), gf_changelog_mt_inode_ctx_t);
+        if (!ctx)
+                goto out;
+
+        ret = __changelog_inode_ctx_set (this, inode, ctx);
+        if (ret) {
+                GF_FREE (ctx);
+                ctx = NULL;
+        }
+
+ out:
+        if (ctx && iver && version) {
+                *iver = CHANGELOG_INODE_VERSION_TYPE (ctx, type);
+                *version = **iver;
+        }
+
+        return ctx;
+}
+
+static changelog_inode_ctx_t *
+changelog_inode_ctx_get (xlator_t *this,
+                         inode_t *inode, unsigned long **iver,
+                         unsigned long *version, changelog_log_type type)
+{
+        changelog_inode_ctx_t *ctx = NULL;
+
+        LOCK (&inode->lock);
+        {
+                ctx = __changelog_inode_ctx_get (this,
+                                                 inode, iver, version, type);
+        }
+        UNLOCK (&inode->lock);
+
+        return ctx;
+}
+
+/**
+ * This is the main update routine. Locking has been made granular so as to
+ * maximize parallelism of fops - I'll try to explain it below using execution
+ * timelines.
+ *
+ * Basically, the contention is between multiple execution threads of this
+ * routine and the roll-over thread. So, instead of having a big lock, we hold
+ * granular locks: inode->lock and priv->lock. Now I'll explain what happens
+ * when there is an update and a roll-over at just about the same time.
+ * NOTE:
+ *  - the dispatcher itself synchronizes updates via it's own lock
+ *  - the slice version in incremented by the roll-over thread
+ *
+ * Case 1: When the rollover thread wins before the inode version can be
+ * compared with the slice version.
+ *
+ *          [updater]                 |             [rollover]
+ *                                    |
+ *                                    |           <SLICE: 1, 1, 1>
+ * <changelog_update>                 |
+ *   <changelog_inode_ctx_get>        |
+ *      <CTX: 1, 1, 1>                |
+ *                                    |         <dispatch-rollover-event>
+ *                                    |         LOCK (&priv->lock)
+ *                                    |            <SLICE_VERSION_UPDATE>
+ *                                    |              <SLICE: 2, 2, 2>
+ *                                    |         UNLOCK (&priv->lock)
+ *                                    |
+ * LOCK (&priv->lock)                 |
+ *   <INODE_VERSION_EQUALS_SLICE>     |
+ *    I: 1 <-> S: 2                   |
+ *    update: true                    |
+ * UNLOCK (&priv->lock)               |
+ *                                    |
+ * <if update == true>                |
+ *  <dispath-update-event>            |
+ *  <INODE_VERSION_UPDATE>            |
+ *   LOCK (&inode->lock)              |
+ *    LOCK (&priv->lock)              |
+ *     <CTX: 2, 1, 1>                 |
+ *    UNLOCK (&priv->lock)            |
+ *   UNLOCK (&inode->lock)            |
+ *
+ * Therefore, the change gets recorded in the next change (no lost change). If
+ * the slice version was ahead of the inode version (say I:1, S: 2), then
+ * anyway the comparison would result in a update (I: 1, S: 3).
+ *
+ * If the rollover time is too less, then there is another contention when the
+ * updater tries to bring up inode version to the slice version (this is also
+ * the case when the roll-over thread wakes up during INODE_VERSION_UPDATE.
+ *
+ *   <CTX: 1, 1, 1>                   |       <SLICE: 2, 2, 2>
+ *                                    |
+ *                                    |
+ * <dispath-update-event>             |
+ * <INODE_VERSION_UPDATE>             |
+ *  LOCK (&inode->lock)               |
+ *   LOCK (&priv->lock)               |
+ *    <CTX: 2, 1, 1>                  |
+ *   UNLOCK (&priv->lock)             |
+ *  UNLOCK (&inode->lock)             |
+ *                                    |         <dispatch-rollover-event>
+ *                                    |         LOCK (&priv->lock)
+ *                                    |            <SLICE_VERSION_UPDATE>
+ *                                    |              <SLICE: 3, 3, 3>
+ *                                    |         UNLOCK (&priv->lock)
+ *
+ *
+ * Case 2: When the fop thread wins
+ *
+ *          [updater]                 |             [rollover]
+ *                                    |
+ *                                    |           <SLICE: 1, 1, 1>
+ * <changelog_update>                 |
+ *   <changelog_inode_ctx_get>        |
+ *      <CTX: 0, 0, 0>                |
+ *                                    |
+ * LOCK (&priv->lock)                 |
+ *   <INODE_VERSION_EQUALS_SLICE>     |
+ *    I: 0 <-> S: 1                   |
+ *    update: true                    |
+ * UNLOCK (&priv->lock)               |
+ *                                    |         <dispatch-rollover-event>
+ *                                    |         LOCK (&priv->lock)
+ *                                    |            <SLICE_VERSION_UPDATE>
+ *                                    |              <SLICE: 2, 2, 2>
+ *                                    |         UNLOCK (&priv->lock)
+ * <if update == true>                |
+ *  <dispath-update-event>            |
+ *  <INODE_VERSION_UPDATE>            |
+ *   LOCK (&inode->lock)              |
+ *    LOCK (&priv->lock)              |
+ *     <CTX: 2, 0, 0>                 |
+ *    UNLOCK (&priv->lock)            |
+ *   UNLOCK (&inode->lock)            |
+ *
+ * Here again, if the inode version was equal to the slice version (I: 1, S: 1)
+ * then there is no need to record an update (as the equality of the two version
+ * signifies an update was recorded in the current time slice).
+ */
+inline void
+changelog_update (xlator_t *this,
+                  changelog_priv_t *priv,
+                  changelog_local_t *local,
+                  changelog_log_type type)
+{
+        int                     ret      = 0;
+        unsigned long          *iver     = NULL;
+        unsigned long           version  = 0;
+        inode_t                *inode    = NULL;
+        changelog_time_slice_t *slice    = NULL;
+        changelog_inode_ctx_t  *ctx      = NULL;
+        changelog_log_data_t   *cld_0    = NULL;
+        gf_boolean_t            need_upd = _gf_true;
+
+        slice = &priv->slice;
+
+        /**
+         * for fops that do not require inode version checking
+         */
+        if (local->update_no_check)
+                goto update;
+
+        inode = local->inode;
+
+        ctx = changelog_inode_ctx_get (this,
+                                       inode, &iver, &version, type);
+        if (!ctx)
+                goto update;
+
+        INODE_VERSION_EQUALS_SLICE (priv, version, slice, type, need_upd);
+
+ update:
+        if (need_upd) {
+                cld_0 = &local->cld;
+                cld_0->cld_type = type;
+
+                ret = priv->cd.dispatchfn (this, priv,
+                                           priv->cd.cd_data, local, cld_0);
+
+                /**
+                 * update after the dispatcher has successfully done
+                 * it's job.
+                 */
+                if (!local->update_no_check && iver && !ret)
+                        INODE_VERSION_UPDATE (priv, inode, iver, slice, type);
+        }
+
+        return;
+}
diff --git a/xlators/features/changelog/src/changelog-helpers.h b/xlators/features/changelog/src/changelog-helpers.h
new file mode 100644
index 000000000..e4e2dfc96
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-helpers.h
@@ -0,0 +1,578 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_HELPERS_H
+#define _CHANGELOG_HELPERS_H
+
+#include "locking.h"
+#include "timer.h"
+#include "pthread.h"
+#include "iobuf.h"
+
+#include "changelog-misc.h"
+
+/**
+ * structures representing the changelog entries
+ */
+typedef struct changelog_write_data {
+        /**
+         * sincd gfid is _always_ a necessity, it's not a part
+         * of the iobuf. by doing this we do not add any overhead
+         * for data and metadata related fops.
+         */
+        uuid_t        cwd_gfid;
+
+        /**
+         * iobufs are used for optionals records: pargfid, path,
+         * write offsets etc.. It's the fop implementers job
+         * to allocate (iobuf_get() in the fop) and get unref'ed
+         * in the callback (CHANGELOG_STACK_UNWIND).
+         */
+        struct iobuf *cwd_iobuf;
+
+        /**
+         * after allocation you can point this to the length of
+         * usable data, but make sure it does not exceed the
+         * the size of the requested iobuf.
+         */
+        size_t        cwd_iobuf_len;
+        #define cwd_ptr     cwd_iobuf->ptr
+        #define cwd_ptr_len cwd_iobuf_len
+
+        /**
+         * number of optional records
+         */
+        int cwd_xtra_records;
+} changelog_write_data_t;
+
+typedef struct changelog_rollover_data {
+        /**
+         * need a changelog reopen?
+         */
+        gf_boolean_t crd_finale;
+
+        /**
+         * changelog file name to be opened after a rollover
+         */
+        char crd_changelog_name[PATH_MAX];
+
+        /**
+         * changelog file name before rollover
+         */
+        char crd_changelog_oname[PATH_MAX];
+
+        /**
+         * use @crd_roll_key as suffix during roll-over
+         */
+        gf_boolean_t crd_use_suffix;
+
+        /**
+         * suffix used when rolling a changelog
+         */
+        unsigned long crd_roll_key;
+
+        /**
+         * preallocation? if yes, how much?
+         */
+        off_t crd_prealloc_size;
+} changelog_rollover_data_t;
+
+/**
+ * the changelog entry: structure representing the type of entry
+ * and a union encapsulating the above declared structures.
+ */
+typedef struct changelog_log_data {
+        /**
+         * type of the log data entry
+         */
+        changelog_log_type cld_type;
+
+        /**
+         * union for the type of changelog operations. @fsync() does
+         * not have a corresponding entry in this union as it just
+         * performs and @fsync() on ->changelog_fd.
+         */
+        union {
+                changelog_write_data_t    cld_wdata;
+                changelog_rollover_data_t cld_roll;
+        };
+} changelog_log_data_t;
+
+typedef struct changelog_local changelog_local_t;
+
+/**
+ * holder for dispatch function and private data
+ */
+
+typedef struct changelog_priv changelog_priv_t;
+
+typedef struct changelog_dispatcher {
+        void *cd_data;
+        int (*dispatchfn) (xlator_t *,
+                           changelog_priv_t *, void *,
+                           changelog_local_t *, changelog_log_data_t *);
+} changelog_dispatcher_t;
+
+struct changelog_bootstrap {
+        changelog_mode_t mode;
+        int (*ctor) (xlator_t *, changelog_dispatcher_t *, gf_boolean_t);
+        int (*dtor) (xlator_t *, changelog_dispatcher_t *);
+};
+
+struct changelog_encoder {
+        changelog_encoder_t encoder;
+        int (*encode) (xlator_t *,
+                       changelog_local_t *, changelog_log_data_t *);
+};
+
+struct changelog_ops {
+        /* changelog open */
+        int (*open) (xlator_t *, changelog_priv_t *,
+                     void *, char *, gf_boolean_t);
+
+        /* changelog close */
+        int (*close) (xlator_t *, changelog_priv_t *, void *);
+
+        /* changelog rollover */
+        int (*rollover) (xlator_t *,
+                         changelog_priv_t *,
+                         void  *, char *, gf_boolean_t);
+
+        int (*sync) (xlator_t *, changelog_priv_t *, void *);
+
+        /* changelog write */
+        int (*write) (xlator_t *,
+                      changelog_priv_t *, void *,
+                      changelog_local_t *, changelog_log_type);
+
+        /* changelog read */
+        int (*read) (xlator_t *,
+                     changelog_priv_t *, void *, char *);
+
+        int (*unlink) (xlator_t *,
+                       changelog_priv_t *, void *, char *);
+
+        /* {get|set} offset */
+        off_t  (*get_offset) (xlator_t *this,
+                           changelog_priv_t *, void *, changelog_local_t *);
+
+        void (*set_offset) (xlator_t *this,
+                            changelog_priv_t *, void *,
+                            changelog_local_t *, off_t);
+
+        void (*reset_offset) (xlator_t *this, changelog_priv_t *,
+                              void *, changelog_local_t *);
+};
+
+/**
+ * This structure is _filled_ by the policy init (@init_policy) routine.
+ * Default @fops and @cops are passed to the init routine, which can
+ * choose to override the file operation or changelog operation behaviour.
+ * Just by _replacing_ the function pointers, a policy can change it's
+ * file and changelog operation behaviour. Kind of inheritance...
+ */
+struct changelog_logpolicy {
+        /* current changelog name */
+        char changelog_name[PATH_MAX];
+
+        /* private data */
+        void *cpriv;
+
+        /* file ops for the policy */
+        struct xlator_fops     *fops;
+
+        /* changelog operations for the policy */
+        struct changelog_ops   *cops;
+
+        /* current active policy */
+        changelog_log_policy_t  policy;
+
+        int (*init_policy) (xlator_t *,
+                            changelog_priv_t *priv,
+                            struct changelog_logpolicy *);
+        int (*fini_policy) (xlator_t *, struct changelog_logpolicy *);
+};
+
+#define CHANGELOG_FNAME_FROM_POLICY(c) c->changelog_name
+
+#define CHANGELOG_INVOKE_FOP(priv,fop,...) priv->cp->fops->fop (__VA_ARGS__)
+
+#define CHANGELOG_INVOKE_CFOP(this,priv,fop,...)                        \
+        priv->cp->cops->fop (this, priv, priv->cp->cpriv, ##__VA_ARGS__)
+
+/* xlator private */
+
+typedef struct changelog_time_slice {
+        /**
+         * just in case we need nanosecond granularity some day.
+         * field is unused as of now (maybe we'd need it later).
+         */
+        struct timeval tv_start;
+
+        /**
+         * version of changelog file, incremented each time changes
+         * rollover.
+         */
+        unsigned long changelog_version[CHANGELOG_MAX_TYPE];
+} changelog_time_slice_t;
+
+typedef struct changelog_rollover {
+        /* rollover thread */
+        pthread_t rollover_th;
+
+        xlator_t *this;
+} changelog_rollover_t;
+
+typedef struct changelog_fsync {
+        /* fsync() thread */
+        pthread_t fsync_th;
+
+        xlator_t *this;
+} changelog_fsync_t;
+
+# define CHANGELOG_MAX_CLIENTS  5
+typedef struct changelog_notify {
+        /* reader end of the pipe */
+        int rfd;
+
+        /* notifier thread */
+        pthread_t notify_th;
+
+        /* unique socket path */
+        char sockpath[UNIX_PATH_MAX];
+
+        int socket_fd;
+
+        /**
+         * simple array of accept()'ed fds. Not scalable at all
+         * for large number of clients, but it's okay as we have
+         * a ahrd limit in this version (@CHANGELOG_MAX_CLIENTS).
+         */
+        int client_fd[CHANGELOG_MAX_CLIENTS];
+
+        xlator_t *this;
+} changelog_notify_t;
+
+struct changelog_priv {
+        gf_boolean_t active;
+
+        /**
+         * write the record header?
+         */
+        gf_boolean_t no_gfid_hdr;
+
+        gf_boolean_t lockless_update;
+
+        /* to generate unique socket file per brick */
+        char *changelog_brick;
+
+        /* logging directory */
+        char *changelog_dir;
+
+        /* one file for all changelog types */
+        int changelog_fd;
+
+        gf_lock_t lock;
+
+        /* writen end of the pipe */
+        int wfd;
+
+        /* rollover time */
+        int32_t rollover_time;
+
+        /* fsync() interval */
+        int32_t fsync_interval;
+
+        /* changelog type maps */
+        const char *maps[CHANGELOG_MAX_TYPE];
+
+        /* time slicer */
+        changelog_time_slice_t slice;
+
+        /* context of the updater */
+        changelog_dispatcher_t cd;
+
+        /* context of the rollover thread */
+        changelog_rollover_t cr;
+
+        /* context of fsync thread */
+        changelog_fsync_t cf;
+
+        /* context of the notifier thread */
+        changelog_notify_t cn;
+
+        /* operation mode */
+        changelog_mode_t op_mode;
+
+        /* bootstrap routine for 'current' logger */
+        struct changelog_bootstrap *cb;
+
+        /* encoder mode */
+        changelog_encoder_t encode_mode;
+
+        /* encoder */
+        struct changelog_encoder *ce;
+
+        /* logging policy */
+        changelog_log_policy_t policy;
+
+        /* policy logger */
+        struct changelog_logpolicy *cp;
+
+        /* current NSR term */
+        uint32_t term;
+};
+
+struct changelog_local {
+        inode_t              *inode;
+
+        /**
+         * fops that do not need inode version checks
+         */
+        gf_boolean_t          update_no_check;
+
+        /**
+         * the log data entry
+         */
+        changelog_log_data_t  cld;
+
+        /**
+         * number of bytes written: used for continuation
+         */
+        off_t nr_bytes;
+
+        /**
+         * temporary scratch pads
+         */
+        union {
+                void          *ptr;
+                unsigned long  val;
+        } lu;
+};
+
+/* inode version is stored in inode ctx */
+typedef struct changelog_inode_ctx {
+        unsigned long iversion[CHANGELOG_MAX_TYPE];
+} changelog_inode_ctx_t;
+
+#define CHANGELOG_INODE_VERSION_TYPE(ctx, type)  &(ctx->iversion[type])
+
+/**
+ * Optional Records:
+ *  fops that need to save additional information request a array of
+ *  @changelog_opt_t struct. The array is allocated via @iobufs.
+ */
+typedef enum {
+        CHANGELOG_OPT_REC_FOP,
+        CHANGELOG_OPT_REC_ULL,
+        CHANGELOG_OPT_REC_UUID,
+        CHANGELOG_OPT_REC_NAME,
+        CHANGELOG_OPT_REC_ENTRY,
+        CHANGELOG_OPT_REC_INT32,
+        CHANGELOG_OPT_REC_UINT32,
+} changelog_optional_rec_type_t;
+
+struct changelog_entry_fields {
+        uuid_t  cef_uuid;
+        char   *cef_bname;
+};
+
+typedef struct {
+        /**
+         * @co_covert can be used to do post-processing of the record before
+         * it's persisted to the CHANGELOG. If this is NULL, then the record
+         * is persisted as per it's in memory format.
+         */
+        size_t (*co_convert) (void *data, char *buffer, gf_boolean_t encode);
+
+        /* release routines */
+        void (*co_free) (void *data);
+
+        /* type of the field */
+        changelog_optional_rec_type_t co_type;
+
+        /**
+         * sizeof of the 'valid' field in the union. This field is not used if
+         * @co_convert is specified.
+         */
+        size_t co_len;
+
+        union {
+                uuid_t                        co_uuid;
+                glusterfs_fop_t               co_fop;
+                int                           co_int32;
+                unsigned int                  co_uint32;
+                unsigned long long            co_number;
+                struct changelog_entry_fields co_entry;
+        };
+} changelog_opt_t;
+
+#define CHANGELOG_OPT_RECORD_LEN  sizeof (changelog_opt_t)
+
+/**
+ * helpers routines
+ */
+
+void
+changelog_thread_cleanup (xlator_t *this, pthread_t thr_id);
+
+void *
+changelog_get_usable_buffer (changelog_local_t *local);
+
+void
+changelog_set_usable_record_and_length (changelog_local_t *local,
+                                        size_t len, int xr);
+void
+changelog_local_cleanup (xlator_t *xl, changelog_local_t *local);
+changelog_local_t *
+changelog_local_init (xlator_t *this, inode_t *inode, uuid_t gfid,
+                      int xtra_records, gf_boolean_t update_flag);
+int
+changelog_inject_single_event (xlator_t *this,
+                               changelog_priv_t *priv,
+                               changelog_local_t *local,
+                               changelog_log_data_t *cld);
+size_t
+changelog_entry_length ();
+int
+changelog_write (int fd, char *buffer, size_t len);
+int
+changelog_write_change (xlator_t *this, changelog_priv_t *priv,
+                        changelog_local_t *local, char *buffer, size_t len);
+inline int
+changelog_handle_change (xlator_t *this,
+                         changelog_priv_t *priv,
+                         changelog_local_t *local, changelog_log_data_t *cld);
+inline void
+changelog_update (xlator_t *this,
+                  changelog_priv_t *priv,
+                  changelog_local_t *local,
+                  changelog_log_type type);
+void *
+changelog_rollover (void *data);
+void *
+changelog_fsync_thread (void *data);
+int
+changelog_forget (xlator_t *this, inode_t *inode);
+
+/* macros */
+
+#define CHANGELOG_STACK_UNWIND(fop, frame, params ...) do {             \
+                changelog_local_t *__local = NULL;                      \
+                xlator_t          *__xl    = NULL;                      \
+                if (frame) {                                            \
+                        __local      = frame->local;                    \
+                        __xl         = frame->this;                     \
+                        frame->local = NULL;                            \
+                }                                                       \
+                STACK_UNWIND_STRICT (fop, frame, params);               \
+                changelog_local_cleanup (__xl, __local);                \
+        } while (0)
+
+#define CHANGELOG_IOBUF_REF(iobuf) do {         \
+                if (iobuf)                      \
+                        iobuf_ref (iobuf);      \
+        } while (0)
+
+#define CHANGELOG_IOBUF_UNREF(iobuf) do {       \
+                if (iobuf)                      \
+                        iobuf_unref (iobuf);    \
+        } while (0)
+
+#define CHANGELOG_FILL_BUFFER(buffer, off, val, len) do {       \
+                memcpy (buffer + off, val, len);                \
+                off += len;                                     \
+        } while (0)
+
+#define SLICE_VERSION_UPDATE(slice) do {                \
+                int i = 0;                              \
+                for (; i < CHANGELOG_MAX_TYPE; i++) {   \
+                        slice->changelog_version[i]++;  \
+                }                                       \
+        } while (0)
+
+#define CHANGELOG_FILL_INT32(co, number, converter, xlen) do {  \
+                co->co_convert = converter;                     \
+                co->co_free = NULL;                             \
+                co->co_type = CHANGELOG_OPT_REC_INT32;          \
+                co->co_int32 = number;                          \
+                xlen += sizeof (int);                           \
+        } while (0)
+
+#define CHANGELOG_FILL_UINT32(co, number, converter, xlen) do { \
+                co->co_convert = converter;                     \
+                co->co_free = NULL;                             \
+                co->co_type = CHANGELOG_OPT_REC_UINT32;         \
+                co->co_uint32 = number;                         \
+                xlen += sizeof (unsigned int);                  \
+        } while (0)
+
+#define CHANGELOG_FILL_FOP_NUMBER(co, fop, converter, xlen) do { \
+                co->co_convert = converter;                      \
+                co->co_free = NULL;                              \
+                co->co_type = CHANGELOG_OPT_REC_FOP;             \
+                co->co_fop = fop;                                \
+                xlen += sizeof (fop);                            \
+        } while (0)
+
+#define CHANGELOG_FILL_NAME(co, name, freefn, xlen, label)       \
+        do {                                                     \
+                co->co_convert = NULL;                           \
+                co->co_free = freefn;                            \
+                co->co_type = CHANGELOG_OPT_REC_NAME;            \
+                co->co_entry.cef_bname = gf_strdup(name);        \
+                if (!co->co_entry.cef_bname)                     \
+                        goto label;                              \
+                co->co_len = strlen (name);                      \
+                xlen += co->co_len;                              \
+        } while(0)                                               \
+
+#define CHANGELOG_FILL_ENTRY(co, pargfid, bname,                        \
+                             converter, freefn, xlen, label)            \
+        do {                                                            \
+                co->co_convert = converter;                             \
+                co->co_free = freefn;                                   \
+                co->co_type = CHANGELOG_OPT_REC_ENTRY;                  \
+                uuid_copy (co->co_entry.cef_uuid, pargfid);             \
+                co->co_entry.cef_bname = gf_strdup(bname);              \
+                if (!co->co_entry.cef_bname)                            \
+                        goto label;                                     \
+                xlen += (UUID_CANONICAL_FORM_LEN + strlen (bname));     \
+        } while (0)
+
+#define CHANGELOG_INIT(this, local, inode, gfid, xrec)                  \
+        local = changelog_local_init (this, inode, gfid, xrec, _gf_false)
+
+#define CHANGELOG_INIT_NOCHECK(this, local, inode, gfid, xrec)          \
+        local = changelog_local_init (this, inode, gfid, xrec, _gf_true)
+
+#define CHANGELOG_NOT_ACTIVE_THEN_GOTO(frame, priv, label) do { \
+                if (!priv->active)                              \
+                        goto label;                             \
+                /* ignore rebalance process's activity. */      \
+                if (frame->root->pid == GF_CLIENT_PID_DEFRAG)   \
+                        goto label;                             \
+        } while (0)
+
+/* ignore internal fops */
+#define CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO(dict, label) do {           \
+                if (dict && dict_get (dict, GLUSTERFS_INTERNAL_FOP_KEY)) \
+                        goto label;                                     \
+        } while (0)
+
+#define CHANGELOG_COND_GOTO(priv, cond, label) do {                    \
+                if (!priv->active || cond)                             \
+                        goto label;                                    \
+        } while (0)
+
+int
+changelog_open (xlator_t *this, changelog_priv_t *priv, changelog_local_t *local, changelog_rollover_data_t *crd);
+
+#endif /* _CHANGELOG_HELPERS_H */
diff --git a/xlators/features/changelog/src/changelog-mem-types.h b/xlators/features/changelog/src/changelog-mem-types.h
new file mode 100644
index 000000000..a65bbb4f2
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-mem-types.h
@@ -0,0 +1,30 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_MEM_TYPES_H
+#define _CHANGELOG_MEM_TYPES_H
+
+#include "mem-types.h"
+
+enum gf_changelog_mem_types {
+        gf_changelog_mt_priv_t                  = gf_common_mt_end + 1,
+        gf_changelog_mt_str_t                   = gf_common_mt_end + 2,
+        gf_changelog_mt_batch_t                 = gf_common_mt_end + 3,
+        gf_changelog_mt_rt_t                    = gf_common_mt_end + 4,
+        gf_changelog_mt_inode_ctx_t             = gf_common_mt_end + 5,
+        gf_changelog_mt_fop_policy_t            = gf_common_mt_end + 6,
+        gf_changelog_mt_libgfchangelog_t        = gf_common_mt_end + 7,
+        gf_changelog_mt_libgfchangelog_rl_t     = gf_common_mt_end + 8,
+        gf_changelog_mt_libgfchangelog_dirent_t = gf_common_mt_end + 9,
+        gf_changelog_mt_changelog_buffer_t      = gf_common_mt_end + 10,
+        gf_changelog_mt_end
+};
+
+#endif
diff --git a/xlators/features/changelog/src/changelog-misc.h b/xlators/features/changelog/src/changelog-misc.h
new file mode 100644
index 000000000..58bd3279d
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-misc.h
@@ -0,0 +1,107 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_MISC_H
+#define _CHANGELOG_MISC_H
+
+#include "glusterfs.h"
+#include "common-utils.h"
+
+#define CHANGELOG_MAX_TYPE  3
+#define CHANGELOG_FILE_NAME "CHANGELOG"
+
+#define CHANGELOG_VERSION_MAJOR  1
+#define CHANGELOG_VERSION_MINOR  0
+
+#define CHANGELOG_UNIX_SOCK  DEFAULT_VAR_RUN_DIRECTORY"/changelog-%s.sock"
+
+/**
+ * header starts with the version and the format of the changelog.
+ * 'version' not much of a use now.
+ */
+#define CHANGELOG_HEADER                                                \
+        "GlusterFS Changelog | version: v%d.%d | encoding : %d\n"
+
+#define CHANGELOG_MAKE_SOCKET_PATH(brick_path, sockpath, len) do {      \
+                char md5_sum[MD5_DIGEST_LENGTH*2+1] = {0,};             \
+                md5_wrapper((unsigned char *) brick_path,               \
+                            strlen(brick_path),                         \
+                            md5_sum);                                   \
+                (void) snprintf (sockpath, len,                         \
+                                 CHANGELOG_UNIX_SOCK, md5_sum);         \
+        } while (0)
+
+/**
+ * ... used by libgfchangelog.
+ */
+#define CHANGELOG_GET_ENCODING(fd, buffer, len, enc, enc_len) do {      \
+                FILE *fp;                                               \
+                int fd_dup, maj, min;                                   \
+                                                                        \
+                enc = -1;                                               \
+                fd_dup = dup (fd);                                      \
+                                                                        \
+                if (fd_dup != -1) {                                     \
+                        fp = fdopen (fd_dup, "r");                      \
+                        if (fp) {                                       \
+                                if (fgets (buffer, len, fp)) {          \
+                                        elen = strlen (buffer);         \
+                                        sscanf (buffer,                 \
+                                                CHANGELOG_HEADER,       \
+                                                &maj, &min, &enc);      \
+                                }                                       \
+                                fclose (fp);                            \
+                        } else {                                        \
+                                close (fd_dup);                         \
+                        }                                               \
+                }                                                       \
+        } while (0)
+
+/**
+ * everything after @CHANGELOG_TYPE_ENTRY are internal types
+ * (ie. none of the fops trigger this type of event), hence
+ * CHANGELOG_MAX_TYPE = 3
+ */
+typedef enum {
+        CHANGELOG_TYPE_DATA = 0,
+        CHANGELOG_TYPE_METADATA,
+        CHANGELOG_TYPE_ENTRY,
+        CHANGELOG_TYPE_ROLLOVER,
+        CHANGELOG_TYPE_FSYNC,
+} changelog_log_type;
+
+/* operation modes - RT for now */
+typedef enum {
+        CHANGELOG_MODE_RT = 0,
+} changelog_mode_t;
+
+/* encoder types */
+
+typedef enum {
+        CHANGELOG_ENCODE_MIN = 0,
+        CHANGELOG_ENCODE_BINARY,
+        CHANGELOG_ENCODE_ASCII,
+        CHANGELOG_ENCODE_MAX,
+} changelog_encoder_t;
+
+/* logging policies */
+typedef enum {
+        CHANGELOG_LOG_POLICY_DEFAULT = 0,
+        CHANGELOG_LOG_POLICY_REPLICATE,
+} changelog_log_policy_t;
+
+#define CHANGELOG_VALID_ENCODING(enc)                                   \
+        (enc > CHANGELOG_ENCODE_MIN && enc < CHANGELOG_ENCODE_MAX)
+
+#define CHANGELOG_TYPE_IS_ENTRY(type)  (type == CHANGELOG_TYPE_ENTRY)
+#define CHANGELOG_TYPE_IS_ROLLOVER(type)  (type == CHANGELOG_TYPE_ROLLOVER)
+#define CHANGELOG_TYPE_IS_FSYNC(type)  (type == CHANGELOG_TYPE_FSYNC)
+
+#endif /* _CHANGELOG_MISC_H */
diff --git a/xlators/features/changelog/src/changelog-notifier.c b/xlators/features/changelog/src/changelog-notifier.c
new file mode 100644
index 000000000..5f3d063a8
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-notifier.c
@@ -0,0 +1,314 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "changelog-notifier.h"
+
+#include <pthread.h>
+
+inline static void
+changelog_notify_clear_fd (changelog_notify_t *cn, int i)
+{
+        cn->client_fd[i] = -1;
+}
+
+inline static void
+changelog_notify_save_fd (changelog_notify_t *cn, int i, int fd)
+{
+        cn->client_fd[i] = fd;
+}
+
+static int
+changelog_notify_insert_fd (xlator_t *this, changelog_notify_t *cn, int fd)
+{
+        int i   = 0;
+        int ret = 0;
+
+        for (; i < CHANGELOG_MAX_CLIENTS; i++) {
+                if (cn->client_fd[i] == -1)
+                        break;
+        }
+
+        if (i == CHANGELOG_MAX_CLIENTS) {
+                /**
+                 * this case should not be hit as listen() would limit
+                 * the number of completely established connections.
+                 */
+                gf_log (this->name, GF_LOG_WARNING,
+                        "hit max client limit (%d)", CHANGELOG_MAX_CLIENTS);
+                ret = -1;
+        }
+        else
+                changelog_notify_save_fd (cn, i, fd);
+
+        return ret;
+}
+
+static void
+changelog_notify_fill_rset (changelog_notify_t *cn, fd_set *rset, int *maxfd)
+{
+        int i = 0;
+
+        FD_ZERO (rset);
+
+        FD_SET (cn->socket_fd, rset);
+        *maxfd = cn->socket_fd;
+
+        FD_SET (cn->rfd, rset);
+        *maxfd = max (*maxfd, cn->rfd);
+
+        for (; i < CHANGELOG_MAX_CLIENTS; i++) {
+                if (cn->client_fd[i] != -1) {
+                        FD_SET (cn->client_fd[i], rset);
+                        *maxfd = max (*maxfd, cn->client_fd[i]);
+                }
+        }
+
+        *maxfd = *maxfd + 1;
+}
+
+static int
+changelog_notify_client (changelog_notify_t *cn, char *path, ssize_t len)
+{
+        int i = 0;
+        int ret = 0;
+
+        for (; i < CHANGELOG_MAX_CLIENTS; i++) {
+                if (cn->client_fd[i] == -1)
+                        continue;
+
+                if (changelog_write (cn->client_fd[i],
+                                     path, len)) {
+                        ret = -1;
+
+                        close (cn->client_fd[i]);
+                        changelog_notify_clear_fd (cn, i);
+                }
+        }
+
+        return ret;
+}
+
+static void
+changelog_notifier_init (changelog_notify_t *cn)
+{
+        int i = 0;
+
+        cn->socket_fd = -1;
+
+        for (; i < CHANGELOG_MAX_CLIENTS; i++) {
+                changelog_notify_clear_fd (cn, i);
+        }
+}
+
+static void
+changelog_close_client_conn (changelog_notify_t *cn)
+{
+        int i = 0;
+
+        for (; i < CHANGELOG_MAX_CLIENTS; i++) {
+                if (cn->client_fd[i] == -1)
+                        continue;
+
+                close (cn->client_fd[i]);
+                changelog_notify_clear_fd (cn, i);
+        }
+}
+
+static void
+changelog_notifier_cleanup (void *arg)
+{
+        changelog_notify_t *cn = NULL;
+
+        cn = (changelog_notify_t *) arg;
+
+        changelog_close_client_conn (cn);
+
+        if (cn->socket_fd != -1)
+                close (cn->socket_fd);
+
+        if (cn->rfd)
+                close (cn->rfd);
+
+        if (unlink (cn->sockpath))
+                gf_log ("", GF_LOG_WARNING,
+                        "could not unlink changelog socket file"
+                        " %s (reason: %s", cn->sockpath, strerror (errno));
+}
+
+void *
+changelog_notifier (void *data)
+{
+        int                 i         = 0;
+        int                 fd        = 0;
+        int                 max_fd    = 0;
+        int                 len       = 0;
+        ssize_t             readlen   = 0;
+        xlator_t           *this      = NULL;
+        changelog_priv_t   *priv      = NULL;
+        changelog_notify_t *cn        = NULL;
+        struct sockaddr_un  local     = {0,};
+        char path[PATH_MAX]           = {0,};
+        char abspath[PATH_MAX]        = {0,};
+
+        char buffer;
+        fd_set rset;
+
+        priv = (changelog_priv_t *) data;
+
+        cn = &priv->cn;
+        this = cn->this;
+
+        pthread_cleanup_push (changelog_notifier_cleanup, cn);
+
+        changelog_notifier_init (cn);
+
+        cn->socket_fd = socket (AF_UNIX, SOCK_STREAM, 0);
+        if (cn->socket_fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "changelog socket error (reason: %s)",
+                        strerror (errno));
+                goto out;
+        }
+
+        CHANGELOG_MAKE_SOCKET_PATH (priv->changelog_brick,
+                                    cn->sockpath, UNIX_PATH_MAX);
+        if (unlink (cn->sockpath) < 0) {
+                if (errno != ENOENT) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Could not unlink changelog socket file (%s)"
+                                " (reason: %s)",
+                                CHANGELOG_UNIX_SOCK, strerror (errno));
+                        goto cleanup;
+                }
+        }
+
+        local.sun_family = AF_UNIX;
+        strcpy (local.sun_path, cn->sockpath);
+
+        len = strlen (local.sun_path) + sizeof (local.sun_family);
+
+        /* bind to the unix domain socket */
+        if (bind (cn->socket_fd, (struct sockaddr *) &local, len) < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Could not bind to changelog socket (reason: %s)",
+                        strerror (errno));
+                goto cleanup;
+        }
+
+        /* listen for incoming connections */
+        if (listen (cn->socket_fd, CHANGELOG_MAX_CLIENTS) < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "listen() error on changelog socket (reason: %s)",
+                        strerror (errno));
+                goto cleanup;
+        }
+
+        /**
+         * simple select() on all to-be-read file descriptors. This method
+         * though old school works pretty well when you have a handfull of
+         * fd's to be watched (clients).
+         *
+         * Future TODO: move this to epoll based notification facility if
+         *              number of clients increase.
+         */
+        for (;;) {
+                changelog_notify_fill_rset (cn, &rset, &max_fd);
+
+                if (select (max_fd, &rset, NULL, NULL, NULL) < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "select() returned -1 (reason: %s)",
+                                strerror (errno));
+                        sleep (2);
+                        continue;
+                }
+
+                if (FD_ISSET (cn->socket_fd, &rset)) {
+                        fd = accept (cn->socket_fd, NULL, NULL);
+                        if (fd < 0) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "accept error on changelog socket"
+                                        " (reason: %s)", strerror (errno));
+                        } else if (changelog_notify_insert_fd (this, cn, fd)) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "hit max client limit");
+                        }
+                }
+
+                if (FD_ISSET (cn->rfd, &rset)) {
+                        /**
+                         * read changelog filename and notify all connected
+                         * clients.
+                         */
+                        readlen = 0;
+                        while (readlen < PATH_MAX) {
+                                len = read (cn->rfd, &path[readlen++], 1);
+                                if (len == -1) {
+                                        break;
+                                }
+
+                                if (len == 0) {
+                                        gf_log (this->name, GF_LOG_ERROR,
+                                                "rollover thread sent EOF"
+                                                " on pipe - possibly a crash.");
+                                        /* be blunt and close all connections */
+                                        pthread_exit(NULL);
+                                }
+
+                                if (path[readlen - 1] == '\0')
+                                        break;
+                        }
+
+                        /* should we close all client connections here too? */
+                        if (len < 0 || readlen == PATH_MAX) {
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "Could not get pathname from rollover"
+                                        " thread or pathname too long");
+                                goto process_rest;
+                        }
+
+                        (void) snprintf (abspath, PATH_MAX,
+                                         "%s/%s", priv->changelog_dir, path);
+                        if (changelog_notify_client (cn, abspath,
+                                                     strlen (abspath) + 1))
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "could not notify some clients with new"
+                                        " changelogs");
+                }
+
+        process_rest:
+                for (i = 0; i < CHANGELOG_MAX_CLIENTS; i++) {
+                        if ( (fd = cn->client_fd[i]) == -1 )
+                                continue;
+
+                        if (FD_ISSET (fd, &rset)) {
+                                /**
+                                 * the only data we accept from the client is a
+                                 * disconnect. Anything else is treated as bogus
+                                 * and is silently discarded (also warned!!!).
+                                 */
+                                if ( (readlen = read (fd, &buffer, 1)) <= 0 ) {
+                                        close (fd);
+                                        changelog_notify_clear_fd (cn, i);
+                                } else {
+                                        /* silently discard data and log */
+                                        gf_log (this->name, GF_LOG_WARNING,
+                                                "misbehaving changelog client");
+                                }
+                        }
+                }
+
+        }
+
+ cleanup:;
+        pthread_cleanup_pop (1);
+
+ out:
+        return NULL;
+}
diff --git a/xlators/features/changelog/src/changelog-notifier.h b/xlators/features/changelog/src/changelog-notifier.h
new file mode 100644
index 000000000..55e728356
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-notifier.h
@@ -0,0 +1,19 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_NOTIFIER_H
+#define _CHANGELOG_NOTIFIER_H
+
+#include "changelog-helpers.h"
+
+void *
+changelog_notifier (void *data);
+
+#endif
diff --git a/xlators/features/changelog/src/changelog-rt.c b/xlators/features/changelog/src/changelog-rt.c
new file mode 100644
index 000000000..91d47e059
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-rt.c
@@ -0,0 +1,83 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "logging.h"
+
+#include "changelog-rt.h"
+#include "changelog-mem-types.h"
+
+int
+changelog_rt_init (xlator_t *this,
+                   changelog_dispatcher_t *cd, gf_boolean_t lockless_update)
+{
+        changelog_rt_t *crt = NULL;
+
+        crt = GF_CALLOC (1, sizeof (*crt),
+                         gf_changelog_mt_rt_t);
+        if (!crt)
+                return -1;
+
+        /* TBD: don't init (and destroy) if lock-less update */
+        LOCK_INIT (&crt->lock);
+
+        cd->cd_data = crt;
+        cd->dispatchfn = lockless_update ?
+                &changelog_rt_enqueue_lockless : &changelog_rt_enqueue;
+
+        return 0;
+}
+
+int
+changelog_rt_fini (xlator_t *this, changelog_dispatcher_t *cd)
+{
+        changelog_rt_t *crt = NULL;
+
+        crt = cd->cd_data;
+
+        LOCK_DESTROY (&crt->lock);
+        GF_FREE (crt);
+
+        return 0;
+}
+
+int
+changelog_rt_enqueue (xlator_t *this,
+                      changelog_priv_t *priv, void *cbatch,
+                      changelog_local_t *local, changelog_log_data_t *cld_0)
+{
+        int             ret = 0;
+        changelog_rt_t *crt = NULL;
+
+        crt = (changelog_rt_t *) cbatch;
+
+        LOCK (&crt->lock);
+        {
+                ret = changelog_handle_change (this, priv, local, cld_0);
+        }
+        UNLOCK (&crt->lock);
+
+        return ret;
+}
+
+int
+changelog_rt_enqueue_lockless (xlator_t *this,
+                               changelog_priv_t *priv, void *cbatch,
+                               changelog_local_t *local,
+                               changelog_log_data_t *cld_0)
+{
+        return changelog_handle_change (this, priv, local, cld_0);
+}
diff --git a/xlators/features/changelog/src/changelog-rt.h b/xlators/features/changelog/src/changelog-rt.h
new file mode 100644
index 000000000..634b7473b
--- /dev/null
+++ b/xlators/features/changelog/src/changelog-rt.h
@@ -0,0 +1,40 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_RT_H
+#define _CHANGELOG_RT_H
+
+#include "locking.h"
+#include "timer.h"
+#include "pthread.h"
+
+#include "changelog-helpers.h"
+
+/* unused as of now - may be you would need it later */
+typedef struct changelog_rt {
+        gf_lock_t lock;
+} changelog_rt_t;
+
+int
+changelog_rt_init (xlator_t *this,
+                   changelog_dispatcher_t *cd, gf_boolean_t lockless_update);
+int
+changelog_rt_fini (xlator_t *this, changelog_dispatcher_t *cd);
+int
+changelog_rt_enqueue (xlator_t *this,
+                      changelog_priv_t *priv, void *cbatch,
+                      changelog_local_t *local, changelog_log_data_t *cld_0);
+int
+changelog_rt_enqueue_lockless (xlator_t *this,
+                               changelog_priv_t *priv, void *cbatch,
+                               changelog_local_t *local,
+                               changelog_log_data_t *cld_0);
+
+#endif /* _CHANGELOG_RT_H */
diff --git a/xlators/features/changelog/src/changelog.c b/xlators/features/changelog/src/changelog.c
new file mode 100644
index 000000000..6d4b502de
--- /dev/null
+++ b/xlators/features/changelog/src/changelog.c
@@ -0,0 +1,1389 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "logging.h"
+#include "iobuf.h"
+#include <pthread.h>
+
+#include "changelog-rt.h"
+#include "changelog-notifier.h"
+#include "changelog-encoders.h"
+#include "changelog-mem-types.h"
+
+#include "changelog-fops.h"
+#include "changelog-policy.h"
+
+static struct changelog_bootstrap
+cb_bootstrap[] = {
+        {
+                .mode = CHANGELOG_MODE_RT,
+                .ctor = changelog_rt_init,
+                .dtor = changelog_rt_fini,
+        },
+};
+
+static struct changelog_encoder
+cb_encoder[] = {
+        [CHANGELOG_ENCODE_BINARY] =
+        {
+                .encoder = CHANGELOG_ENCODE_BINARY,
+                .encode = changelog_encode_binary,
+        },
+        [CHANGELOG_ENCODE_ASCII] =
+        {
+                .encoder = CHANGELOG_ENCODE_ASCII,
+                .encode = changelog_encode_ascii,
+        },
+};
+
+static struct changelog_logpolicy
+cb_policy[] = {
+        [CHANGELOG_LOG_POLICY_DEFAULT] =
+        {
+                .fops = NULL,
+                .cops = NULL,
+                .policy = CHANGELOG_LOG_POLICY_DEFAULT,
+                .init_policy = changelog_default_policy_init,
+                .fini_policy = changelog_default_policy_fini,
+        },
+        [CHANGELOG_LOG_POLICY_REPLICATE] =
+        {
+                .fops = NULL,
+                .cops = NULL,
+                .policy = CHANGELOG_LOG_POLICY_REPLICATE,
+                .init_policy = changelog_replication_policy_init,
+                .fini_policy = changelog_replication_policy_fini,
+        },
+};
+
+/* Entry operations - TYPE III */
+
+/* {{{ */
+
+/* rmdir */
+
+int32_t
+changelog_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        CHANGELOG_INVOKE_CFOP (this, priv,
+                               write, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (rmdir, frame, op_ret, op_errno,
+                                preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_rmdir (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, int xflags, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INVOKE_FOP (priv, rmdir, frame, this, loc, xflags, xdata);
+
+ wind:
+        STACK_WIND (frame, changelog_rmdir_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->rmdir,
+                    loc, xflags, xdata);
+        return 0;
+}
+
+/* unlink */
+
+int32_t
+changelog_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        CHANGELOG_INVOKE_CFOP (this, priv,
+                               write, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (unlink, frame, op_ret, op_errno,
+                                preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_unlink (call_frame_t *frame, xlator_t *this,
+                  loc_t *loc, int xflags, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (xdata, wind);
+
+        CHANGELOG_INVOKE_FOP (priv, unlink, frame, this, loc, xflags, xdata);
+
+ wind:
+        STACK_WIND (frame, changelog_unlink_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->unlink,
+                    loc, xflags, xdata);
+        return 0;
+}
+
+/* rename */
+
+int32_t
+changelog_rename_cbk (call_frame_t *frame,
+                      void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno,
+                      struct iatt *buf, struct iatt *preoldparent,
+                      struct iatt *postoldparent, struct iatt *prenewparent,
+                      struct iatt *postnewparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        CHANGELOG_INVOKE_CFOP (this, priv,
+                               write, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (rename, frame, op_ret, op_errno,
+                                buf, preoldparent, postoldparent,
+                                prenewparent, postnewparent, xdata);
+        return 0;
+}
+
+
+int32_t
+changelog_rename (call_frame_t *frame, xlator_t *this,
+                  loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INVOKE_FOP (priv, rename, frame, this, oldloc, newloc, xdata);
+
+ wind:
+        STACK_WIND (frame, changelog_rename_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->rename,
+                    oldloc, newloc, xdata);
+        return 0;
+}
+
+/* link */
+
+int32_t
+changelog_link_cbk (call_frame_t *frame,
+                    void *cookie, xlator_t *this, int32_t op_ret,
+                    int32_t op_errno, inode_t *inode,
+                    struct iatt *buf, struct iatt *preparent,
+                    struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        CHANGELOG_INVOKE_CFOP (this, priv,
+                               write, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (link, frame, op_ret, op_errno,
+                                inode, buf, preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_link (call_frame_t *frame,
+                xlator_t *this, loc_t *oldloc,
+                loc_t *newloc, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+        CHANGELOG_IF_INTERNAL_FOP_THEN_GOTO (xdata, wind);
+
+        CHANGELOG_INVOKE_FOP (priv, link, frame, this, oldloc, newloc, xdata);
+
+ wind:
+        STACK_WIND (frame, changelog_link_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->link,
+                    oldloc, newloc, xdata);
+        return 0;
+}
+
+/* mkdir */
+
+int32_t
+changelog_mkdir_cbk (call_frame_t *frame,
+                     void *cookie, xlator_t *this, int32_t op_ret,
+                     int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        CHANGELOG_INVOKE_CFOP (this, priv,
+                               write, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (mkdir, frame, op_ret, op_errno,
+                                inode, buf, preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_mkdir (call_frame_t *frame, xlator_t *this,
+                 loc_t *loc, mode_t mode, mode_t umask, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INVOKE_FOP (priv, mkdir, frame, this,
+                              loc, mode, umask, xdata);
+
+ wind:
+        STACK_WIND (frame, changelog_mkdir_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->mkdir,
+                    loc, mode, umask, xdata);
+        return 0;
+}
+
+/* symlink */
+
+int32_t
+changelog_symlink_cbk (call_frame_t *frame,
+                       void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno,
+                       inode_t *inode, struct iatt *buf, struct iatt *preparent,
+                       struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        CHANGELOG_INVOKE_CFOP (this, priv,
+                               write, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (symlink, frame, op_ret, op_errno,
+                                inode, buf, preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_symlink (call_frame_t *frame, xlator_t *this,
+                   const char *linkname, loc_t *loc,
+                   mode_t umask, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INVOKE_FOP (priv, symlink, frame, this,
+                              linkname, loc, umask, xdata);
+
+ wind:
+        STACK_WIND (frame, changelog_symlink_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->symlink,
+                    linkname, loc, umask, xdata);
+        return 0;
+}
+
+/* mknod */
+
+int32_t
+changelog_mknod_cbk (call_frame_t *frame,
+                     void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, inode_t *inode,
+                     struct iatt *buf, struct iatt *preparent,
+                     struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        CHANGELOG_INVOKE_CFOP (this, priv,
+                               write, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (mknod, frame, op_ret, op_errno,
+                                inode, buf, preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_mknod (call_frame_t *frame,
+                 xlator_t *this, loc_t *loc,
+                 mode_t mode, dev_t dev, mode_t umask, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INVOKE_FOP (priv, mknod, frame, this,
+                              loc, mode, dev, umask, xdata);
+
+ wind:
+        STACK_WIND (frame, changelog_mknod_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->mknod,
+                    loc, mode, dev, umask, xdata);
+        return 0;
+}
+
+/* creat */
+
+int32_t
+changelog_create_cbk (call_frame_t *frame,
+                      void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno,
+                      fd_t *fd, inode_t *inode, struct iatt *buf,
+                      struct iatt *preparent,
+                      struct iatt *postparent, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        CHANGELOG_INVOKE_CFOP (this, priv,
+                               write, local, CHANGELOG_TYPE_ENTRY);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (create, frame,
+                                op_ret, op_errno, fd, inode,
+                                buf, preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+changelog_create (call_frame_t *frame, xlator_t *this,
+                  loc_t *loc, int32_t flags, mode_t mode,
+                  mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INVOKE_FOP (priv, create, frame, this, loc,
+                              flags, mode, umask, fd, xdata);
+
+ wind:
+        STACK_WIND (frame, changelog_create_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+        return 0;
+}
+
+/* }}} */
+
+
+/* Metadata modification fops - TYPE II */
+
+/* {{{ */
+
+/* {f}setattr */
+
+int32_t
+changelog_fsetattr_cbk (call_frame_t *frame,
+                        void *cookie, xlator_t *this, int32_t op_ret,
+                        int32_t op_errno, struct iatt *preop_stbuf,
+                        struct iatt *postop_stbuf, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        CHANGELOG_INVOKE_CFOP (this, priv,
+                               write, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (fsetattr, frame, op_ret, op_errno,
+                                preop_stbuf, postop_stbuf, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_fsetattr (call_frame_t *frame,
+                    xlator_t *this, fd_t *fd,
+                    struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INVOKE_FOP (priv, fsetattr,
+                              frame, this, fd, stbuf, valid, xdata);
+
+ wind:
+        STACK_WIND (frame, changelog_fsetattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetattr,
+                    fd, stbuf, valid, xdata);
+        return 0;
+
+
+}
+
+int32_t
+changelog_setattr_cbk (call_frame_t *frame,
+                       void *cookie, xlator_t *this, int32_t op_ret,
+                       int32_t op_errno, struct iatt *preop_stbuf,
+                       struct iatt *postop_stbuf, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        CHANGELOG_INVOKE_CFOP (this, priv,
+                               write, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (setattr, frame, op_ret, op_errno,
+                                preop_stbuf, postop_stbuf, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_setattr (call_frame_t *frame,
+                   xlator_t *this, loc_t *loc,
+                   struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INVOKE_FOP (priv, setattr,
+                              frame, this, loc, stbuf, valid, xdata);
+
+ wind:
+        STACK_WIND (frame, changelog_setattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->setattr,
+                    loc, stbuf, valid, xdata);
+        return 0;
+}
+
+/* {f}removexattr */
+
+int32_t
+changelog_fremovexattr_cbk (call_frame_t *frame,
+                            void *cookie, xlator_t *this,
+                            int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        CHANGELOG_INVOKE_CFOP (this, priv,
+                               write, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (fremovexattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_fremovexattr (call_frame_t *frame, xlator_t *this,
+                        fd_t *fd, const char *name, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INVOKE_FOP (priv, fremovexattr,
+                              frame, this, fd, name, xdata);
+
+ wind:
+        STACK_WIND (frame, changelog_fremovexattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fremovexattr,
+                    fd, name, xdata);
+        return 0;
+}
+
+int32_t
+changelog_removexattr_cbk (call_frame_t *frame,
+                           void *cookie, xlator_t *this,
+                           int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        CHANGELOG_INVOKE_CFOP (this, priv,
+                               write, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (removexattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_removexattr (call_frame_t *frame, xlator_t *this,
+                       loc_t *loc, const char *name, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INVOKE_FOP (priv, removexattr, frame, this, loc, name, xdata);
+
+ wind:
+        STACK_WIND (frame, changelog_removexattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->removexattr,
+                    loc, name, xdata);
+        return 0;
+}
+
+/* {f}setxattr */
+
+int32_t
+changelog_setxattr_cbk (call_frame_t *frame,
+                        void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        CHANGELOG_INVOKE_CFOP (this, priv,
+                               write, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (setxattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_setxattr (call_frame_t *frame,
+                    xlator_t *this, loc_t *loc,
+                    dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INVOKE_FOP (priv, setxattr,
+                              frame, this, loc, dict, flags, xdata);
+
+ wind:
+        STACK_WIND (frame, changelog_setxattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->setxattr,
+                    loc, dict, flags, xdata);
+        return 0;
+}
+
+int32_t
+changelog_fsetxattr_cbk (call_frame_t *frame,
+                         void *cookie, xlator_t *this, int32_t op_ret,
+                         int32_t op_errno, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        CHANGELOG_INVOKE_CFOP (this, priv,
+                               write, local, CHANGELOG_TYPE_METADATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (fsetxattr, frame, op_ret, op_errno, xdata);
+
+        return 0;
+}
+
+int32_t
+changelog_fsetxattr (call_frame_t *frame,
+                     xlator_t *this, fd_t *fd, dict_t *dict,
+                     int32_t flags, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INVOKE_FOP (priv, fsetxattr,
+                              frame, this, fd, dict, flags, xdata);
+
+ wind:
+        STACK_WIND (frame, changelog_fsetxattr_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->fsetxattr,
+                    fd, dict, flags, xdata);
+        return 0;
+}
+
+/* }}} */
+
+
+/* Data modification fops - TYPE I */
+
+/* {{{ */
+
+/* {f}truncate() */
+
+int32_t
+changelog_truncate_cbk (call_frame_t *frame,
+                        void *cookie, xlator_t *this, int32_t op_ret,
+                        int32_t op_errno, struct iatt *prebuf,
+                        struct iatt *postbuf, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        CHANGELOG_INVOKE_CFOP (this, priv, write, local, CHANGELOG_TYPE_DATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (truncate, frame,
+                                op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
+}
+
+int32_t
+changelog_truncate (call_frame_t *frame,
+                    xlator_t *this, loc_t *loc, off_t offset, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INVOKE_FOP (priv, truncate, frame, this, loc, offset, xdata);
+
+ wind:
+        STACK_WIND (frame, changelog_truncate_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->truncate,
+                    loc, offset, xdata);
+        return 0;
+}
+
+int32_t
+changelog_ftruncate_cbk (call_frame_t *frame,
+                         void *cookie, xlator_t *this, int32_t op_ret,
+                         int32_t op_errno, struct iatt *prebuf,
+                         struct iatt *postbuf, dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret < 0) || !local), unwind);
+
+        CHANGELOG_INVOKE_CFOP (this, priv, write, local, CHANGELOG_TYPE_DATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (ftruncate, frame,
+                                op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
+}
+
+int32_t
+changelog_ftruncate (call_frame_t *frame,
+                     xlator_t *this, fd_t *fd, off_t offset, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INVOKE_FOP (priv, ftruncate, frame, this, fd, offset, xdata);
+
+ wind:
+        STACK_WIND (frame, changelog_ftruncate_cbk,
+                    FIRST_CHILD (this), FIRST_CHILD (this)->fops->ftruncate,
+                    fd, offset, xdata);
+        return 0;
+}
+
+/* writev() */
+
+int32_t
+changelog_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                    struct iatt *postbuf,
+                    dict_t *xdata)
+{
+        changelog_priv_t  *priv  = NULL;
+        changelog_local_t *local = NULL;
+
+        priv  = this->private;
+        local = frame->local;
+
+        CHANGELOG_COND_GOTO (priv, ((op_ret <= 0) || !local), unwind);
+
+        CHANGELOG_INVOKE_CFOP (this, priv, write, local, CHANGELOG_TYPE_DATA);
+
+ unwind:
+        CHANGELOG_STACK_UNWIND (writev, frame,
+                                op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
+}
+
+int32_t
+changelog_writev (call_frame_t *frame,
+                  xlator_t *this, fd_t *fd, struct iovec *vector,
+                  int32_t count, off_t offset, uint32_t flags,
+                  struct iobref *iobref, dict_t *xdata)
+{
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+
+        CHANGELOG_NOT_ACTIVE_THEN_GOTO (frame, priv, wind);
+
+        CHANGELOG_INVOKE_FOP (priv, writev, frame, this, fd,
+                              vector, count, offset, flags, iobref, xdata);
+
+ wind:
+        STACK_WIND (frame, changelog_writev_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->writev, fd, vector,
+                    count, offset, flags, iobref, xdata);
+        return 0;
+}
+
+/* }}} */
+
+/**
+ * The
+ *   - @init ()
+ *   - @fini ()
+ *   - @reconfigure ()
+ *   ... and helper routines
+ */
+
+/**
+ * needed if there are more operation modes in the future.
+ */
+static void
+changelog_assign_opmode (changelog_priv_t *priv, char *mode)
+{
+        if ( strncmp (mode, "realtime", 8) == 0 ) {
+                priv->op_mode = CHANGELOG_MODE_RT;
+        }
+}
+
+static void
+changelog_assign_encoding (changelog_priv_t *priv, char *enc)
+{
+        if ( strncmp (enc, "binary", 6) == 0 ) {
+                priv->encode_mode = CHANGELOG_ENCODE_BINARY;
+        } else if ( strncmp (enc, "ascii", 5) == 0 ) {
+                priv->encode_mode = CHANGELOG_ENCODE_ASCII;
+        }
+}
+
+static void
+changelog_assign_policy (changelog_priv_t *priv, char *pol)
+{
+        if ( strncmp (pol, "default", 7) == 0 )
+                priv->policy = CHANGELOG_LOG_POLICY_DEFAULT;
+        else if ( strncmp (pol, "replication", 11) == 0 )
+                priv->policy = CHANGELOG_LOG_POLICY_REPLICATE;
+}
+
+/* cleanup any helper threads that are running */
+static void
+changelog_cleanup_helper_threads (xlator_t *this, changelog_priv_t *priv)
+{
+        if (priv->cr.rollover_th) {
+                changelog_thread_cleanup (this, priv->cr.rollover_th);
+                priv->cr.rollover_th = 0;
+        }
+
+        if (priv->cf.fsync_th) {
+                changelog_thread_cleanup (this, priv->cf.fsync_th);
+                priv->cf.fsync_th = 0;
+        }
+}
+
+/* spawn helper thread; cleaning up in case of errors */
+static int
+changelog_spawn_helper_threads (xlator_t *this, changelog_priv_t *priv)
+{
+        int ret = 0;
+
+        priv->cr.this = this;
+        if (priv->rollover_time) {
+                ret = pthread_create (&priv->cr.rollover_th,
+                                      NULL, changelog_rollover, priv);
+                if (ret)
+                        goto out;
+        }
+
+        if (priv->fsync_interval) {
+                priv->cf.this = this;
+                ret = pthread_create (&priv->cf.fsync_th,
+                                      NULL, changelog_fsync_thread, priv);
+        }
+
+        if (ret)
+                changelog_cleanup_helper_threads (this, priv);
+
+ out:
+        return ret;
+}
+
+/* cleanup the notifier thread */
+static int
+changelog_cleanup_notifier (xlator_t *this, changelog_priv_t *priv)
+{
+        int ret = 0;
+
+        if (priv->cn.notify_th) {
+                changelog_thread_cleanup (this, priv->cn.notify_th);
+                priv->cn.notify_th = 0;
+
+                ret = close (priv->wfd);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error closing writer end of notifier pipe"
+                                " (reason: %s)", strerror (errno));
+        }
+
+        return ret;
+}
+
+/* spawn the notifier thread - nop if already running */
+static int
+changelog_spawn_notifier (xlator_t *this, changelog_priv_t *priv)
+{
+        int ret        = 0;
+        int flags      = 0;
+        int pipe_fd[2] = {0, 0};
+
+        if (priv->cn.notify_th)
+                goto out; /* notifier thread already running */
+
+        ret = pipe (pipe_fd);
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Cannot create pipe (reason: %s)", strerror (errno));
+                goto out;
+        }
+
+        /* writer is non-blocking */
+        flags = fcntl (pipe_fd[1], F_GETFL);
+        flags |= O_NONBLOCK;
+
+        ret = fcntl (pipe_fd[1], F_SETFL, flags);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to set O_NONBLOCK flag");
+                goto out;
+        }
+
+        priv->wfd = pipe_fd[1];
+
+        priv->cn.this = this;
+        priv->cn.rfd  = pipe_fd[0];
+
+        ret = pthread_create (&priv->cn.notify_th,
+                              NULL, changelog_notifier, priv);
+
+ out:
+        return ret;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_changelog_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_WARNING, "Memory accounting"
+                        " init failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+static int
+changelog_init (xlator_t *this, changelog_priv_t *priv)
+{
+        int             i     = 0;
+        int             ret   = -1;
+        char           *cname = NULL;
+        struct timeval  tv    = {0,};
+
+        ret = gettimeofday (&tv, NULL);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "gettimeofday() failure");
+                goto out;
+        }
+
+        priv->slice.tv_start = tv;
+
+        priv->maps[CHANGELOG_TYPE_DATA]     = "D ";
+        priv->maps[CHANGELOG_TYPE_METADATA] = "M ";
+        priv->maps[CHANGELOG_TYPE_ENTRY]    = "E ";
+
+        for (; i < CHANGELOG_MAX_TYPE; i++) {
+                /* start with version 1 */
+                priv->slice.changelog_version[i] = 1;
+        }
+
+        if (!priv->active)
+                return ret;
+
+        /* spawn the notifier thread */
+        ret = changelog_spawn_notifier (this, priv);
+        if (ret)
+                goto out;
+
+        cname = CHANGELOG_FNAME_FROM_POLICY (priv->cp);
+
+        LOCK (&priv->lock);
+        {
+                ret = CHANGELOG_INVOKE_CFOP (this, priv,
+                                             open, cname, _gf_false);
+        }
+        UNLOCK (&priv->lock);
+
+        if (ret)
+                goto out;
+
+        /* ... and finally spawn the helpers threads */
+        ret = changelog_spawn_helper_threads (this, priv);
+
+ out:
+        return ret;
+}
+
+int
+reconfigure (xlator_t *this, dict_t *options)
+{
+        int                     ret            = 0;
+        char                   *tmp            = NULL;
+        char                   *cname          = NULL;
+        changelog_priv_t       *priv           = NULL;
+        gf_boolean_t            active_earlier = _gf_true;
+        gf_boolean_t            active_now     = _gf_true;
+        changelog_time_slice_t *slice          = NULL;
+
+        priv = this->private;
+        if (!priv)
+                goto out;
+
+        ret = -1;
+        active_earlier = priv->active;
+
+        /* first stop the rollover and the fsync thread */
+        changelog_cleanup_helper_threads (this, priv);
+
+        GF_OPTION_RECONF ("changelog-dir", tmp, options, str, out);
+        if (!tmp) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "\"changelog-dir\" option is not set");
+                goto out;
+        }
+
+        GF_FREE (priv->changelog_dir);
+        priv->changelog_dir = gf_strdup (tmp);
+        if (!priv->changelog_dir)
+                goto out;
+
+        ret = mkdir_p (priv->changelog_dir, 0600, _gf_true);
+        if (ret)
+                goto out;
+
+        GF_OPTION_RECONF ("changelog", active_now, options, bool, out);
+
+        /**
+         * changelog_handle_change() handles changes that could possibly
+         * have been submit changes before changelog deactivation.
+         */
+        if (!active_now)
+                priv->active = _gf_false;
+
+        GF_OPTION_RECONF ("op-mode", tmp, options, str, out);
+        changelog_assign_opmode (priv, tmp);
+
+        tmp = NULL;
+
+        GF_OPTION_RECONF ("encoding", tmp, options, str, out);
+        changelog_assign_encoding (priv, tmp);
+
+        GF_OPTION_RECONF ("rollover-time",
+                          priv->rollover_time, options, int32, out);
+        GF_OPTION_RECONF ("fsync-interval",
+                          priv->fsync_interval, options, int32, out);
+
+        if (active_now || active_earlier) {
+                slice = &priv->slice;
+                cname = CHANGELOG_FNAME_FROM_POLICY (priv->cp);
+
+                LOCK (&priv->lock);
+                {
+                        ret = CHANGELOG_INVOKE_CFOP (this, priv, rollover,
+                                                     cname, !active_now);
+                        if (!ret && active_now)
+                                SLICE_VERSION_UPDATE (slice);
+                }
+                UNLOCK (&priv->lock);
+
+                if (ret)
+                        goto out;
+
+                if (active_now) {
+                        ret = changelog_spawn_notifier (this, priv);
+                        if (!ret)
+                                ret = changelog_spawn_helper_threads (this,
+                                                                      priv);
+                } else
+                        ret = changelog_cleanup_notifier (this, priv);
+        }
+
+ out:
+        if (ret) {
+                ret = changelog_cleanup_notifier (this, priv);
+        } else {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "changelog reconfigured");
+                if (active_now)
+                        priv->active = _gf_true;
+        }
+
+        return ret;
+}
+
+int32_t
+init (xlator_t *this)
+{
+        int               ret  = -1;
+        char             *tmp  = NULL;
+        changelog_priv_t *priv = NULL;
+
+        GF_VALIDATE_OR_GOTO ("changelog", this, out);
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "translator needs a single subvolume");
+                goto out;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "dangling volume. please check volfile");
+                goto out;
+        }
+
+        priv = GF_CALLOC (1, sizeof (*priv), gf_changelog_mt_priv_t);
+        if (!priv)
+                goto out;
+
+        this->local_pool = mem_pool_new (changelog_local_t, 64);
+        if (!this->local_pool) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local memory pool");
+                goto out;
+        }
+
+        LOCK_INIT (&priv->lock);
+
+        GF_OPTION_INIT ("changelog-brick", tmp, str, out);
+        if (!tmp) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "\"changelog-brick\" option is not set");
+                goto out;
+        }
+
+        priv->changelog_brick = gf_strdup (tmp);
+        if (!priv->changelog_brick)
+                goto out;
+        tmp = NULL;
+
+        GF_OPTION_INIT ("changelog-dir", tmp, str, out);
+        if (!tmp) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "\"changelog-dir\" option is not set");
+                goto out;
+        }
+
+        priv->changelog_dir = gf_strdup (tmp);
+        if (!priv->changelog_dir)
+                goto out;
+        tmp = NULL;
+
+        /**
+         * create the directory even if change-logging would be inactive
+         * so that consumers can _look_ into it (finding nothing...)
+         */
+        ret = mkdir_p (priv->changelog_dir, 0600, _gf_true);
+        if (ret)
+                goto out;
+
+        GF_OPTION_INIT ("changelog", priv->active, bool, out);
+
+        GF_OPTION_INIT ("op-mode", tmp, str, out);
+        changelog_assign_opmode (priv, tmp);
+
+        tmp = NULL;
+
+        GF_OPTION_INIT ("encoding", tmp, str, out);
+        changelog_assign_encoding (priv, tmp);
+
+        tmp = NULL;
+
+        GF_OPTION_INIT ("policy", tmp, str, out);
+        changelog_assign_policy (priv, tmp);
+
+        GF_OPTION_INIT ("fsync-interval", priv->fsync_interval, int32, out);
+
+        GF_ASSERT (cb_encoder[priv->encode_mode].encoder == priv->encode_mode);
+        priv->ce = &cb_encoder[priv->encode_mode];
+
+        GF_ASSERT (cb_bootstrap[priv->op_mode].mode == priv->op_mode);
+        priv->cb = &cb_bootstrap[priv->op_mode];
+
+        GF_ASSERT (cb_policy[priv->policy].policy == priv->policy);
+        priv->cp = &cb_policy[priv->policy];
+
+
+        /* ... init logging policy */
+        ret = priv->cp->init_policy (this, priv, priv->cp);
+        if (ret)
+                goto out;
+
+        /* ... now bootstrap the logger */
+        ret = priv->cb->ctor (this, &priv->cd, priv->lockless_update);
+        if (ret)
+                goto out;
+
+        /* override the value if set */
+        if (dict_get (this->options, "rollover-time")) {
+                ret = dict_get_int32 (this->options,
+                                      "rollover-time", &priv->rollover_time);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "Cannot get value for \"rollover-time\"");
+                        goto out;
+                }
+        }
+
+        priv->changelog_fd = -1;
+        ret = changelog_init (this, priv);
+        if (ret)
+                goto out;
+
+        gf_log (this->name, GF_LOG_DEBUG, "changelog translator loaded");
+
+ out:
+        if (ret) {
+                if (this->local_pool)
+                        mem_pool_destroy (this->local_pool);
+                if (priv->cb) {
+                        ret = priv->cb->dtor (this, &priv->cd);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_ERROR,
+                                        "error in cleanup during init()");
+                }
+                GF_FREE (priv->changelog_brick);
+                GF_FREE (priv->changelog_dir);
+                GF_FREE (priv);
+                this->private = NULL;
+        } else
+                this->private = priv;
+
+        return ret;
+}
+
+void
+fini (xlator_t *this)
+{
+        int               ret  = -1;
+        changelog_priv_t *priv = NULL;
+
+        priv = this->private;
+
+        if (priv) {
+                ret = priv->cb->dtor (this, &priv->cd);
+                if (ret)
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "error in fini");
+                mem_pool_destroy (this->local_pool);
+                GF_FREE (priv->changelog_brick);
+                GF_FREE (priv->changelog_dir);
+                GF_FREE (priv);
+        }
+
+        this->private = NULL;
+
+        return;
+}
+
+struct xlator_fops fops = {
+        .mknod        = changelog_mknod,
+        .mkdir        = changelog_mkdir,
+        .create       = changelog_create,
+        .symlink      = changelog_symlink,
+        .writev       = changelog_writev,
+        .truncate     = changelog_truncate,
+        .ftruncate    = changelog_ftruncate,
+        .link         = changelog_link,
+        .rename       = changelog_rename,
+        .unlink       = changelog_unlink,
+        .rmdir        = changelog_rmdir,
+        .setattr      = changelog_setattr,
+        .fsetattr     = changelog_fsetattr,
+        .setxattr     = changelog_setxattr,
+        .fsetxattr    = changelog_fsetxattr,
+        .removexattr  = changelog_removexattr,
+        .fremovexattr = changelog_fremovexattr,
+};
+
+struct xlator_cbks cbks = {
+        .forget = changelog_forget,
+};
+
+struct volume_options options[] = {
+        {.key = {"changelog"},
+         .type = GF_OPTION_TYPE_BOOL,
+         .default_value = "on",
+         .description = "enable/disable change-logging"
+        },
+        {.key = {"changelog-brick"},
+         .type = GF_OPTION_TYPE_PATH,
+         .description = "brick path to generate unique socket file name."
+                       " should be the export directory of the volume strictly."
+        },
+        {.key = {"changelog-dir"},
+         .type = GF_OPTION_TYPE_PATH,
+         .description = "directory for the changelog files"
+        },
+        {.key = {"op-mode"},
+         .type = GF_OPTION_TYPE_STR,
+         .default_value = "realtime",
+         .value = {"realtime"},
+         .description = "operation mode - futuristic operation modes"
+        },
+        {.key = {"encoding"},
+         .type = GF_OPTION_TYPE_STR,
+         .default_value = "ascii",
+         .value = {"binary", "ascii"},
+         .description = "encoding type for changelogs"
+        },
+        {.key = {"rollover-time"},
+         .description = "time to switch to a new changelog file (in seconds)"
+        },
+        {.key = {"fsync-interval"},
+         .type = GF_OPTION_TYPE_TIME,
+         .default_value = "5",
+         .description = "do not open CHANGELOG file with O_SYNC mode."
+                        " instead perform fsync() at specified intervals"
+        },
+        {.key = {"policy"},
+         .type = GF_OPTION_TYPE_STR,
+         .default_value = "replication",
+         .value = {"default", "replication"},
+         .description = "Logging policies"
+        },
+        {.key = {NULL}
+        },
+};
diff --git a/xlators/features/changelog/src/policy/changelog-policy-default.c b/xlators/features/changelog/src/policy/changelog-policy-default.c
new file mode 100644
index 000000000..eaa3d107f
--- /dev/null
+++ b/xlators/features/changelog/src/policy/changelog-policy-default.c
@@ -0,0 +1,45 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "changelog-policy.h"
+#include "changelog-fops.h"
+
+int
+changelog_default_policy_init (xlator_t *this,
+                               changelog_priv_t *priv,
+                               struct changelog_logpolicy *cp)
+{
+        priv->rollover_time = 15;
+
+        priv->no_gfid_hdr = _gf_false;
+        priv->lockless_update = _gf_false;
+
+        cp->cpriv = GF_CALLOC (1, sizeof (off_t),
+                                 gf_changelog_mt_fop_policy_t);
+        if (!cp->cpriv)
+                return -1;
+
+        (void) memset (cp->changelog_name, '\0', PATH_MAX);
+        (void) memcpy (cp->changelog_name,
+                       CHANGELOG_FILE_NAME, strlen (CHANGELOG_FILE_NAME));
+
+        cp->fops = &changelog_default_fops; /* default logging policy */
+        cp->cops = &changelog_default_cops; /* default changelog operations */
+
+        return 0;
+}
+
+int
+changelog_default_policy_fini (xlator_t *this,
+                               struct changelog_logpolicy *cp)
+{
+        GF_FREE (cp->cpriv);
+        return 0;
+}
diff --git a/xlators/features/changelog/src/policy/changelog-policy-replication.c b/xlators/features/changelog/src/policy/changelog-policy-replication.c
new file mode 100644
index 000000000..29c049716
--- /dev/null
+++ b/xlators/features/changelog/src/policy/changelog-policy-replication.c
@@ -0,0 +1,1374 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include "changelog-policy.h"
+#include "changelog-encoders.h"
+#include "changelog-fops.h"
+
+#define JOURNAL_NAME  "TERM"
+
+#define JOURNAL_SECTOR_SIZE  128
+
+#define PRE_OP_MARK  0x5F4552505FULL   /* _PRE_ */
+#define POST_OP_MARK 0x5F54534F505FULL /* _POST_ */
+
+/* similar to fop_fn, but... */
+size_t
+int32_fn (void *data, char *buffer, gf_boolean_t encode)
+{
+        size_t bufsz = 0;
+        int    nr    = 0;
+        char buf[20] = {0,};
+
+        nr = *(int *) data;
+
+        if (encode) {
+                (void) snprintf (buf, sizeof (buf), "%d", nr);
+                CHANGELOG_FILL_BUFFER (buffer, bufsz, buf, strlen (buf));
+        } else
+                CHANGELOG_FILL_BUFFER (buffer, bufsz,
+                                       &nr, sizeof (int));
+
+        return bufsz;
+}
+
+
+size_t
+uint32_fn (void *data, char *buffer, gf_boolean_t encode)
+{
+        size_t       bufsz = 0;
+        unsigned int nr    = 0;
+        char buf[20]       = {0,};
+
+        nr = *(unsigned int *) data;
+
+        if (encode) {
+                (void) snprintf (buf, sizeof (buf), "%u", nr);
+                CHANGELOG_FILL_BUFFER (buffer, bufsz, buf, strlen (buf));
+        } else
+                CHANGELOG_FILL_BUFFER (buffer, bufsz,
+                                       &nr, sizeof (unsigned int));
+
+        return bufsz;
+}
+
+size_t
+number_fn (void *data, char *buffer, gf_boolean_t encode)
+{
+        char buf[1024]           = {0,};
+        size_t             bufsz = 0;
+        unsigned long long nr    = 0;
+
+        nr = *(unsigned long long *) data;
+
+        if (encode) {
+                (void) snprintf (buf, sizeof (buf), "%llu", nr);
+                CHANGELOG_FILL_BUFFER (buffer, bufsz, buf, strlen (buf));
+        } else
+                CHANGELOG_FILL_BUFFER (buffer, bufsz,
+                                       &nr, sizeof (unsigned long long));
+
+        return bufsz;
+}
+
+size_t
+uuid_fn (void *data, char *buffer, gf_boolean_t encode)
+{
+        char buf[1024] = {0,};
+        uuid_t uuid    = {0,};
+        size_t bufsz   = 0;
+
+        memcpy (uuid, (uuid_t *) data, sizeof (uuid_t));
+
+        if (encode) {
+                char *tmpbuf = uuid_utoa (uuid);
+                (void) snprintf (buf, sizeof (buf), "%s", tmpbuf);
+                CHANGELOG_FILL_BUFFER (buffer, bufsz, buf, strlen (buf));
+        } else
+                CHANGELOG_FILL_BUFFER (buffer, bufsz, uuid, sizeof (uuid_t));
+
+        return bufsz;
+}
+
+#define CHANGELOG_FILL_USIGNLL(co, number, converter, xlen) do {        \
+                co->co_convert = converter;                             \
+                co->co_free = NULL;                                     \
+                co->co_type = CHANGELOG_OPT_REC_ULL;                    \
+                co->co_number = (unsigned long long) number;            \
+                xlen += sizeof (unsigned long long);                    \
+                if (!co->co_convert)                                    \
+                        co->co_len = sizeof (unsigned long long);       \
+        } while (0)
+
+#define CHANGELOG_FILL_UUID(co, uuid, converter, xlen) do {     \
+                co->co_convert = converter;                     \
+                co->co_free = NULL;                             \
+                co->co_type = CHANGELOG_OPT_REC_UUID;           \
+                uuid_copy (co->co_uuid, uuid);                  \
+                xlen += sizeof (uuid_t);                        \
+        } while (0)
+
+
+/* TBD: move declarations here and nsr.c into a common place */
+#define NSR_TERM_XATTR          "trusted.nsr.term"
+#define NSR_INDEX_XATTR         "trusted.nsr.index"
+#define RECON_TERM_XATTR        "trusted.nsr.recon-term"
+#define RECON_INDEX_XATTR       "trusted.nsr.recon-index"
+
+static gf_boolean_t
+changelog_fix_term(xlator_t *this,
+                   changelog_local_t *local,
+                   dict_t *xdata)
+{
+        int32_t old_term, new_term;
+	uint32_t index;
+        changelog_priv_t  *priv = this->private;
+        int ret   = 0;
+        char nfile[PATH_MAX] = {0,};
+        int32_t recon_term, recon_index;
+        changelog_rollover_data_t crd;
+
+        // If coming via the regular IO path, we should get the dict "nsr-term"
+        // If coming via reconciliation, we should get the dicts "nsr-recon-term"
+        // that indicates the term and "nsr-recon-index" for the index
+        if ((dict_get_int32(xdata,NSR_TERM_XATTR,&new_term) == 0) &&
+	    (dict_get_uint32(xdata, NSR_INDEX_XATTR, &index) == 0)) {
+                old_term = priv->term;
+
+                if (old_term != new_term) {
+                        GF_ASSERT(new_term > old_term);
+                        LOCK (&priv->lock);
+                        priv->term = new_term;
+                        (void) snprintf (nfile, PATH_MAX, "%s.%d",
+                                         JOURNAL_NAME, priv->term);
+                        ret = CHANGELOG_INVOKE_CFOP(this, priv, rollover,
+                                                    nfile, _gf_false);
+                        UNLOCK (&priv->lock);
+                        if (ret != 0)
+                                return  _gf_false;
+                }
+                local->nr_bytes = 0;
+                local->lu.val = index;
+        } else if ((dict_get_int32(xdata, RECON_TERM_XATTR, &recon_term) == 0) &&
+                   (dict_get_int32(xdata, RECON_INDEX_XATTR, &recon_index) == 0)) {
+
+                old_term = priv->term;
+
+                if (old_term != recon_term) {
+                        LOCK (&priv->lock);
+                        {
+                                priv->term = recon_term;
+                                (void) snprintf (crd.crd_changelog_name,
+                                                 PATH_MAX, "%s.%d",
+                                                 JOURNAL_NAME, priv->term);
+                                crd.crd_prealloc_size = 1<<29;
+                                ret = changelog_open(this, priv, local, &crd);
+                        }
+                        UNLOCK (&priv->lock);
+                        if (ret != 0)
+                                return _gf_false;
+                }
+
+                local->nr_bytes = 0;
+                local->lu.val = recon_index;
+        } else {
+                return _gf_false;
+        }
+
+        return _gf_true;
+}
+
+/**
+ * Replication policy records journal entries in the FOP path. This is
+ * quite different that the default policy (used by geo-replication),
+ * which journals records in the callback path on a successfull posix
+ * operation. Additionally, each record starts with a PRE-OP marker and
+ * the index number generated by the leader.
+ * (c.f. nsr_$NAME$() ~/xlator/cluster/nsr/nsr-server/src/all-templates.c)
+ *
+ * POST-OPs are marked asynchronously and not during in the callback path
+ * Marking it in the callback path is incorrect as the actual FOP may not
+ * have been synchronized to the disk. Therefore, POST op marking is done
+ * after a successful file system sync, which is trigerred periodically
+ * by NSR server component. To keep journal updates strictly sequential,
+ * POST-OPs are separate record in the journal.
+ */
+
+/**
+ * Override File Operations
+ *
+ * NOTE: Since journal updates are done in the FOP path, there is no
+ * actual use of @local in cbk. Therefore, @local could have been
+ * declared statically for each FOP (which would remove the overhead
+ * of allocating (here) and deallocating (in cbk)). Let's not do that
+ * and keep it this way for now. Will worry about it later (for code
+ * reasonability and performance).
+ */
+
+int32_t
+changelog_replication_rmdir (call_frame_t *frame, xlator_t *this,
+                              loc_t *loc, int xflags, dict_t *xdata)
+{
+        int                ret      = -1;
+        size_t             xtra_len = 0;
+        changelog_priv_t  *priv     = NULL;
+        changelog_opt_t   *co       = NULL;
+        changelog_local_t *local    = NULL;
+
+        priv = this->private;
+
+        /* <PRE> + IDX + FOP + GFID + UID + GID + Entry */
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, loc->inode->gfid, 7);
+        if (!local)
+                goto out;
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        if (changelog_fix_term(this, local, xdata) == _gf_false)
+                goto out;
+
+        CHANGELOG_FILL_USIGNLL (co, PRE_OP_MARK, NULL, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, local->lu.val, uint32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UUID (co, loc->inode->gfid, uuid_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_INT32 (co, frame->root->uid, int32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_INT32 (co, frame->root->gid, int32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, out);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 7);
+
+        frame->local = local;
+        ret = 0;
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+int32_t
+changelog_replication_unlink (call_frame_t *frame, xlator_t *this,
+                              loc_t *loc, int xflags, dict_t *xdata)
+{
+        return changelog_replication_rmdir (frame, this, loc, xflags, xdata);
+}
+
+int32_t
+changelog_replication_rename (call_frame_t *frame, xlator_t *this,
+                              loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        int                ret      = -1;
+        size_t             xtra_len = 0;
+        changelog_priv_t  *priv     = NULL;
+        changelog_opt_t   *co       = NULL;
+        changelog_local_t *local    = NULL;
+
+        priv = this->private;
+
+        /* <PRE> + IDX + FOP + GFID + UID + GID + OLDLOC + NEWLOC */
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, oldloc->inode->gfid, 8);
+        if (!local)
+                goto out;
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        if (changelog_fix_term(this, local, xdata) == _gf_false)
+                goto out;
+
+        CHANGELOG_FILL_USIGNLL (co, PRE_OP_MARK, NULL, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, local->lu.val, uint32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UUID (co, oldloc->inode->gfid, uuid_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_INT32 (co, frame->root->uid, int32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_INT32 (co, frame->root->gid, int32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_ENTRY (co, oldloc->pargfid, oldloc->name,
+                              entry_fn, entry_free_fn, xtra_len, out);
+        co++;
+
+        CHANGELOG_FILL_ENTRY (co, newloc->pargfid, newloc->name,
+                              entry_fn, entry_free_fn, xtra_len, out);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 8);
+
+        frame->local = local;
+        ret = 0;
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+int32_t
+changelog_replication_link (call_frame_t *frame,
+                            xlator_t *this, loc_t *oldloc,
+                            loc_t *newloc, dict_t *xdata)
+{
+        int                ret      = -1;
+        size_t             xtra_len = 0;
+        changelog_priv_t  *priv     = NULL;
+        changelog_opt_t   *co       = NULL;
+        changelog_local_t *local    = NULL;
+
+        priv = this->private;
+
+        /* <PRE> + IDX + FOP + GFID + UID + GID + Entry */
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, oldloc->gfid, 7);
+        if (!local)
+                goto out;
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        if (changelog_fix_term(this, local, xdata) == _gf_false)
+                goto out;
+
+        CHANGELOG_FILL_USIGNLL (co, PRE_OP_MARK, NULL, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, local->lu.val, uint32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UUID (co, oldloc->gfid, uuid_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_INT32 (co, frame->root->uid, int32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_INT32 (co, frame->root->gid, int32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_ENTRY (co, newloc->pargfid, newloc->name,
+                              entry_fn, entry_free_fn, xtra_len, out);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 7);
+
+        frame->local = local;
+        ret = 0;
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+int32_t
+changelog_replication_mkdir (call_frame_t *frame,
+                             xlator_t *this, loc_t *loc,
+                             mode_t mode, mode_t umask, dict_t *xdata)
+{
+        int                ret      = -1;
+        uuid_t             gfid     = {0,};
+        void              *uuid_req = NULL;
+        size_t             xtra_len = 0;
+        changelog_priv_t  *priv     = NULL;
+        changelog_opt_t   *co       = NULL;
+        changelog_local_t *local    = NULL;
+
+        priv = this->private;
+
+        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get gfid from dict");
+                goto out;
+        }
+        uuid_copy (gfid, uuid_req);
+
+        ret = -1;
+
+        /* <PRE> + IDX + FOP + GFID + MODE + UID + GID + Entry */
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, gfid, 8);
+        if (!local)
+                goto out;
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        if (changelog_fix_term(this, local, xdata) == _gf_false)
+                goto out;
+
+        CHANGELOG_FILL_USIGNLL (co, PRE_OP_MARK, NULL, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, local->lu.val, uint32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UUID (co, gfid, uuid_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, mode | S_IFDIR, uint32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_INT32 (co, frame->root->uid, int32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_INT32 (co, frame->root->gid, int32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, out);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 8);
+
+        frame->local = local;
+        ret = 0;
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+int32_t
+changelog_replication_symlink (call_frame_t *frame, xlator_t *this,
+                               const char *linkname, loc_t *loc,
+                               mode_t umask, dict_t *xdata)
+{
+        int                ret      = -1;
+        size_t             xtra_len = 0;
+        uuid_t             gfid     = {0,};
+        void              *uuid_req = NULL;
+        changelog_priv_t  *priv     = NULL;
+        changelog_opt_t   *co       = NULL;
+        changelog_local_t *local    = NULL;
+
+        priv = this->private;
+
+        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get gfid from dict");
+                goto out;
+        }
+        uuid_copy (gfid, uuid_req);
+
+        ret = -1;
+
+        /* <PRE> + IDX + FOP + GFID + LINKNAME + UID + GID + Entry */
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, gfid, 8);
+        if (!local)
+                goto out;
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        if (changelog_fix_term(this, local, xdata) == _gf_false)
+                goto out;
+
+        CHANGELOG_FILL_USIGNLL (co, PRE_OP_MARK, NULL, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, local->lu.val, uint32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UUID (co, gfid, uuid_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_NAME (co, linkname, entry_free_fn, xtra_len, out);
+        co++;
+
+        CHANGELOG_FILL_INT32 (co, frame->root->uid, int32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_INT32 (co, frame->root->gid, int32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, out);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 8);
+
+        frame->local = local;
+        ret = 0;
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+int32_t
+changelog_replication_mknod (call_frame_t *frame,
+                             xlator_t *this, loc_t *loc,
+                             mode_t mode, dev_t dev,
+                             mode_t umask, dict_t *xdata)
+{
+        int                ret      = -1;
+        uuid_t             gfid     = {0,};
+        void              *uuid_req = NULL;
+        size_t             xtra_len = 0;
+        changelog_priv_t  *priv     = NULL;
+        changelog_opt_t   *co       = NULL;
+        changelog_local_t *local    = NULL;
+
+        priv = this->private;
+
+        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get gfid from dict");
+                goto out;
+        }
+        uuid_copy (gfid, uuid_req);
+
+        ret = -1;
+
+        /* <PRE> + IDX + FOP + GFID + MODE + UID + GID + DEV + Entry */
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, gfid, 9);
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        if (changelog_fix_term (this, local, xdata) == _gf_false)
+                goto out;
+
+        CHANGELOG_FILL_USIGNLL (co, PRE_OP_MARK, NULL, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, local->lu.val, uint32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UUID (co, gfid, uuid_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, mode, uint32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_INT32 (co, frame->root->uid, int32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_INT32 (co, frame->root->gid, int32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_USIGNLL (co, dev, number_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, out);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 9);
+
+        frame->local = local;
+        ret = 0;
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+int32_t
+changelog_replication_create (call_frame_t *frame, xlator_t *this,
+                              loc_t *loc, int32_t flags, mode_t mode,
+                              mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        int                ret      = -1;
+        uuid_t             gfid     = {0,};
+        void              *uuid_req = NULL;
+        size_t             xtra_len = 0;
+        changelog_priv_t  *priv     = NULL;
+        changelog_opt_t   *co       = NULL;
+        changelog_local_t *local    = NULL;
+
+        priv = this->private;
+
+        ret = dict_get_ptr (xdata, "gfid-req", &uuid_req);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "failed to get gfid from dict");
+                goto out;
+        }
+        uuid_copy (gfid, uuid_req);
+
+        ret = -1;
+
+        /* <PRE> + IDX + FOP + GFID + MODE + UID + GID + ENTRY */
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, gfid, 8);
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        if (changelog_fix_term (this, local, xdata) == _gf_false)
+                goto out;
+
+        CHANGELOG_FILL_USIGNLL (co, PRE_OP_MARK, NULL, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, local->lu.val, uint32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UUID (co, gfid, uuid_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, mode, uint32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_INT32 (co, frame->root->uid, int32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_INT32 (co, frame->root->gid, int32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_ENTRY (co, loc->pargfid, loc->name,
+                              entry_fn, entry_free_fn, xtra_len, out);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 8);
+
+        frame->local = local;
+        ret = 0;
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_ENTRY);
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+static int
+_changelog_setattr_fill_common (call_frame_t *frame, xlator_t *this,
+                                int32_t attr, struct iatt *stbuf,
+                                uuid_t gfid, dict_t *xdata)
+{
+        int                ret        = -1;
+        size_t             xtra_len   = 0;
+        int                used_count = 0;
+        changelog_priv_t  *priv       = NULL;
+        changelog_opt_t   *co         = NULL;
+        changelog_local_t *local      = NULL;
+
+        priv = this->private;
+
+        used_count = 7;
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, gfid, used_count);
+        if (!local)
+                goto out;
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        if (changelog_fix_term(this, local, xdata) == _gf_false)
+                goto out;
+
+        /**
+         * - <PRE>
+         * - IDX
+         * - FOP
+         * - GFID
+         * - Valid flag
+         *     GF_SET_ATTR_MODE [chmod]
+         *       ->ia_prot
+         *       ->ia_type
+         *     GF_SET_ATTR_UID | GF_SET_ATTR_GID [chown]
+         *       ->ia_uid
+         *       ->ia_gid
+         *     GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME [utimes]
+         *       ->ia_atime
+         *       ->ia_mtime
+         */
+
+        CHANGELOG_FILL_USIGNLL (co, PRE_OP_MARK, NULL, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, local->lu.val, uint32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UUID (co, gfid, uuid_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, attr, uint32_fn, xtra_len);
+        co++;
+
+        if (attr & GF_SET_ATTR_MODE) {
+                mode_t mode = 0;
+
+                /* ->ia_prot & ->ia_type stored as a consolidated value */
+                used_count--;
+                mode = st_mode_from_ia (stbuf->ia_prot, stbuf->ia_type);
+
+                CHANGELOG_FILL_UINT32 (co, mode, uint32_fn, xtra_len);
+
+        } else if (attr & (GF_SET_ATTR_UID | GF_SET_ATTR_GID)) {
+                uid_t uid = -1;
+                gid_t gid = -1;
+
+                if (attr & GF_SET_ATTR_UID)
+                        uid = stbuf->ia_uid;
+                if (attr & GF_SET_ATTR_GID)
+                        gid = stbuf->ia_gid;
+
+                /* ->ia_uid & ->ia_gid */
+                CHANGELOG_FILL_INT32 (co, uid, int32_fn, xtra_len);
+                co++;
+
+                CHANGELOG_FILL_INT32 (co, gid, int32_fn, xtra_len);
+
+        } else if (attr & (GF_SET_ATTR_ATIME | GF_SET_ATTR_MTIME)) {
+
+                /* ->ia_atime & ->ia_mtime, need usecs? */
+                CHANGELOG_FILL_UINT32 (co,
+                                       stbuf->ia_atime, uint32_fn, xtra_len);
+                co++;
+
+                CHANGELOG_FILL_UINT32 (co,
+                                       stbuf->ia_mtime, uint32_fn, xtra_len);
+        }
+
+        changelog_set_usable_record_and_length (local, xtra_len, used_count);
+
+        ret = 0;
+        frame->local = local;
+
+        changelog_update (this, priv, frame->local, CHANGELOG_TYPE_METADATA);
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+
+        return ret;
+}
+
+int32_t
+changelog_replication_fsetattr (call_frame_t *frame,
+                                xlator_t *this, fd_t *fd,
+                                struct iatt *stbuf, int32_t valid,
+                                dict_t *xdata)
+{
+        return _changelog_setattr_fill_common (frame, this, valid,
+                                               stbuf, fd->inode->gfid, xdata);
+}
+
+int32_t
+changelog_replication_setattr (call_frame_t *frame,
+                               xlator_t *this, loc_t *loc,
+                               struct iatt *stbuf, int32_t valid, dict_t *xdata)
+{
+        return _changelog_setattr_fill_common (frame, this, valid,
+                                               stbuf, loc->inode->gfid, xdata);
+}
+
+int32_t
+changelog_replication_fremovexattr (call_frame_t *frame, xlator_t *this,
+                                    fd_t *fd, const char *name, dict_t *xdata)
+{
+        int                ret      = -1;
+        size_t             xtra_len = 0;
+        changelog_opt_t   *co       = NULL;
+        changelog_priv_t  *priv     = NULL;
+        changelog_local_t *local    = NULL;
+        int32_t            xattr_op;
+
+        priv = this->private;
+
+        /* <PRE> + IDX + FOP + GFID */
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, fd->inode->gfid, 4);
+        if (!local)
+                goto out;
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        if (changelog_fix_term(this, local, xdata) == _gf_false)
+                goto out;
+
+        CHANGELOG_FILL_USIGNLL (co, PRE_OP_MARK, NULL, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, local->lu.val, uint32_fn, xtra_len);
+        co++;
+
+        if (dict_get_int32(xdata, "recon-xattr-opcode", &xattr_op) == 0)
+                CHANGELOG_FILL_FOP_NUMBER (co, (glusterfs_fop_t)xattr_op,
+                                                fop_fn, xtra_len);
+        else
+                CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn,
+                                           xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UUID (co, fd->inode->gfid, uuid_fn, xtra_len);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 4);
+
+        frame->local = local;
+        ret = 0;
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+int32_t
+changelog_replication_removexattr (call_frame_t *frame, xlator_t *this,
+                                   loc_t *loc, const char *name, dict_t *xdata)
+{
+        int                ret      = -1;
+        size_t             xtra_len = 0;
+        changelog_priv_t  *priv     = NULL;
+        changelog_opt_t   *co       = NULL;
+        changelog_local_t *local    = NULL;
+        int32_t            xattr_op;
+
+        priv = this->private;
+
+        /* <PRE> + IDX + FOP + GFID */
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, loc->inode->gfid, 4);
+        if (!local)
+                goto out;
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        if (changelog_fix_term(this, local, xdata) == _gf_false)
+                goto out;
+
+        CHANGELOG_FILL_USIGNLL (co, PRE_OP_MARK, NULL, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, local->lu.val, uint32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, local->lu.val, uint32_fn, xtra_len);
+        co++;
+
+        if (dict_get_int32(xdata, "recon-xattr-opcode", &xattr_op) == 0)
+                CHANGELOG_FILL_FOP_NUMBER (co, (glusterfs_fop_t)xattr_op,
+                                           fop_fn, xtra_len);
+        else
+                CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn,
+                                           xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UUID (co, loc->inode->gfid, uuid_fn, xtra_len);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 4);
+
+        frame->local = local;
+        ret = 0;
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+int32_t
+changelog_replication_setxattr (call_frame_t *frame,
+                                xlator_t *this, loc_t *loc,
+                                dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        int                ret      = -1;
+        size_t             xtra_len = 0;
+        changelog_priv_t  *priv     = NULL;
+        changelog_opt_t   *co       = NULL;
+        changelog_local_t *local    = NULL;
+        int32_t            xattr_op;
+
+        priv = this->private;
+
+        /* <PRE> + IDX + FOP + GFID */
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, loc->inode->gfid, 4);
+        if (!local)
+                goto out;
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        if (changelog_fix_term(this, local, xdata) == _gf_false)
+                goto out;
+
+        CHANGELOG_FILL_USIGNLL (co, PRE_OP_MARK, NULL, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, local->lu.val, uint32_fn, xtra_len);
+        co++;
+
+        if (dict_get_int32(xdata, "recon-xattr-opcode", &xattr_op) == 0)
+                CHANGELOG_FILL_FOP_NUMBER (co, (glusterfs_fop_t)xattr_op,
+                                           fop_fn, xtra_len);
+        else
+                CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn,
+                                           xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UUID (co, loc->inode->gfid, uuid_fn, xtra_len);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 4);
+
+        frame->local = local;
+        ret = 0;
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+int32_t
+changelog_replication_fsetxattr (call_frame_t *frame,
+                                 xlator_t *this, fd_t *fd, dict_t *dict,
+                                 int32_t flags, dict_t *xdata)
+{
+        int                ret      = -1;
+        size_t             xtra_len = 0;
+        changelog_priv_t  *priv     = NULL;
+        changelog_opt_t   *co       = NULL;
+        changelog_local_t *local    = NULL;
+        int32_t            xattr_op;
+
+        priv = this->private;
+
+        /* <PRE> + IDX + FOP + GFID */
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, fd->inode->gfid, 4);
+        if (!local)
+                goto out;
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        if (changelog_fix_term(this, local, xdata) == _gf_false)
+                goto out;
+
+        CHANGELOG_FILL_USIGNLL (co, PRE_OP_MARK, NULL, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, local->lu.val, uint32_fn, xtra_len);
+        co++;
+
+        if (dict_get_int32(xdata, "recon-xattr-opcode", &xattr_op) == 0)
+                CHANGELOG_FILL_FOP_NUMBER (co, (glusterfs_fop_t)xattr_op,
+                                           fop_fn, xtra_len);
+        else
+                CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn,
+                                           xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UUID (co, fd->inode->gfid, uuid_fn, xtra_len);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 4);
+
+        frame->local = local;
+        ret = 0;
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_METADATA);
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+int32_t
+changelog_replication_truncate (call_frame_t *frame,
+                                xlator_t *this, loc_t *loc,
+                                off_t offset, dict_t *xdata)
+{
+        int                ret      = -1;
+        size_t             xtra_len = 0;
+        changelog_priv_t *priv = NULL;
+        changelog_opt_t   *co       = NULL;
+        changelog_local_t *local    = NULL;
+
+        priv = this->private;
+
+        /* <PRE> + IDX + FOP + GFID + Offset */
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, loc->inode->gfid, 5);
+        if (!local)
+                goto out;
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        if (changelog_fix_term(this, local, xdata) == _gf_false)
+                goto out;
+
+        CHANGELOG_FILL_USIGNLL (co, PRE_OP_MARK, NULL, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, local->lu.val, uint32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UUID (co, loc->inode->gfid, uuid_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_USIGNLL (co, offset, number_fn, xtra_len);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 5);
+
+        frame->local = local;
+        ret = 0;
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+int32_t
+changelog_replication_ftruncate (call_frame_t *frame,
+                                 xlator_t *this, fd_t *fd,
+                                 off_t offset, dict_t *xdata)
+{
+        int                ret      = -1;
+        size_t             xtra_len = 0;
+        changelog_priv_t *priv = NULL;
+        changelog_opt_t   *co       = NULL;
+        changelog_local_t *local    = NULL;
+
+        priv = this->private;
+
+        /* <PRE> + IDX + FOP + GFID + Offset */
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, fd->inode->gfid, 5);
+        if (!local)
+                goto out;
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        if (changelog_fix_term(this, local, xdata) == _gf_false)
+                goto out;
+
+        CHANGELOG_FILL_USIGNLL (co, PRE_OP_MARK, NULL, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, local->lu.val, uint32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UUID (co, fd->inode->gfid, uuid_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_USIGNLL (co, offset, number_fn, xtra_len);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 5);
+
+        frame->local = local;
+        ret = 0;
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+int32_t
+changelog_replication_writev (call_frame_t *frame,
+                              xlator_t *this, fd_t *fd, struct iovec *vector,
+                              int32_t count, off_t offset, uint32_t flags,
+                              struct iobref *iobref, dict_t *xdata)
+{
+        int                ret      = -1;
+        size_t             xtra_len = 0;
+        changelog_priv_t *priv = NULL;
+        changelog_opt_t   *co       = NULL;
+        changelog_local_t *local    = NULL;
+
+        priv = this->private;
+
+        /* <PRE> + IDX + FOP + GFID + Offset + Length */
+        CHANGELOG_INIT_NOCHECK (this, local, NULL, fd->inode->gfid, 6);
+        if (!local)
+                goto out;
+
+        co = changelog_get_usable_buffer (local);
+        if (!co)
+                goto out;
+
+        if (changelog_fix_term(this, local, xdata) == _gf_false)
+                goto out;
+
+        CHANGELOG_FILL_USIGNLL (co, PRE_OP_MARK, NULL, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UINT32 (co, local->lu.val, uint32_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_FOP_NUMBER (co, frame->root->op, fop_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_UUID (co, fd->inode->gfid, uuid_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_USIGNLL (co, offset, number_fn, xtra_len);
+        co++;
+
+        CHANGELOG_FILL_USIGNLL (co, iov_length (vector, count),
+                               number_fn, xtra_len);
+
+        changelog_set_usable_record_and_length (local, xtra_len, 6);
+
+        frame->local = local;
+        ret = 0;
+
+        changelog_update (this, priv, local, CHANGELOG_TYPE_DATA);
+
+ out:
+        if (ret)
+                changelog_local_cleanup (this, local);
+        return ret;
+}
+
+/* overriden COPS */
+int
+changelog_replication_cops_open (xlator_t *this,
+                                 changelog_priv_t *priv, void *cpriv,
+                                 char *name, gf_boolean_t last)
+{
+        changelog_local_t local = {0,};
+        changelog_log_data_t       cld = {0,};
+        changelog_rollover_data_t *crd = NULL;
+
+        crd = &cld.cld_roll;
+
+        cld.cld_type = CHANGELOG_TYPE_ROLLOVER;
+
+        crd->crd_finale = last;
+        crd->crd_use_suffix = _gf_false;
+        crd->crd_prealloc_size = 1<<29; /* preallocate 512 MB */
+
+
+        (void) strcpy (crd->crd_changelog_name, name);
+
+        local.lu.val = 0;
+        local.nr_bytes = 0;
+
+        return changelog_inject_single_event (this, priv, &local, &cld);
+}
+
+/**
+ * NO-OP changelog write (from changelog.c). Records are journaled
+ * in the FOP path.
+ */
+int
+changelog_replication_cops_write (xlator_t *this,
+                                  changelog_priv_t *priv, void *cpriv,
+                                  changelog_local_t *local,
+                                  changelog_log_type type)
+{
+        return 0;
+}
+
+/**
+ * no implicit rollover
+ */
+int
+changelog_replication_cops_rollover (xlator_t *this,
+                                     changelog_priv_t *priv, void *cpriv,
+                                     char *name, gf_boolean_t last)
+{
+        return changelog_replication_cops_open (this, priv, cpriv, name, last);
+}
+
+off_t
+changelog_replication_cops_get_offset (xlator_t *this,
+                                       changelog_priv_t *priv, void *cpriv,
+                                       changelog_local_t *local)
+{
+        if (!local)
+                return 0;
+
+        return (local->lu.val * JOURNAL_SECTOR_SIZE) + local->nr_bytes;
+}
+
+void
+changelog_replication_cops_set_offset (xlator_t *this,
+                                       changelog_priv_t *priv, void *cpriv,
+                                       changelog_local_t *local, off_t bytes)
+{
+        local->nr_bytes += bytes;
+}
+
+void
+changelog_replication_cops_reset_offset (xlator_t *this, changelog_priv_t *priv,
+                                         void *cpriv, changelog_local_t *local)
+{
+        return;
+}
+
+int
+changelog_replication_policy_init (xlator_t *this,
+                                   changelog_priv_t *priv,
+                                   struct changelog_logpolicy *cp)
+{
+        struct xlator_fops   *r_fops = NULL;
+        struct changelog_ops *r_cops = NULL;
+
+        r_fops = GF_CALLOC (1, sizeof (struct xlator_fops),
+                            gf_changelog_mt_fop_policy_t);
+        if (!r_fops)
+                return -1;
+
+        r_cops = GF_CALLOC (1, sizeof (struct changelog_ops),
+                            gf_changelog_mt_fop_policy_t);
+        if (!r_cops) {
+                GF_FREE (r_fops);
+                return -1;
+        }
+
+        cp->cpriv = GF_CALLOC (1, sizeof (off_t),
+                               gf_changelog_mt_fop_policy_t);
+        if (!cp->cpriv) {
+                GF_FREE (r_fops);
+                GF_FREE (r_cops);
+                return -1;
+        }
+
+        /* no roll-over, one big fat journal per term */
+        priv->rollover_time = 0;
+
+        /* fsync() is internally trigerred by NSR */
+        priv->fsync_interval = 0;
+
+        /* no record header: extra data (via iobufs) are always persisted */
+        priv->no_gfid_hdr = _gf_true;
+
+        priv->lockless_update = _gf_false;
+
+        memcpy (r_fops, &changelog_default_fops, sizeof (struct xlator_fops));
+        memcpy (r_cops, &changelog_default_cops, sizeof (struct changelog_ops));
+
+        priv->term = 0;
+        (void) memset (cp->changelog_name, '\0', PATH_MAX);
+        memcpy(cp->changelog_name, JOURNAL_NAME, strlen(JOURNAL_NAME));
+#if 0
+        (void) snprintf (cp->changelog_name, PATH_MAX,
+                         JOURNAL_NAME, priv->term);
+#endif
+
+        /* overload all fops */
+        r_fops->writev       = changelog_replication_writev;
+        r_fops->ftruncate    = changelog_replication_ftruncate;
+        r_fops->truncate     = changelog_replication_truncate;
+        r_fops->fsetxattr    = changelog_replication_fsetxattr;
+        r_fops->setxattr     = changelog_replication_setxattr;
+        r_fops->removexattr  = changelog_replication_removexattr;
+        r_fops->fremovexattr = changelog_replication_fremovexattr;
+        r_fops->setattr      = changelog_replication_setattr;
+        r_fops->fsetattr     = changelog_replication_fsetattr;
+        r_fops->create       = changelog_replication_create;
+        r_fops->mknod        = changelog_replication_mknod;
+        r_fops->symlink      = changelog_replication_symlink;
+        r_fops->mkdir        = changelog_replication_mkdir;
+        r_fops->link         = changelog_replication_link;
+        r_fops->rename       = changelog_replication_rename;
+        r_fops->unlink       = changelog_replication_unlink;
+        r_fops->rmdir        = changelog_replication_rmdir;
+
+        /* overload cops */
+        r_cops->open         = changelog_replication_cops_open;
+        r_cops->write        = changelog_replication_cops_write;
+        r_cops->rollover     = changelog_replication_cops_rollover;
+        r_cops->get_offset   = changelog_replication_cops_get_offset;
+        r_cops->set_offset   = changelog_replication_cops_set_offset;
+        r_cops->reset_offset = changelog_replication_cops_reset_offset;
+
+
+        cp->fops = r_fops;
+        cp->cops = r_cops;
+
+        return 0;
+}
+
+int
+changelog_replication_policy_fini (xlator_t *this,
+                                   struct changelog_logpolicy *cp)
+{
+        GF_FREE (cp->fops);
+        GF_FREE (cp->cops);
+        GF_FREE (cp->cpriv);
+        return 0;
+}
diff --git a/xlators/features/changelog/src/policy/changelog-policy.h b/xlators/features/changelog/src/policy/changelog-policy.h
new file mode 100644
index 000000000..73fdc1a98
--- /dev/null
+++ b/xlators/features/changelog/src/policy/changelog-policy.h
@@ -0,0 +1,41 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CHANGELOG_POLICY_H
+#define _CHANGELOG_POLICY_H
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "logging.h"
+
+#include "changelog-mem-types.h"
+#include "changelog-helpers.h"
+
+int
+changelog_default_policy_init (xlator_t *this,
+                               changelog_priv_t *priv,
+                               struct changelog_logpolicy *);
+int
+changelog_default_policy_fini (xlator_t *this,
+                               struct changelog_logpolicy *);
+int
+changelog_replication_policy_init (xlator_t *this,
+                                   changelog_priv_t *priv,
+                                   struct changelog_logpolicy *cp);
+int
+changelog_replication_policy_fini (xlator_t *this,
+                                   struct changelog_logpolicy *cp);
+
+#endif /* _CHANGELOG_POLICY_H */
diff --git a/xlators/features/compress/Makefile.am b/xlators/features/compress/Makefile.am
new file mode 100644
index 000000000..a985f42a8
--- /dev/null
+++ b/xlators/features/compress/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/features/compress/src/Makefile.am b/xlators/features/compress/src/Makefile.am
new file mode 100644
index 000000000..263b21b78
--- /dev/null
+++ b/xlators/features/compress/src/Makefile.am
@@ -0,0 +1,17 @@
+xlator_LTLIBRARIES = cdc.la
+
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+noinst_HEADERS = cdc.h cdc-mem-types.h
+
+cdc_la_LDFLAGS = -module -avoid-version $(LIBZ_LIBS)
+
+cdc_la_SOURCES = cdc.c cdc-helper.c
+cdc_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -D$(GF_HOST_OS) \
+-shared $(LIBZ_CFLAGS)
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/compress/src/cdc-helper.c b/xlators/features/compress/src/cdc-helper.c
new file mode 100644
index 000000000..54432ff45
--- /dev/null
+++ b/xlators/features/compress/src/cdc-helper.c
@@ -0,0 +1,547 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+
+#include "cdc.h"
+#include "cdc-mem-types.h"
+
+#ifdef HAVE_LIB_Z
+#include "zlib.h"
+#endif
+
+#ifdef HAVE_LIB_Z
+/* gzip header looks something like this
+ * (RFC 1950)
+ *
+ * +---+---+---+---+---+---+---+---+---+---+
+ * |ID1|ID2|CM |FLG|     MTIME     |XFL|OS |
+ * +---+---+---+---+---+---+---+---+---+---+
+ *
+ * Data is usually sent without this header i.e
+ * Data sent = <compressed-data> + trailer(8)
+ * The trailer contains the checksum.
+ *
+ * gzip_header is added only during debugging.
+ * Refer to the function cdc_dump_iovec_to_disk
+ */
+static const char gzip_header[10] =
+        {
+                '\037', '\213', Z_DEFLATED, 0,
+                0,      0,      0,          0,
+                0,      GF_CDC_OS_ID
+        };
+
+static int32_t
+cdc_next_iovec (xlator_t *this, cdc_info_t *ci)
+{
+        int ret = -1;
+
+        ci->ncount++;
+        /* check for iovec overflow -- should not happen */
+        if (ci->ncount == MAX_IOVEC) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Zlib output buffer overflow"
+                        " ->ncount (%d) | ->MAX_IOVEC (%d)",
+                        ci->ncount, MAX_IOVEC);
+                goto out;
+        }
+
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+static void
+cdc_put_long (unsigned char *string, unsigned long x)
+{
+        string[0] = (unsigned char) (x & 0xff);
+        string[1] = (unsigned char) ((x & 0xff00) >> 8);
+        string[2] = (unsigned char) ((x & 0xff0000) >> 16);
+        string[3] = (unsigned char) ((x & 0xff000000) >> 24);
+}
+
+static unsigned long
+cdc_get_long (unsigned char *buf)
+{
+        return ((unsigned long) buf[0])
+                | (((unsigned long) buf[1]) << 8)
+                | (((unsigned long) buf[2]) << 16)
+                | (((unsigned long) buf[3]) << 24);
+}
+
+static int32_t
+cdc_init_gzip_trailer (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
+{
+        int   ret = -1;
+        char *buf = NULL;
+
+        ret = cdc_next_iovec (this, ci);
+        if (ret)
+                goto out;
+
+        buf = CURR_VEC(ci).iov_base =
+                (char *) GF_CALLOC (1, GF_CDC_VALIDATION_SIZE,
+                                    gf_cdc_mt_gzip_trailer_t);
+
+        if (!CURR_VEC(ci).iov_base)
+                goto out;
+
+        CURR_VEC(ci).iov_len = GF_CDC_VALIDATION_SIZE;
+
+        cdc_put_long ((unsigned char *)&buf[0], ci->crc);
+        cdc_put_long ((unsigned char *)&buf[4], ci->stream.total_in);
+
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+static int32_t
+cdc_alloc_iobuf_and_init_vec (xlator_t *this,
+                              cdc_priv_t *priv, cdc_info_t *ci,
+                              int size)
+{
+        int           ret       = -1;
+        int           alloc_len = 0;
+        struct iobuf *iobuf     = NULL;
+
+        ret = cdc_next_iovec (this, ci);
+        if (ret)
+                goto out;
+
+        alloc_len = size ? size : ci->buffer_size;
+
+        iobuf = iobuf_get2 (this->ctx->iobuf_pool, alloc_len);
+        if (!iobuf)
+                goto out;
+
+        ret = iobref_add (ci->iobref, iobuf);
+        if (ret)
+                goto out;
+
+        /* Initialize this iovec */
+        CURR_VEC(ci).iov_base = iobuf->ptr;
+        CURR_VEC(ci).iov_len  = alloc_len;
+
+        ret = 0;
+
+ out:
+        return ret;
+}
+
+static void
+cdc_init_zlib_output_stream (cdc_priv_t *priv, cdc_info_t *ci, int size)
+{
+        ci->stream.next_out  = (unsigned char *) CURR_VEC(ci).iov_base;
+        ci->stream.avail_out = size ? size : ci->buffer_size;
+}
+
+/* This routine is for testing and debugging only.
+ * Data written = header(10) + <compressed-data> + trailer(8)
+ * So each gzip dump file is at least 18 bytes in size.
+ */
+void
+cdc_dump_iovec_to_disk (xlator_t *this, cdc_info_t *ci, const char *file)
+{
+        int    i      = 0;
+        int    fd     = 0;
+        size_t writen = 0;
+        size_t total_writen = 0;
+
+        fd = open (file, O_WRONLY|O_CREAT|O_TRUNC, 0777 );
+        if (fd < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Cannot open file: %s", file);
+                return;
+        }
+
+        writen = write (fd, (char *) gzip_header, 10);
+        total_writen += writen;
+        for (i = 0; i < ci->ncount; i++) {
+                writen = write (fd, (char *) ci->vec[i].iov_base, ci->vec[i].iov_len);
+                total_writen += writen;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                        "dump'd %zu bytes to %s", total_writen, GF_CDC_DEBUG_DUMP_FILE );
+
+        close (fd);
+}
+
+static int32_t
+cdc_flush_libz_buffer (cdc_priv_t *priv, xlator_t *this, cdc_info_t *ci,
+                       int (*libz_func)(z_streamp, int),
+                       int flush)
+{
+        int32_t ret = Z_OK;
+        int done = 0;
+        unsigned int deflate_len = 0;
+
+        for (;;) {
+                deflate_len = ci->buffer_size - ci->stream.avail_out;
+
+                if (deflate_len != 0) {
+                        CURR_VEC(ci).iov_len = deflate_len;
+
+                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+                        if (ret) {
+                                ret = Z_MEM_ERROR;
+                                break;
+                        }
+
+                        /* Re-position Zlib output buffer */
+                        cdc_init_zlib_output_stream (priv, ci, 0);
+                }
+
+                if (done) {
+                        ci->ncount--;
+                        break;
+                }
+
+                ret = libz_func (&ci->stream, flush);
+
+                if (ret == Z_BUF_ERROR) {
+                        ret = Z_OK;
+                        ci->ncount--;
+                        break;
+                }
+
+                done = (ci->stream.avail_out != 0 || ret == Z_STREAM_END);
+
+                if (ret != Z_OK && ret != Z_STREAM_END)
+                        break;
+        }
+
+        return ret;
+}
+
+static int32_t
+do_cdc_compress (struct iovec *vec, xlator_t *this, cdc_priv_t *priv,
+                 cdc_info_t *ci)
+{
+        int ret = -1;
+
+        /* Initialize defalte */
+        ret = deflateInit2 (&ci->stream, priv->cdc_level, Z_DEFLATED,
+                            priv->window_size, priv->mem_level,
+                            Z_DEFAULT_STRATEGY);
+
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "unable to init Zlib (retval: %d)", ret);
+                goto out;
+        }
+
+        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+        if (ret)
+                goto out;
+
+        /* setup output buffer */
+        cdc_init_zlib_output_stream (priv, ci, 0);
+
+        /* setup input buffer */
+        ci->stream.next_in  = (unsigned char *) vec->iov_base;
+        ci->stream.avail_in = vec->iov_len;
+
+        ci->crc = crc32 (ci->crc, (const Bytef *) vec->iov_base, vec->iov_len);
+
+        gf_log (this->name, GF_LOG_DEBUG, "crc=%lu len=%d buffer_size=%d",
+                ci->crc, ci->stream.avail_in, ci->buffer_size);
+
+        /* compress !! */
+        while (ci->stream.avail_in != 0) {
+                if (ci->stream.avail_out == 0) {
+
+                        CURR_VEC(ci).iov_len = ci->buffer_size;
+
+                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+                        if (ret)
+                                break;
+
+                        /* Re-position Zlib output buffer */
+                        cdc_init_zlib_output_stream (priv, ci, 0);
+                }
+
+                ret = deflate (&ci->stream, Z_NO_FLUSH);
+                if (ret != Z_OK)
+                        break;
+        }
+
+ out:
+        return ret;
+}
+
+int32_t
+cdc_compress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci,
+              dict_t **xdata)
+{
+        int ret = -1;
+        int i   = 0;
+
+        ci->iobref = iobref_new ();
+        if (!ci->iobref)
+                goto out;
+
+        if (!*xdata) {
+                *xdata = dict_new ();
+                if (!*xdata) {
+                        gf_log (this->name, GF_LOG_ERROR, "Cannot allocate xdata"
+                                " dict");
+                        goto out;
+                }
+        }
+
+        /* data */
+        for (i = 0; i < ci->count; i++) {
+                ret = do_cdc_compress (&ci->vector[i], this, priv, ci);
+                if (ret != Z_OK)
+                        goto deflate_cleanup_out;
+        }
+
+        /* flush zlib buffer */
+        ret = cdc_flush_libz_buffer (priv, this, ci, deflate, Z_FINISH);
+        if (!(ret == Z_OK || ret == Z_STREAM_END)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Compression Error: ret (%d)", ret);
+                ret = -1;
+                goto deflate_cleanup_out;
+        }
+
+        /* trailer */
+        ret = cdc_init_gzip_trailer (this, priv, ci);
+        if (ret)
+                goto deflate_cleanup_out;
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Compressed %ld to %ld bytes",
+                ci->stream.total_in, ci->stream.total_out);
+
+        ci->nbytes = ci->stream.total_out + GF_CDC_VALIDATION_SIZE;
+
+        /* set deflated canary value for identification */
+        ret = dict_set_int32 (*xdata, GF_CDC_DEFLATE_CANARY_VAL, 1);
+        if (ret) {
+                /* Send uncompressed data if we can't _tell_ the client
+                 * that deflated data is on it's way. So, we just log
+                 * the faliure and continue as usual.
+                 */
+                 gf_log (this->name, GF_LOG_ERROR,
+                 "Data deflated, but could not set canary"
+                 " value in dict for identification");
+        }
+
+        /* This is to be used in testing */
+        if ( priv->debug ) {
+                cdc_dump_iovec_to_disk (this, ci, GF_CDC_DEBUG_DUMP_FILE );
+        }
+
+ deflate_cleanup_out:
+        (void) deflateEnd(&ci->stream);
+
+ out:
+        return ret;
+}
+
+
+/* deflate content is checked by the presence of a canary
+ * value in the dict as the key
+ */
+static int32_t
+cdc_check_content_for_deflate (dict_t *xdata)
+{
+        return dict_get (xdata, GF_CDC_DEFLATE_CANARY_VAL) ? -1 : 0;
+}
+
+static unsigned long
+cdc_extract_crc (char *trailer)
+{
+        return cdc_get_long ((unsigned char *) &trailer[0]);
+}
+
+static unsigned long
+cdc_extract_size (char *trailer)
+{
+        return cdc_get_long ((unsigned char *) &trailer[4]);
+}
+
+static int32_t
+cdc_validate_inflate (cdc_info_t *ci, unsigned long crc,
+                      unsigned long len)
+{
+        return !((crc == ci->crc)
+                 /* inflated length is hidden inside
+                  * Zlib stream struct */
+                 && (len == ci->stream.total_out));
+}
+
+static int32_t
+do_cdc_decompress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci)
+{
+        int            ret          = -1;
+        int            i            = 0;
+        int            len          = 0;
+        char          *inflte       = NULL;
+        char          *trailer      = NULL;
+        struct iovec   vec          = {0,};
+        unsigned long  computed_crc = 0;
+        unsigned long  computed_len = 0;
+
+        ret = inflateInit2 (&ci->stream, priv->window_size);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Zlib: Unable to initialize inflate");
+                goto out;
+        }
+
+        vec = THIS_VEC(ci, 0);
+
+        trailer = (char *) (((char *) vec.iov_base) + vec.iov_len
+                - GF_CDC_VALIDATION_SIZE);
+
+        /* CRC of uncompressed data */
+        computed_crc = cdc_extract_crc (trailer);
+
+        /* size of uncomrpessed data */
+        computed_len = cdc_extract_size (trailer);
+
+        gf_log (this->name, GF_LOG_DEBUG, "crc=%lu len=%lu buffer_size=%d",
+                computed_crc, computed_len, ci->buffer_size);
+
+        inflte = vec.iov_base ;
+        len = vec.iov_len - GF_CDC_VALIDATION_SIZE;
+
+        /* allocate buffer of the original length of the data */
+        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+        if (ret)
+                goto out;
+
+        /* setup output buffer */
+        cdc_init_zlib_output_stream (priv, ci, 0);
+
+        /* setup input buffer */
+        ci->stream.next_in = (unsigned char *) inflte;
+        ci->stream.avail_in = len;
+
+        while (ci->stream.avail_in != 0) {
+                if (ci->stream.avail_out == 0) {
+                        CURR_VEC(ci).iov_len = ci->buffer_size;
+
+                        ret = cdc_alloc_iobuf_and_init_vec (this, priv, ci, 0);
+                        if (ret)
+                                break;
+
+                        /* Re-position Zlib output buffer */
+                        cdc_init_zlib_output_stream (priv, ci, 0);
+                }
+
+                ret = inflate (&ci->stream, Z_NO_FLUSH);
+                if (ret == Z_STREAM_ERROR)
+                        break;
+        }
+
+        /* flush zlib buffer */
+        ret = cdc_flush_libz_buffer (priv, this, ci, inflate, Z_SYNC_FLUSH);
+        if (!(ret == Z_OK || ret == Z_STREAM_END)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Decompression Error: ret (%d)", ret);
+                ret = -1;
+                goto out;
+        }
+
+        /* compute CRC of the uncompresses data to check for
+         * correctness */
+
+        for (i = 0; i < ci->ncount; i++) {
+                ci->crc = crc32 (ci->crc,
+                                 (const Bytef *) ci->vec[i].iov_base,
+                                 ci->vec[i].iov_len);
+        }
+
+        /* validate inflated data */
+        ret = cdc_validate_inflate (ci, computed_crc, computed_len);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Checksum or length mismatched in inflated data");
+        }
+
+ out:
+        return ret;
+}
+
+int32_t
+cdc_decompress (xlator_t *this, cdc_priv_t *priv, cdc_info_t *ci,
+                dict_t *xdata)
+{
+        int32_t ret = -1;
+
+        /* check for deflate content */
+        if (!cdc_check_content_for_deflate (xdata)) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Content not deflated, passing through ...");
+                goto passthrough_out;
+        }
+
+        ci->iobref = iobref_new ();
+        if (!ci->iobref)
+                goto passthrough_out;
+
+        /* do we need to do this? can we assume that one iovec
+         * will hold per request data everytime?
+         *
+         * server/client protocol seems to deal with a single
+         * iovec even if op_ret > 1M. So, it looks ok to
+         * assume that a single iovec will contain all the
+         * data (This saves us a lot from finding the trailer
+         * and the data since it could have been split-up onto
+         * two adjacent iovec's.
+         *
+         * But, in case this translator is loaded above quick-read
+         * for some reason, then it's entirely possible that we get
+         * multiple iovec's...
+         *
+         * This case (handled below) is not tested. (by loading the
+         * xlator below quick-read)
+         */
+
+        /* @@ I_HOPE_THIS_IS_NEVER_HIT */
+        if (ci->count > 1) {
+                gf_log (this->name, GF_LOG_WARNING, "unable to handle"
+                        " multiple iovecs (%d in number)", ci->count);
+                goto inflate_cleanup_out;
+                /* TODO: coallate all iovecs in one */
+        }
+
+        ret = do_cdc_decompress (this, priv, ci);
+        if (ret)
+                goto inflate_cleanup_out;
+
+        ci->nbytes = ci->stream.total_out;
+
+        gf_log (this->name, GF_LOG_DEBUG,
+                "Inflated %ld to %ld bytes",
+                ci->stream.total_in, ci->stream.total_out);
+
+ inflate_cleanup_out:
+        (void) inflateEnd (&ci->stream);
+
+ passthrough_out:
+        return ret;
+}
+
+#endif
diff --git a/xlators/features/compress/src/cdc-mem-types.h b/xlators/features/compress/src/cdc-mem-types.h
new file mode 100644
index 000000000..ead2c70ba
--- /dev/null
+++ b/xlators/features/compress/src/cdc-mem-types.h
@@ -0,0 +1,23 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CDC_MEM_TYPES_H
+#define __CDC_MEM_TYPES_H
+
+#include "mem-types.h"
+
+enum gf_cdc_mem_types {
+        gf_cdc_mt_priv_t         = gf_common_mt_end + 1,
+        gf_cdc_mt_vec_t          = gf_common_mt_end + 2,
+        gf_cdc_mt_gzip_trailer_t = gf_common_mt_end + 3,
+        gf_cdc_mt_end            = gf_common_mt_end + 4,
+};
+
+#endif
diff --git a/xlators/features/compress/src/cdc.c b/xlators/features/compress/src/cdc.c
new file mode 100644
index 000000000..67fc52505
--- /dev/null
+++ b/xlators/features/compress/src/cdc.c
@@ -0,0 +1,361 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#include <sys/uio.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "xlator.h"
+#include "defaults.h"
+#include "logging.h"
+
+#include "cdc.h"
+#include "cdc-mem-types.h"
+
+static void
+cdc_cleanup_iobref (cdc_info_t *ci)
+{
+        assert(ci->iobref != NULL);
+        iobref_clear (ci->iobref);
+}
+
+int32_t
+cdc_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno,
+               struct iovec *vector, int32_t count,
+               struct iatt *stbuf, struct iobref *iobref,
+               dict_t *xdata)
+{
+        int         ret   = -1;
+        cdc_priv_t *priv  = NULL;
+        cdc_info_t  ci    = {0,};
+
+        GF_VALIDATE_OR_GOTO ("cdc", this, default_out);
+        GF_VALIDATE_OR_GOTO (this->name, frame, default_out);
+
+        priv = this->private;
+
+        if (op_ret <= 0)
+                goto default_out;
+
+        if ( (priv->min_file_size != 0)
+             && (op_ret < priv->min_file_size) )
+                goto default_out;
+
+        ci.count       = count;
+        ci.ibytes      = op_ret;
+        ci.vector      = vector;
+        ci.buf         = NULL;
+        ci.iobref      = NULL;
+        ci.ncount      = 0;
+        ci.crc         = 0;
+        ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
+
+/* A readv compresses on the server side and decompresses on the client side
+ */
+        if (priv->op_mode == GF_CDC_MODE_SERVER) {
+                ret = cdc_compress (this, priv, &ci, &xdata);
+        } else if (priv->op_mode == GF_CDC_MODE_CLIENT) {
+                ret = cdc_decompress (this, priv, &ci, xdata);
+        } else {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Invalid operation mode (%d)", priv->op_mode);
+        }
+
+        if (ret)
+                goto default_out;
+
+        STACK_UNWIND_STRICT (readv, frame, ci.nbytes, op_errno,
+                             ci.vec, ci.ncount, stbuf, iobref,
+                             xdata);
+        cdc_cleanup_iobref (&ci);
+        return 0;
+
+ default_out:
+        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
+                             vector, count, stbuf, iobref, xdata);
+        return 0;
+}
+
+int32_t
+cdc_readv (call_frame_t *frame, xlator_t *this,
+           fd_t *fd, size_t size, off_t offset, uint32_t flags,
+           dict_t *xdata)
+{
+        fop_readv_cbk_t cbk = NULL;
+
+#ifdef HAVE_LIB_Z
+        cbk = cdc_readv_cbk;
+#else
+        cbk = default_readv_cbk;
+#endif
+        STACK_WIND (frame, cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readv,
+                    fd, size, offset, flags, xdata);
+        return 0;
+}
+
+int32_t
+cdc_writev_cbk (call_frame_t *frame,
+                void *cookie,
+                xlator_t *this,
+                int32_t op_ret,
+                int32_t op_errno,
+                struct iatt *prebuf,
+                struct iatt *postbuf, dict_t *xdata)
+{
+
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf, xdata);
+        return 0;
+}
+
+int32_t
+cdc_writev (call_frame_t *frame,
+            xlator_t *this,
+            fd_t *fd,
+            struct iovec *vector,
+            int32_t count,
+            off_t offset,
+            uint32_t flags,
+            struct iobref *iobref, dict_t *xdata)
+{
+        int          ret   = -1;
+        cdc_priv_t  *priv  = NULL;
+        cdc_info_t   ci    = {0,};
+        size_t       isize = 0;
+
+        GF_VALIDATE_OR_GOTO ("cdc", this, default_out);
+        GF_VALIDATE_OR_GOTO (this->name, frame, default_out);
+
+        priv = this->private;
+
+        isize =  iov_length(vector, count);
+
+        if (isize <= 0)
+            goto default_out;
+
+        if ( (priv->min_file_size != 0) && (isize < priv->min_file_size) )
+            goto default_out;
+
+        ci.count       = count;
+        ci.ibytes      = isize;
+        ci.vector      = vector;
+        ci.buf         = NULL;
+        ci.iobref      = NULL;
+        ci.ncount      = 0;
+        ci.crc         = 0;
+        ci.buffer_size = GF_CDC_DEF_BUFFERSIZE;
+
+/* A writev compresses on the client side and decompresses on the server side
+ */
+	    if (priv->op_mode == GF_CDC_MODE_CLIENT) {
+		    ret = cdc_compress (this, priv, &ci, &xdata);
+	    } else if (priv->op_mode == GF_CDC_MODE_SERVER) {
+		    ret = cdc_decompress (this, priv, &ci, xdata);
+	    } else {
+		    gf_log (this->name, GF_LOG_ERROR, "Invalid operation mode (%d) ", priv->op_mode);
+	    }
+
+	    if (ret)
+		    goto default_out;
+
+	    STACK_WIND (frame,
+                    cdc_writev_cbk,
+                    FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->writev,
+                    fd, ci.vec, ci.ncount, offset, flags,
+                    iobref, xdata);
+
+        cdc_cleanup_iobref (&ci);
+        return 0;
+
+ default_out:
+        STACK_WIND (frame,
+                    cdc_writev_cbk,
+                    FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->writev,
+                    fd, vector, count, offset, flags,
+                    iobref, xdata);
+        return 0;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_cdc_mt_end);
+
+        if (ret != 0) {
+                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
+                       "failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+int32_t
+init (xlator_t *this)
+{
+        int         ret      = -1;
+        char       *temp_str = NULL;
+        cdc_priv_t *priv     = NULL;
+
+        GF_VALIDATE_OR_GOTO ("cdc", this, err);
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Need subvolume == 1");
+                goto err;
+        }
+
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Dangling volume. Check volfile");
+        }
+
+        priv = GF_CALLOC (1, sizeof (*priv), gf_cdc_mt_priv_t);
+        if (!priv) {
+                goto err;
+        }
+
+        /* Check if debug mode is turned on */
+        GF_OPTION_INIT ("debug", priv->debug, bool, err);
+        if( priv->debug ) {
+                gf_log (this->name, GF_LOG_DEBUG, "CDC debug option turned on");
+        }
+
+        /* Set Gzip Window Size */
+        GF_OPTION_INIT ("window-size", priv->window_size, int32, err);
+        if ( (priv->window_size > GF_CDC_MAX_WINDOWSIZE)
+             || (priv->window_size < GF_CDC_DEF_WINDOWSIZE) ) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Invalid gzip window size (%d), using default",
+                        priv->window_size);
+                priv->window_size = GF_CDC_DEF_WINDOWSIZE;
+        }
+
+        /* Set Gzip (De)Compression Level */
+        GF_OPTION_INIT ("compression-level", priv->cdc_level, int32, err);
+        if ( ((priv->cdc_level < 1) || (priv->cdc_level > 9))
+             && (priv->cdc_level != GF_CDC_DEF_COMPRESSION) ) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Invalid gzip (de)compression level (%d),"
+                        " using default", priv->cdc_level);
+                priv->cdc_level = GF_CDC_DEF_COMPRESSION;
+        }
+
+        /* Set Gzip Memory Level */
+        GF_OPTION_INIT ("mem-level", priv->mem_level, int32, err);
+        if ( (priv->mem_level < 1) || (priv->mem_level > 9) ) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Invalid gzip memory level, using the default");
+                priv->mem_level = GF_CDC_DEF_MEMLEVEL;
+        }
+
+        /* Set min file size to enable compression */
+        GF_OPTION_INIT ("min-size", priv->min_file_size, int32, err);
+
+        /* Mode of operation - Server/Client */
+        ret = dict_get_str (this->options, "mode", &temp_str);
+        if (ret) {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Operation mode not specified !!");
+                goto err;
+        }
+
+        if (GF_CDC_MODE_IS_CLIENT (temp_str)) {
+                priv->op_mode = GF_CDC_MODE_CLIENT;
+        } else if (GF_CDC_MODE_IS_SERVER (temp_str)) {
+                priv->op_mode = GF_CDC_MODE_SERVER;
+        } else {
+                gf_log (this->name, GF_LOG_CRITICAL,
+                        "Bogus operation mode (%s) specified", temp_str);
+                goto err;
+        }
+
+        this->private = priv;
+        gf_log (this->name, GF_LOG_DEBUG, "CDC xlator loaded in (%s) mode",temp_str);
+        return 0;
+
+ err:
+        if (priv)
+                GF_FREE (priv);
+
+        return -1;
+}
+
+void
+fini (xlator_t *this)
+{
+        cdc_priv_t *priv = this->private;
+
+        if (priv)
+                GF_FREE (priv);
+        this->private = NULL;
+        return;
+}
+
+struct xlator_fops fops = {
+        .readv  = cdc_readv,
+        .writev = cdc_writev,
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+        { .key  = {"window-size"},
+          .default_value = "-15",
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Size of the zlib history buffer."
+        },
+        { .key  = {"mem-level"},
+          .default_value = "8",
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Memory allocated for internal compression state. "
+                         "1 uses minimum memory but is slow and reduces "
+                         "compression ratio; memLevel=9 uses maximum memory "
+                         "for optimal speed. The default value is 8."
+        },
+        { .key  = {"compression-level"},
+          .default_value = "-1",
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Compression levels \n"
+                         "0 : no compression, 1 : best speed, \n"
+                         "9 : best compression, -1 : default compression "
+        },
+        { .key  = {"min-size"},
+          .default_value = "0",
+          .type = GF_OPTION_TYPE_INT,
+          .description = "Data is compressed only when its size exceeds this."
+        },
+        { .key  = {"mode"},
+          .value = {"server", "client"},
+          .type = GF_OPTION_TYPE_STR,
+          .description = "Set on the basis of where the xlator is loaded. "
+                         "This option should NOT be configured by user."
+        },
+        { .key = {"debug"},
+          .default_value = "false",
+          .type = GF_OPTION_TYPE_BOOL,
+          .description = "This is used in testing. Will dump compressed data "
+                         "to disk as a gzip file."
+        },
+        { .key  = {NULL}
+        },
+};
diff --git a/xlators/features/compress/src/cdc.h b/xlators/features/compress/src/cdc.h
new file mode 100644
index 000000000..71f4d2317
--- /dev/null
+++ b/xlators/features/compress/src/cdc.h
@@ -0,0 +1,107 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __CDC_H
+#define __CDC_H
+
+#ifdef HAVE_LIB_Z
+#include "zlib.h"
+#endif
+
+#include "xlator.h"
+
+#ifndef MAX_IOVEC
+#define MAX_IOVEC 16
+#endif
+
+typedef struct cdc_priv {
+        int window_size;
+        int mem_level;
+        int cdc_level;
+        int min_file_size;
+        int op_mode;
+        gf_boolean_t debug;
+        gf_lock_t lock;
+} cdc_priv_t;
+
+typedef struct cdc_info {
+        /* input bits */
+        int            count;
+        int32_t        ibytes;
+        struct iovec  *vector;
+        struct iatt   *buf;
+
+        /* output bits */
+        int            ncount;
+        int            nbytes;
+        int            buffer_size;
+        struct iovec   vec[MAX_IOVEC];
+        struct iobref *iobref;
+
+        /* zlib bits */
+#ifdef HAVE_LIB_Z
+        z_stream      stream;
+#endif
+        unsigned long crc;
+} cdc_info_t;
+
+#define NVEC(ci) (ci->ncount - 1)
+#define CURR_VEC(ci) ci->vec[ci->ncount - 1]
+#define THIS_VEC(ci, i) ci->vector[i]
+
+/* Gzip defaults */
+#define GF_CDC_DEF_WINDOWSIZE  -15 /* default value */
+#define GF_CDC_MAX_WINDOWSIZE  -8  /* max value     */
+
+#ifdef HAVE_LIB_Z
+#define GF_CDC_DEF_COMPRESSION Z_DEFAULT_COMPRESSION
+#else
+#define GF_CDC_DEF_COMPRESSION -1
+#endif
+
+#define GF_CDC_DEF_MEMLEVEL    8
+#define GF_CDC_DEF_BUFFERSIZE  262144 // 256K - default compression buffer size
+
+/* Operation mode
+ * If xlator is loaded on client, readv decompresses and writev compresses
+ * If xlator is loaded on server, readv compresses and writev decompresses
+ */
+#define GF_CDC_MODE_CLIENT   0
+#define GF_CDC_MODE_SERVER   1
+
+/* min size of data to do cmpression
+ * 0 == compress even 1byte
+ */
+#define GF_CDC_MIN_CHUNK_SIZE 0
+
+#define GF_CDC_VALIDATION_SIZE 8
+
+#define GF_CDC_OS_ID 0xFF
+#define GF_CDC_DEFLATE_CANARY_VAL "deflate"
+#define GF_CDC_DEBUG_DUMP_FILE "/tmp/cdcdump.gz"
+
+#define GF_CDC_MODE_IS_CLIENT(m) \
+        (strcmp (m, "client") == 0)
+
+#define GF_CDC_MODE_IS_SERVER(m) \
+        (strcmp (m, "server") == 0)
+
+int32_t
+cdc_compress (xlator_t *this,
+              cdc_priv_t *priv,
+              cdc_info_t *ci,
+              dict_t **xdata);
+int32_t
+cdc_decompress (xlator_t *this,
+                cdc_priv_t *priv,
+                cdc_info_t *ci,
+                dict_t *xdata);
+
+#endif
diff --git a/xlators/features/filter/src/Makefile.am b/xlators/features/filter/src/Makefile.am
index d473b9ea1..d1fda8b0a 100644
--- a/xlators/features/filter/src/Makefile.am
+++ b/xlators/features/filter/src/Makefile.am
@@ -1,15 +1,16 @@
 xlator_LTLIBRARIES = filter.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/testing/features
 
-filter_la_LDFLAGS = -module -avoidversion
+filter_la_LDFLAGS = -module -avoid-version
 
 filter_la_SOURCES = filter.c
 filter_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
 
 noinst_HEADERS = filter-mem-types.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
 
 CLEANFILES = 
 
diff --git a/xlators/features/filter/src/filter-mem-types.h b/xlators/features/filter/src/filter-mem-types.h
index db41a8877..47a17249b 100644
--- a/xlators/features/filter/src/filter-mem-types.h
+++ b/xlators/features/filter/src/filter-mem-types.h
@@ -1,22 +1,12 @@
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef __FILTER_MEM_TYPES_H__
 #define __FILTER_MEM_TYPES_H__
 
diff --git a/xlators/features/filter/src/filter.c b/xlators/features/filter/src/filter.c
index a813e0c63..1d4887b71 100644
--- a/xlators/features/filter/src/filter.c
+++ b/xlators/features/filter/src/filter.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
diff --git a/xlators/bindings/python/Makefile.am b/xlators/features/gfid-access/Makefile.am
index af437a64d..af437a64d 100644
--- a/xlators/bindings/python/Makefile.am
+++ b/xlators/features/gfid-access/Makefile.am
diff --git a/xlators/features/gfid-access/src/Makefile.am b/xlators/features/gfid-access/src/Makefile.am
new file mode 100644
index 000000000..db53affaa
--- /dev/null
+++ b/xlators/features/gfid-access/src/Makefile.am
@@ -0,0 +1,15 @@
+xlator_LTLIBRARIES = gfid-access.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+gfid_access_la_LDFLAGS = -module -avoid-version
+
+gfid_access_la_SOURCES = gfid-access.c
+gfid_access_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = gfid-access.h gfid-access-mem-types.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/gfid-access/src/gfid-access-mem-types.h b/xlators/features/gfid-access/src/gfid-access-mem-types.h
new file mode 100644
index 000000000..168d67b43
--- /dev/null
+++ b/xlators/features/gfid-access/src/gfid-access-mem-types.h
@@ -0,0 +1,23 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _GFID_ACCESS_MEM_TYPES_H
+#define _GFID_ACCESS_MEM_TYPES_H
+
+#include "mem-types.h"
+
+enum gf_changelog_mem_types {
+        gf_gfid_access_mt_priv_t = gf_common_mt_end + 1,
+        gf_gfid_access_mt_gfid_t,
+        gf_gfid_access_mt_end
+};
+
+#endif
+
diff --git a/xlators/features/gfid-access/src/gfid-access.c b/xlators/features/gfid-access/src/gfid-access.c
new file mode 100644
index 000000000..5cb6ecfbd
--- /dev/null
+++ b/xlators/features/gfid-access/src/gfid-access.c
@@ -0,0 +1,1299 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "gfid-access.h"
+#include "inode.h"
+#include "byte-order.h"
+
+
+
+void
+ga_newfile_args_free (ga_newfile_args_t *args)
+{
+        if (!args)
+                goto out;
+
+        GF_FREE (args->bname);
+
+        if (S_ISLNK (args->st_mode) && args->args.symlink.linkpath) {
+                GF_FREE (args->args.symlink.linkpath);
+                args->args.symlink.linkpath = NULL;
+        }
+
+        mem_put (args);
+out:
+        return;
+}
+
+
+void
+ga_heal_args_free (ga_heal_args_t *args)
+{
+        if (!args)
+                goto out;
+
+        GF_FREE (args->bname);
+
+        mem_put (args);
+out:
+        return;
+}
+
+
+ga_newfile_args_t *
+ga_newfile_parse_args (xlator_t *this, data_t *data)
+{
+        ga_newfile_args_t *args     = NULL;
+        ga_private_t      *priv     = NULL;
+        int                len      = 0;
+        int                blob_len = 0;
+        int                min_len  = 0;
+        void              *blob     = NULL;
+
+        priv = this->private;
+
+        blob = data->data;
+        blob_len = data->len;
+
+        min_len = sizeof (args->uid) + sizeof (args->gid) + sizeof (args->gfid)
+                + sizeof (args->st_mode) + 2 + 2;
+        if (blob_len < min_len) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Invalid length: Total length is less "
+                        "than minimum length.");
+                goto err;
+        }
+
+        args = mem_get0 (priv->newfile_args_pool);
+        if (args == NULL)
+                goto err;
+
+        args->uid = ntoh32 (*(uint32_t *)blob);
+        blob += sizeof (uint32_t);
+        blob_len -= sizeof (uint32_t);
+
+        args->gid = ntoh32 (*(uint32_t *)blob);
+        blob += sizeof (uint32_t);
+        blob_len -= sizeof (uint32_t);
+
+        memcpy (args->gfid, blob, sizeof (args->gfid));
+        blob += sizeof (args->gfid);
+        blob_len -= sizeof (args->gfid);
+
+        args->st_mode = ntoh32 (*(uint32_t *)blob);
+        blob += sizeof (uint32_t);
+        blob_len -= sizeof (uint32_t);
+
+        len = strnlen (blob, blob_len);
+        if (len == blob_len)
+        if (len == blob_len) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "gfid: %s. No null byte present.",
+                        args->gfid);
+                goto err;
+        }
+
+        args->bname = GF_CALLOC (1, (len + 1), gf_common_mt_char);
+        if (args->bname == NULL)
+                goto err;
+
+        memcpy (args->bname, blob, (len + 1));
+        blob += (len + 1);
+        blob_len -= (len + 1);
+
+        if (S_ISDIR (args->st_mode)) {
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.mkdir.mode = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.mkdir.umask = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+                if (blob_len < 0) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+        } else if (S_ISLNK (args->st_mode)) {
+                len = strnlen (blob, blob_len);
+                if (len == blob_len) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.symlink.linkpath = GF_CALLOC (1, len + 1,
+                                                         gf_common_mt_char);
+                if (args->args.symlink.linkpath == NULL)
+                        goto err;
+
+                memcpy (args->args.symlink.linkpath, blob, (len + 1));
+                blob += (len + 1);
+                blob_len -= (len + 1);
+        } else {
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.mknod.mode = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.mknod.rdev = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+
+                if (blob_len < sizeof (uint32_t)) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "gfid: %s. Invalid length",
+                                args->gfid);
+                        goto err;
+                }
+                args->args.mknod.umask = ntoh32 (*(uint32_t *)blob);
+                blob += sizeof (uint32_t);
+                blob_len -= sizeof (uint32_t);
+        }
+
+        if (blob_len) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "gfid: %s. Invalid length",
+                        args->gfid);
+                goto err;
+        }
+
+        return args;
+
+err:
+        if (args)
+                ga_newfile_args_free (args);
+
+        return NULL;
+}
+
+ga_heal_args_t *
+ga_heal_parse_args (xlator_t *this, data_t *data)
+{
+        ga_heal_args_t *args     = NULL;
+        ga_private_t   *priv     = NULL;
+        void           *blob     = NULL;
+        int             len      = 0;
+        int             blob_len = 0;
+
+        blob = data->data;
+        blob_len = data->len;
+
+        priv = this->private;
+
+        /* bname should at least contain a character */
+        if (blob_len < (sizeof (args->gfid) + 2))
+                goto err;
+
+        args = mem_get0 (priv->heal_args_pool);
+        if (!args)
+                goto err;
+
+        memcpy (args->gfid, blob, sizeof (args->gfid));
+        blob += sizeof (args->gfid);
+        blob_len -= sizeof (args->gfid);
+
+        len = strnlen (blob, blob_len);
+        if (len == blob_len)
+                goto err;
+
+        args->bname = GF_CALLOC (1, len + 1, gf_common_mt_char);
+        if (!args->bname)
+                goto err;
+
+        memcpy (args->bname, blob, len);
+        blob_len -= (len + 1);
+
+        if (blob_len)
+                goto err;
+
+        return args;
+
+err:
+        if (args)
+                ga_heal_args_free (args);
+
+        return NULL;
+}
+
+static int32_t
+ga_fill_tmp_loc (loc_t *loc, xlator_t *this, uuid_t gfid,
+                 char *bname, dict_t *xdata, loc_t *new_loc)
+{
+        int       ret    = -1;
+        uint64_t  value  = 0;
+        inode_t  *parent = NULL;
+
+        parent = loc->inode;
+        ret = inode_ctx_get (loc->inode, this, &value);
+        if (!ret) {
+                parent = (void *)value;
+                if (uuid_is_null (parent->gfid))
+                        parent = loc->inode;
+        }
+
+        /* parent itself should be looked up */
+        uuid_copy (new_loc->pargfid, parent->gfid);
+        new_loc->parent = inode_ref (parent);
+
+        new_loc->inode = inode_grep (parent->table, parent, bname);
+        if (!new_loc->inode)
+                new_loc->inode = inode_new (parent->table);
+
+        loc_path (new_loc, bname);
+        new_loc->name = basename (new_loc->path);
+
+        /* As GFID would not be set on the entry yet, lets not send entry
+           gfid in the request */
+        /*uuid_copy (new_loc->gfid, (const unsigned char *)gfid); */
+
+        ret = dict_set_static_bin (xdata, "gfid-req", gfid, 16);
+        if (ret < 0)
+                goto out;
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+
+
+static gf_boolean_t
+__is_gfid_access_dir (uuid_t gfid)
+{
+        uuid_t  aux_gfid;
+
+        memset (aux_gfid, 0, 16);
+        aux_gfid[15] = GF_AUX_GFID;
+
+        if (uuid_compare (gfid, aux_gfid) == 0)
+                return _gf_true;
+
+        return _gf_false;
+}
+
+int32_t
+ga_forget (xlator_t *this, inode_t *inode)
+{
+        int       ret = -1;
+        uint64_t  value = 0;
+        inode_t  *tmp_inode = NULL;
+
+        ret = inode_ctx_del (inode, this, &value);
+        if (ret)
+                goto out;
+
+        tmp_inode = (void *)value;
+        inode_unref (tmp_inode);
+
+out:
+        return 0;
+}
+
+
+static int
+ga_heal_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+             int32_t op_ret, int32_t op_errno,
+             inode_t *inode, struct iatt *stat, dict_t *dict,
+             struct iatt *postparent)
+{
+        call_frame_t *orig_frame = NULL;
+
+        orig_frame = frame->local;
+        frame->local = NULL;
+
+        /* don't worry about inode linking and other stuff. They'll happen on
+         * the next lookup.
+         */
+        STACK_DESTROY (frame->root);
+
+        STACK_UNWIND_STRICT (setxattr, orig_frame, op_ret, op_errno, dict);
+
+        return 0;
+}
+
+static int32_t
+ga_newentry_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                         int32_t op_ret, int32_t op_errno, struct iatt *statpre,
+                         struct iatt *statpost,
+                         dict_t *xdata)
+{
+        ga_local_t *local = NULL;
+
+        local = frame->local;
+        frame->local = NULL;
+
+        /* don't worry about inode linking and other stuff. They'll happen on
+         * the next lookup.
+         */
+        STACK_DESTROY (frame->root);
+
+        STACK_UNWIND_STRICT (setxattr, local->orig_frame, op_ret,
+                             op_errno, xdata);
+
+        loc_wipe (&local->loc);
+        mem_put (local);
+
+        return 0;
+}
+
+static int
+ga_newentry_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 inode_t *inode, struct iatt *buf,
+                 struct iatt *preparent, struct iatt *postparent,
+                 dict_t *xdata)
+{
+        ga_local_t *local = NULL;
+        struct iatt temp_stat = {0,};
+
+        local = frame->local;
+
+        if (!local->uid && !local->gid)
+                goto done;
+
+        temp_stat.ia_uid = local->uid;
+        temp_stat.ia_gid = local->gid;
+
+        STACK_WIND (frame, ga_newentry_setattr_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->setattr, &local->loc, &temp_stat,
+                    (GF_SET_ATTR_UID | GF_SET_ATTR_GID), xdata);
+
+        return 0;
+
+done:
+        /* don't worry about inode linking and other stuff. They'll happen on
+         * the next lookup.
+         */
+        frame->local = NULL;
+        STACK_DESTROY (frame->root);
+
+        STACK_UNWIND_STRICT (setxattr, local->orig_frame, op_ret,
+                             op_errno, xdata);
+
+        loc_wipe (&local->loc);
+        mem_put (local);
+
+        return 0;
+}
+
+int32_t
+ga_new_entry (call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *data,
+              dict_t *xdata)
+{
+        int                ret       = -1;
+        ga_newfile_args_t *args      = NULL;
+        loc_t              tmp_loc   = {0,};
+        call_frame_t      *new_frame = NULL;
+        mode_t             mode      = 0;
+        ga_local_t        *local     = NULL;
+        uuid_t             gfid      = {0,};
+
+        args = ga_newfile_parse_args (this, data);
+        if (!args)
+                goto out;
+
+        ret = uuid_parse (args->gfid, gfid);
+        if (ret)
+                goto out;
+
+        if (!xdata)
+                xdata = dict_new ();
+
+        ret = ga_fill_tmp_loc (loc, this, gfid,
+                               args->bname, xdata, &tmp_loc);
+        if (ret)
+                goto out;
+
+        new_frame = copy_frame (frame);
+        if (!new_frame)
+                goto out;
+
+        local = mem_get0 (this->local_pool);
+        local->orig_frame = frame;
+
+        local->uid = args->uid;
+        local->gid = args->gid;
+
+        loc_copy (&local->loc, &tmp_loc);
+
+        new_frame->local = local;
+
+        if (S_ISDIR (args->st_mode)) {
+                STACK_WIND (new_frame, ga_newentry_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->mkdir,
+                            &tmp_loc, args->args.mkdir.mode,
+                            args->args.mkdir.umask, xdata);
+        } else if (S_ISLNK (args->st_mode)) {
+                STACK_WIND (new_frame, ga_newentry_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
+                            args->args.symlink.linkpath,
+                            &tmp_loc, 0, xdata);
+        } else {
+                /* use 07777 (4 7s) for considering the Sticky bits etc) */
+                mode = (S_IFMT & args->st_mode) |
+                        (07777 & args->args.mknod.mode);;
+
+                STACK_WIND (new_frame, ga_newentry_cbk,
+                            FIRST_CHILD(this), FIRST_CHILD(this)->fops->mknod,
+                            &tmp_loc, mode,
+                            args->args.mknod.rdev, args->args.mknod.umask,
+                            xdata);
+        }
+
+        ret = 0;
+out:
+        ga_newfile_args_free (args);
+
+        return ret;
+}
+
+int32_t
+ga_heal_entry (call_frame_t *frame, xlator_t *this, loc_t *loc, data_t *data,
+               dict_t *xdata)
+{
+        int             ret       = -1;
+        ga_heal_args_t *args      = NULL;
+        loc_t           tmp_loc   = {0,};
+        call_frame_t   *new_frame = NULL;
+        uuid_t          gfid      = {0,};
+
+        args = ga_heal_parse_args (this, data);
+        if (!args)
+                goto out;
+
+        ret = uuid_parse (args->gfid, gfid);
+        if (ret)
+                goto out;
+
+        if (!xdata)
+                xdata = dict_new ();
+
+        ret = ga_fill_tmp_loc (loc, this, gfid, args->bname,
+                               xdata, &tmp_loc);
+        if (ret)
+                goto out;
+
+        new_frame = copy_frame (frame);
+        if (!new_frame)
+                goto out;
+        new_frame->local = (void *)frame;
+
+        STACK_WIND (new_frame, ga_heal_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD(this)->fops->lookup,
+                    &tmp_loc, xdata);
+
+        ret = 0;
+out:
+        if (args)
+                ga_heal_args_free (args);
+
+        return ret;
+}
+
+int32_t
+ga_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 dict_t *xdata)
+{
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+ga_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
+             int32_t flags, dict_t *xdata)
+{
+
+        data_t  *data     = NULL;
+        int      op_errno = ENOMEM;
+        int      ret      = 0;
+        inode_t *unref    = NULL;
+
+        if ((loc->name && !strcmp (GF_GFID_DIR, loc->name)) &&
+            ((loc->parent &&
+              __is_root_gfid (loc->parent->gfid)) ||
+             __is_root_gfid (loc->pargfid))) {
+                op_errno = EPERM;
+                goto err;
+        }
+
+        data = dict_get (dict, GF_FUSE_AUX_GFID_NEWFILE);
+        if (data) {
+                ret = ga_new_entry (frame, this, loc, data, xdata);
+                if (ret)
+                        goto err;
+                return 0;
+        }
+
+        data = dict_get (dict, GF_FUSE_AUX_GFID_HEAL);
+        if (data) {
+                ret = ga_heal_entry (frame, this, loc, data, xdata);
+                if (ret)
+                        goto err;
+                return 0;
+        }
+
+        //If the inode is a virtual inode change the inode otherwise perform
+        //the operation on same inode
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, ga_setxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setxattr, loc, dict, flags,
+                    xdata);
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (setxattr, frame, -1, op_errno, xdata);
+        return 0;
+}
+
+
+int32_t
+ga_virtual_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, inode_t *inode,
+                       struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+        int             j                       = 0;
+        int             i                       = 0;
+        int             ret                     = 0;
+        uint64_t        temp_ino                = 0;
+        inode_t         *cbk_inode              = NULL;
+        inode_t         *true_inode             = NULL;
+        uuid_t          random_gfid             = {0,};
+        inode_t         *linked_inode           = NULL;
+
+        if (frame->local)
+                cbk_inode = frame->local;
+        else
+                cbk_inode = inode_ref (inode);
+
+        frame->local = NULL;
+        if (op_ret)
+                goto unwind;
+
+        if (!IA_ISDIR (buf->ia_type))
+                goto unwind;
+
+        /* need to send back a different inode for linking in itable */
+        if (cbk_inode == inode) {
+                /* check if the inode is in the 'itable' or
+                   if its just previously discover()'d inode */
+                true_inode = inode_find (inode->table, buf->ia_gfid);
+                if (!true_inode) {
+                        /* This unref is for 'inode_ref()' done in beginning.
+                           This is needed as cbk_inode is allocated new inode
+                           whose unref is taken at the end*/
+                        inode_unref (cbk_inode);
+                        cbk_inode = inode_new (inode->table);
+
+                        if (!cbk_inode) {
+                                op_ret = -1;
+                                op_errno = ENOMEM;
+                                goto unwind;
+                        }
+                        /* the inode is not present in itable, ie, the actual
+                           path is not yet looked up. Use the current inode
+                           itself for now */
+
+                        linked_inode = inode_link (inode, NULL, NULL, buf);
+                        inode = linked_inode;
+                } else {
+                        /* 'inode_ref()' has been done in inode_find() */
+                        inode = true_inode;
+                }
+
+                ret = inode_ctx_put (cbk_inode, this, (uint64_t)inode);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "failed to set the inode ctx with"
+                                "the actual inode");
+                        if (inode)
+                                inode_unref (inode);
+                }
+                inode = NULL;
+        }
+
+        if (!uuid_is_null (cbk_inode->gfid)) {
+                /* if the previous linked inode is used, use the
+                   same gfid */
+                uuid_copy (random_gfid, cbk_inode->gfid);
+        } else {
+                /* replace the buf->ia_gfid to a random gfid
+                   for directory, for files, what we received is fine */
+                uuid_generate (random_gfid);
+        }
+
+        uuid_copy (buf->ia_gfid, random_gfid);
+
+        for (i = 15; i > (15 - 8); i--) {
+                temp_ino += (uint64_t)(buf->ia_gfid[i]) << j;
+                j += 8;
+        }
+        buf->ia_ino = temp_ino;
+
+unwind:
+        /* Lookup on non-existing gfid returns ESTALE.
+           Convert into ENOENT for virtual lookup*/
+        if (op_errno == ESTALE)
+               op_errno = ENOENT;
+
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, cbk_inode, buf,
+                             xdata, postparent);
+
+        /* Also handles inode_unref of frame->local if done in ga_lookup */
+        if (cbk_inode)
+               inode_unref (cbk_inode);
+
+        return 0;
+}
+
+int32_t
+ga_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+        ga_private_t *priv = NULL;
+
+        /* if the entry in question is not 'root',
+           then follow the normal path */
+        if (op_ret || !__is_root_gfid(buf->ia_gfid))
+                goto unwind;
+
+        priv = this->private;
+
+        /* do we need to copy root stbuf everytime? */
+        /* mostly yes, as we want to have the 'stat' info show latest
+           in every _cbk() */
+
+        /* keep the reference for root stat buf */
+        priv->root_stbuf = *buf;
+        priv->gfiddir_stbuf = priv->root_stbuf;
+        priv->gfiddir_stbuf.ia_gfid[15] = GF_AUX_GFID;
+        priv->gfiddir_stbuf.ia_ino = GF_AUX_GFID;
+
+unwind:
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
+                             xdata, postparent);
+        return 0;
+}
+
+int32_t
+ga_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *xdata)
+{
+        ga_private_t *priv     = NULL;
+        int           ret      = -1;
+        uuid_t        tmp_gfid = {0,};
+        loc_t         tmp_loc  = {0,};
+        uint64_t      value    = 0;
+        inode_t      *inode    = NULL;
+        inode_t      *true_inode    = NULL;
+        int32_t       op_errno = ENOENT;
+
+        /* if its discover(), no need for any action here */
+        if (!loc->name)
+                goto wind;
+
+        /* if its revalidate, and inode is not of type directory,
+           proceed with 'wind' */
+        if (loc->inode && loc->inode->ia_type &&
+            !IA_ISDIR (loc->inode->ia_type)) {
+
+                /* a revalidate on ".gfid/<dentry>" is possible, check for it */
+                if (((loc->parent &&
+                      __is_gfid_access_dir (loc->parent->gfid)) ||
+                     __is_gfid_access_dir (loc->pargfid))) {
+
+                        /* here, just send 'loc->gfid' and 'loc->inode' */
+                        tmp_loc.inode = inode_ref (loc->inode);
+                        uuid_copy (tmp_loc.gfid, loc->inode->gfid);
+
+                        STACK_WIND (frame, default_lookup_cbk,
+                                    FIRST_CHILD(this),
+                                    FIRST_CHILD(this)->fops->lookup,
+                                    &tmp_loc, xdata);
+
+                        inode_unref (tmp_loc.inode);
+
+                        return 0;
+                }
+
+                /* not something to bother, continue the flow */
+                goto wind;
+        }
+
+        priv = this->private;
+
+        /* need to check if the lookup is on virtual dir */
+        if ((loc->name && !strcmp (GF_GFID_DIR, loc->name)) &&
+            ((loc->parent && __is_root_gfid (loc->parent->gfid)) ||
+             __is_root_gfid (loc->pargfid))) {
+                /* this means, the query is on '/.gfid', return the fake stat,
+                   and say success */
+
+                STACK_UNWIND_STRICT (lookup, frame, 0, 0, loc->inode,
+                                     &priv->gfiddir_stbuf, xdata,
+                                     &priv->root_stbuf);
+                return 0;
+        }
+
+        /* now, check if the lookup() is on an existing entry,
+           but on gfid-path */
+        if (!((loc->parent && __is_gfid_access_dir (loc->parent->gfid)) ||
+              __is_gfid_access_dir (loc->pargfid))) {
+                if (!loc->parent)
+                        goto wind;
+
+                ret = inode_ctx_get (loc->parent, this, &value);
+                if (ret)
+                        goto wind;
+
+                inode = (inode_t *) value;
+
+                ret = loc_copy_overload_parent (&tmp_loc, loc, inode);
+                if (ret)
+                        goto err;
+
+                STACK_WIND (frame, ga_lookup_cbk, FIRST_CHILD (this),
+                            FIRST_CHILD (this)->fops->lookup, &tmp_loc, xdata);
+
+                loc_wipe (&tmp_loc);
+                return 0;
+        }
+
+        /* make sure the 'basename' is actually a 'canonical-gfid',
+           otherwise, return error */
+        ret = uuid_parse (loc->name, tmp_gfid);
+        if (ret)
+                goto err;
+
+        /* if its fresh lookup, go ahead and send it down, if not,
+           for directory, we need indirection to actual dir inode */
+        if (!(loc->inode && loc->inode->ia_type))
+                goto discover;
+
+        /* revalidate on directory */
+        ret = inode_ctx_get (loc->inode, this, &value);
+        if (ret)
+                goto err;
+
+        inode = (void *)value;
+
+        /* valid inode, already looked up, work on that */
+        if (inode->ia_type)
+                goto discover;
+
+        /* check if the inode is in the 'itable' or
+           if its just previously discover()'d inode */
+        true_inode = inode_find (loc->inode->table, tmp_gfid);
+        if (true_inode) {
+                /* time do another lookup and update the context
+                   with proper inode */
+                op_errno = ESTALE;
+                /* 'inode_ref()' done in inode_find */
+                inode_unref (true_inode);
+                goto err;
+        }
+
+discover:
+        /* for the virtual entries, we don't need to send 'gfid-req' key, as
+           for these entries, we don't want to 'set' a new gfid */
+        if (xdata)
+                dict_del (xdata, "gfid-req");
+
+        uuid_copy (tmp_loc.gfid, tmp_gfid);
+
+        /* if revalidate, then we need to have the proper reference */
+        if (inode) {
+                tmp_loc.inode = inode_ref (inode);
+                frame->local = inode_ref (loc->inode);
+        } else {
+                tmp_loc.inode = inode_ref (loc->inode);
+        }
+
+        STACK_WIND (frame, ga_virtual_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, &tmp_loc, xdata);
+
+        inode_unref (tmp_loc.inode);
+
+        return 0;
+
+wind:
+        /* used for all the normal lookup path */
+        STACK_WIND (frame, ga_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
+
+        return 0;
+
+err:
+        STACK_UNWIND_STRICT (lookup, frame, -1, op_errno, loc->inode,
+                             &priv->gfiddir_stbuf, xdata,
+                             &priv->root_stbuf);
+        return 0;
+}
+
+int
+ga_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          mode_t umask, dict_t *xdata)
+{
+        int op_errno = 0;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        STACK_WIND (frame, default_mkdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask,
+                    xdata);
+
+        return 0;
+
+err:
+        STACK_UNWIND_STRICT (mkdir, frame, -1, op_errno, loc->inode,
+                             NULL, NULL, NULL, xdata);
+        return 0;
+}
+
+
+int
+ga_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int flags,
+           mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        int op_errno = 0;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        STACK_WIND (frame, default_create_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (create, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, NULL, xdata);
+
+        return 0;
+
+}
+
+int
+ga_symlink (call_frame_t *frame, xlator_t *this, const char *linkname,
+            loc_t *loc, mode_t umask, dict_t *xdata)
+{
+        int op_errno = 0;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        STACK_WIND (frame, default_symlink_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->symlink,
+                    linkname, loc, umask, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (symlink, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, xdata);
+
+        return 0;
+}
+
+int
+ga_mknod (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+          dev_t rdev, mode_t umask, dict_t *xdata)
+{
+        int op_errno = 0;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        STACK_WIND (frame, default_mknod_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->mknod, loc, mode, rdev,
+                    umask, xdata);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (mknod, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, xdata);
+
+        return 0;
+}
+
+int
+ga_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+          dict_t *xdata)
+{
+        int op_errno = 0;
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_rmdir_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rmdir,
+                    loc, flag, xdata);
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (rmdir, frame, -1, op_errno, NULL,
+                             NULL, xdata);
+
+        return 0;
+}
+
+int
+ga_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t xflag,
+           dict_t *xdata)
+{
+        int op_errno = 0;
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_unlink_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->unlink,
+                    loc, xflag, xdata);
+
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (unlink, frame, -1, op_errno, NULL,
+                             NULL, xdata);
+
+        return 0;
+}
+
+int
+ga_rename (call_frame_t *frame, xlator_t *this,
+           loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        int op_errno = 0;
+        inode_t *oldloc_unref = NULL;
+        inode_t *newloc_unref = NULL;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (oldloc, op_errno, err);
+        GFID_ACCESS_ENTRY_OP_CHECK (newloc, op_errno, err);
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, oldloc, oldloc_unref,
+                                         handle_newloc);
+
+handle_newloc:
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, newloc, newloc_unref, wind);
+
+wind:
+        STACK_WIND (frame, default_rename_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->rename,
+                    oldloc, newloc, xdata);
+
+        if (oldloc_unref)
+                inode_unref (oldloc_unref);
+
+        if (newloc_unref)
+                inode_unref (newloc_unref);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (rename, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, NULL, xdata);
+
+        return 0;
+}
+
+
+int
+ga_link (call_frame_t *frame, xlator_t *this,
+         loc_t *oldloc, loc_t *newloc, dict_t *xdata)
+{
+        int op_errno = 0;
+        inode_t *oldloc_unref = NULL;
+        inode_t *newloc_unref = NULL;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (oldloc, op_errno, err);
+        GFID_ACCESS_ENTRY_OP_CHECK (newloc, op_errno, err);
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, oldloc, oldloc_unref,
+                                         handle_newloc);
+
+handle_newloc:
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, newloc, newloc_unref, wind);
+
+wind:
+        STACK_WIND (frame, default_link_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->link,
+                    oldloc, newloc, xdata);
+
+        if (oldloc_unref)
+                inode_unref (oldloc_unref);
+
+        if (newloc_unref)
+                inode_unref (newloc_unref);
+
+        return 0;
+err:
+        STACK_UNWIND_STRICT (link, frame, -1, op_errno, NULL,
+                             NULL, NULL, NULL, xdata);
+
+        return 0;
+}
+
+int32_t
+ga_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            fd_t *fd, dict_t *xdata)
+{
+        int op_errno = 0;
+
+        GFID_ACCESS_ENTRY_OP_CHECK (loc, op_errno, err);
+
+        /* also check if the loc->inode itself is virtual
+           inode, if yes, return with failure, mainly because we
+           can't handle all the readdirp and other things on it. */
+        if (inode_ctx_get (loc->inode, this, NULL) == 0) {
+                op_errno = ENOTSUP;
+                goto err;
+        }
+
+        STACK_WIND (frame, default_opendir_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->opendir,
+                    loc, fd, xdata);
+        return 0;
+err:
+        STACK_UNWIND_STRICT (opendir, frame, -1, op_errno, NULL, xdata);
+
+        return 0;
+}
+
+int32_t
+ga_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             const char *name, dict_t *xdata)
+{
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+}
+
+int32_t
+ga_stat (call_frame_t *frame, xlator_t *this, loc_t *loc,
+         dict_t *xdata)
+{
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_stat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->stat, loc, xdata);
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+}
+
+int32_t
+ga_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            struct iatt *stbuf, int32_t valid,
+            dict_t *xdata)
+{
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_setattr_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->setattr, loc, stbuf, valid,
+                    xdata);
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+}
+
+int32_t
+ga_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
+{
+        inode_t *unref = NULL;
+
+        GFID_ACCESS_GET_VALID_DIR_INODE (this, loc, unref, wind);
+
+wind:
+        STACK_WIND (frame, default_removexattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr, loc, name,
+                    xdata);
+        if (unref)
+                inode_unref (unref);
+
+        return 0;
+}
+
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_gfid_access_mt_end + 1);
+
+        if (ret != 0) {
+                gf_log (this->name, GF_LOG_WARNING, "Memory accounting"
+                        " init failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+int32_t
+init (xlator_t *this)
+{
+        ga_private_t *priv = NULL;
+        int ret = -1;
+
+        if (!this->children || this->children->next) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "not configured with exactly one child. exiting");
+                goto out;
+        }
+
+        /* This can be the top of graph in certain cases */
+        if (!this->parents) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "dangling volume. check volfile ");
+        }
+
+        /* TODO: define a mem-type structure */
+        priv = GF_CALLOC (1, sizeof (*priv), gf_gfid_access_mt_priv_t);
+        if (!priv)
+                goto out;
+
+        priv->newfile_args_pool = mem_pool_new (ga_newfile_args_t, 512);
+        if (!priv->newfile_args_pool)
+                goto out;
+
+        priv->heal_args_pool = mem_pool_new (ga_heal_args_t, 512);
+        if (!priv->heal_args_pool)
+                goto out;
+
+        this->local_pool = mem_pool_new (ga_local_t, 16);
+        if (!this->local_pool) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto out;
+        }
+
+        this->private = priv;
+
+        ret = 0;
+out:
+        if (ret && priv) {
+                if (priv->newfile_args_pool)
+                        mem_pool_destroy (priv->newfile_args_pool);
+                GF_FREE (priv);
+        }
+
+        return ret;
+}
+
+void
+fini (xlator_t *this)
+{
+        ga_private_t *priv = NULL;
+        priv = this->private;
+        this->private = NULL;
+
+        if (priv) {
+                if (priv->newfile_args_pool)
+                        mem_pool_destroy (priv->newfile_args_pool);
+                if (priv->heal_args_pool)
+                        mem_pool_destroy (priv->heal_args_pool);
+                GF_FREE (priv);
+        }
+
+        return;
+}
+
+
+struct xlator_fops fops = {
+        .lookup = ga_lookup,
+
+        /* entry fops */
+        .mkdir   = ga_mkdir,
+        .mknod   = ga_mknod,
+        .create  = ga_create,
+        .symlink = ga_symlink,
+        .link    = ga_link,
+        .unlink  = ga_unlink,
+        .rmdir   = ga_rmdir,
+        .rename  = ga_rename,
+
+        /* handle any other directory operations here */
+        .opendir  = ga_opendir,
+        .stat     = ga_stat,
+        .setattr  = ga_setattr,
+        .getxattr = ga_getxattr,
+        .removexattr = ga_removexattr,
+
+        /* special fop to handle more entry creations */
+        .setxattr = ga_setxattr,
+};
+
+struct xlator_cbks cbks = {
+        .forget = ga_forget,
+};
+
+struct volume_options options[] = {
+        /* This translator doesn't take any options, or provide any options */
+        { .key  = {NULL} },
+};
diff --git a/xlators/features/gfid-access/src/gfid-access.h b/xlators/features/gfid-access/src/gfid-access.h
new file mode 100644
index 000000000..e883eca69
--- /dev/null
+++ b/xlators/features/gfid-access/src/gfid-access.h
@@ -0,0 +1,134 @@
+/*
+   Copyright (c) 2013 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __GFID_ACCESS_H__
+#define __GFID_ACCESS_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "logging.h"
+#include "dict.h"
+#include "xlator.h"
+#include "defaults.h"
+#include "gfid-access-mem-types.h"
+
+#define UUID_CANONICAL_FORM_LEN 36
+
+#define GF_FUSE_AUX_GFID_NEWFILE "glusterfs.gfid.newfile"
+#define GF_FUSE_AUX_GFID_HEAL    "glusterfs.gfid.heal"
+
+#define GF_GFID_KEY "GLUSTERFS_GFID"
+#define GF_GFID_DIR ".gfid"
+#define GF_AUX_GFID 0xd
+
+#define GFID_ACCESS_GET_VALID_DIR_INODE(x,l,unref,lbl) do {             \
+                int       ret       = 0;                                \
+                uint64_t  value     = 0;                                \
+                inode_t  *tmp_inode = NULL;                             \
+                                                                        \
+                /* if its an entry operation, on the virtual */         \
+                /* directory inode as parent, we need to handle */      \
+                /* it properly */                                       \
+                if (l->parent) {                                        \
+                        ret = inode_ctx_get (l->parent, x, &value);     \
+                        if (ret)                                        \
+                                goto lbl;                               \
+                        tmp_inode = (inode_t *)value;                   \
+                        l->parent = inode_ref (tmp_inode);                          \
+                        /* if parent is virtual, no need to handle */   \
+                        /* loc->inode */                                \
+                        break;                                          \
+                }                                                       \
+                                                                        \
+                /* if its an inode operation, on the virtual */         \
+                /* directory inode itself, we need to handle */         \
+                /* it properly */                                       \
+                if (l->inode) {                                         \
+                        ret = inode_ctx_get (l->inode, x, &value);      \
+                        if (ret)                                        \
+                                goto lbl;                               \
+                        tmp_inode = (inode_t *)value;                   \
+                        l->inode = inode_ref (tmp_inode);                           \
+                }                                                       \
+                                                                        \
+        } while (0)
+
+#define GFID_ACCESS_ENTRY_OP_CHECK(loc,err,lbl)    do {                 \
+                /* need to check if the lookup is on virtual dir */     \
+                if ((loc->name && !strcmp (GF_GFID_DIR, loc->name)) &&  \
+                    ((loc->parent &&                                    \
+                      __is_root_gfid (loc->parent->gfid)) ||            \
+                      __is_root_gfid (loc->pargfid))) {                 \
+                        err = EEXIST;                                   \
+                        goto lbl;                                       \
+                }                                                       \
+                                                                        \
+                /* now, check if the lookup() is on an existing */      \
+                /* entry, but on gfid-path */                           \
+                if ((loc->parent &&                                     \
+                     __is_gfid_access_dir (loc->parent->gfid)) ||       \
+                    __is_gfid_access_dir (loc->pargfid)) {              \
+                        err = EPERM;                                    \
+                        goto lbl;                                       \
+                }                                                       \
+        } while (0)
+
+
+typedef struct {
+        unsigned int  uid;
+        unsigned int  gid;
+        char          gfid[UUID_CANONICAL_FORM_LEN + 1];
+        unsigned int  st_mode;
+        char         *bname;
+
+        union {
+                struct _symlink_in {
+                        char     *linkpath;
+                } __attribute__ ((__packed__)) symlink;
+
+                struct _mknod_in {
+                        unsigned int mode;
+                        unsigned int rdev;
+                        unsigned int umask;
+                } __attribute__ ((__packed__)) mknod;
+
+                struct _mkdir_in {
+                        unsigned int mode;
+                        unsigned int umask;
+                } __attribute__ ((__packed__)) mkdir;
+        } __attribute__ ((__packed__)) args;
+} __attribute__((__packed__)) ga_newfile_args_t;
+
+typedef struct {
+        char      gfid[UUID_CANONICAL_FORM_LEN + 1];
+        char     *bname; /* a null terminated basename */
+} __attribute__((__packed__)) ga_heal_args_t;
+
+struct ga_private {
+        /* root inode's stbuf */
+        struct iatt root_stbuf;
+        struct iatt gfiddir_stbuf;
+        struct mem_pool *newfile_args_pool;
+        struct mem_pool *heal_args_pool;
+};
+typedef struct ga_private ga_private_t;
+
+struct __ga_local {
+        call_frame_t *orig_frame;
+        unsigned int uid;
+        unsigned int gid;
+        loc_t loc;
+};
+typedef struct __ga_local ga_local_t;
+
+#endif /* __GFID_ACCESS_H__ */
diff --git a/xlators/features/glupy/Makefile.am b/xlators/features/glupy/Makefile.am
new file mode 100644
index 000000000..060429ecf
--- /dev/null
+++ b/xlators/features/glupy/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src examples
+
+CLEANFILES =
diff --git a/xlators/features/glupy/doc/README.md b/xlators/features/glupy/doc/README.md
new file mode 100644
index 000000000..2d7b30ef6
--- /dev/null
+++ b/xlators/features/glupy/doc/README.md
@@ -0,0 +1,44 @@
+This is just the very start for a GlusterFS[1] meta-translator that will
+allow translator code to be written in Python.  It's based on the standard
+Python embedding (not extending) techniques, plus a dash of the ctypes module.
+The interface is a pretty minimal adaptation of the dispatches and callbacks
+from the C API[2] to Python, as follows:
+
+* Dispatch functions and callbacks must be defined on an "xlator" class
+  derived from gluster.Translator so that they'll be auto-registered with
+  the C translator during initialization.
+
+* For each dispatch or callback function you want to intercept, you define a
+  Python function using the xxx\_fop\_t or xxx\_cbk\_t decorator.
+
+* The arguments for each operation are different, so you'll need to refer to
+  the C API.  GlusterFS-specific types are used (though only loc\_t is fully
+  defined so far) and type correctness is enforced by ctypes.
+
+* If you do intercept a dispatch function, it is your responsibility to call
+  xxx\_wind (like STACK\_WIND in the C API but operation-specific) to pass
+  the request to the next translator.  If you do not intercept a function, it
+  will default the same way as for C (pass through to the same operation with
+  the same arguments on the first child translator).
+
+* If you intercept a callback function, it is your responsibility to call
+  xxx\_unwind (like STACK\_UNWIND\_STRICT in the C API) to pass the request back
+  to the caller.
+
+So far only the lookup and create operations are handled this way, to support
+the "negative lookup" example.  Now that the basic infrastructure is in place,
+adding more functions should be very quick, though with that much boilerplate I
+might pause to write a code generator.  I also plan to add structure
+definitions and interfaces for some of the utility functions in libglusterfs
+(especially those having to do with inode and fd context) in the fairly near
+future.  Note that you can also use ctypes to get at anything not explicitly
+exposed to Python already.
+
+_If you're coming here because of the Linux Journal article, please note that
+the code has evolved since that was written. The version that matches the
+article is here:_
+
+https://github.com/jdarcy/glupy/tree/4bbae91ba459ea46ef32f2966562492e4ca9187a
+
+[1] http://www.gluster.org
+[2] http://hekafs.org/dist/xlator_api_2.html
diff --git a/xlators/features/glupy/doc/TESTING b/xlators/features/glupy/doc/TESTING
new file mode 100644
index 000000000..e05f17f49
--- /dev/null
+++ b/xlators/features/glupy/doc/TESTING
@@ -0,0 +1,9 @@
+Loading a translator written in Python using the glupy meta translator
+-------------------------------------------------------------------------------
+'test.vol' is a simple volfile with the debug-trace Python translator on top
+of a brick. The volfile can be mounted using the following command.
+
+$ glusterfs --debug -f test.vol /path/to/mntpt
+
+If then file operations are performed on the newly mounted file system, log
+output would be printed by the Python translator on the standard output.
diff --git a/xlators/features/glupy/doc/test.vol b/xlators/features/glupy/doc/test.vol
new file mode 100644
index 000000000..0751a488c
--- /dev/null
+++ b/xlators/features/glupy/doc/test.vol
@@ -0,0 +1,10 @@
+volume vol-posix
+    type storage/posix
+    option directory /path/to/brick
+end-volume
+
+volume vol-glupy
+    type features/glupy
+    option module-name debug-trace
+    subvolumes vol-posix
+end-volume
diff --git a/xlators/features/glupy/examples/Makefile.am b/xlators/features/glupy/examples/Makefile.am
new file mode 100644
index 000000000..c26abeaaf
--- /dev/null
+++ b/xlators/features/glupy/examples/Makefile.am
@@ -0,0 +1,5 @@
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+glupyexamplesdir = $(xlatordir)/glupy
+
+glupyexamples_PYTHON = negative.py helloworld.py debug-trace.py
diff --git a/xlators/features/glupy/examples/debug-trace.py b/xlators/features/glupy/examples/debug-trace.py
new file mode 100644
index 000000000..6eef1b58b
--- /dev/null
+++ b/xlators/features/glupy/examples/debug-trace.py
@@ -0,0 +1,775 @@
+import sys
+import stat
+from uuid import UUID
+from time import strftime, localtime
+from gluster.glupy import *
+
+# This translator was written primarily to test the fop entry point definitions
+# and structure definitions in 'glupy.py'.
+
+# It is similar to the C language debug-trace translator, which logs the
+# arguments passed to the fops and their corresponding cbk functions.
+
+dl.get_id.restype = c_long
+dl.get_id.argtypes = [ POINTER(call_frame_t) ]
+
+dl.get_rootunique.restype = c_uint64
+dl.get_rootunique.argtypes = [ POINTER(call_frame_t) ]
+
+def uuid2str (gfid):
+        return str(UUID(''.join(map("{0:02x}".format, gfid))))
+
+
+def st_mode_from_ia (prot, filetype):
+        st_mode = 0
+        type_bit = 0
+        prot_bit = 0
+
+        if filetype == IA_IFREG:
+                type_bit = stat.S_IFREG
+        elif filetype == IA_IFDIR:
+                type_bit = stat.S_IFDIR
+        elif filetype == IA_IFLNK:
+                type_bit = stat.S_IFLNK
+        elif filetype == IA_IFBLK:
+                type_bit = stat.S_IFBLK
+        elif filetype == IA_IFCHR:
+                type_bit = stat.S_IFCHR
+        elif filetype == IA_IFIFO:
+                type_bit = stat.S_IFIFO
+        elif filetype == IA_IFSOCK:
+                type_bit = stat.S_IFSOCK
+        elif filetype == IA_INVAL:
+                pass
+
+
+        if prot.suid:
+                prot_bit |= stat.S_ISUID
+        if prot.sgid:
+                prot_bit |= stat.S_ISGID
+        if prot.sticky:
+                prot_bit |= stat.S_ISVTX
+
+        if prot.owner.read:
+                prot_bit |= stat.S_IRUSR
+        if prot.owner.write:
+                prot_bit |= stat.S_IWUSR
+        if prot.owner.execn:
+                prot_bit |= stat.S_IXUSR
+
+        if prot.group.read:
+                prot_bit |= stat.S_IRGRP
+        if prot.group.write:
+                prot_bit |= stat.S_IWGRP
+        if prot.group.execn:
+                prot_bit |= stat.S_IXGRP
+
+        if prot.other.read:
+                prot_bit |= stat.S_IROTH
+        if prot.other.write:
+                prot_bit |= stat.S_IWOTH
+        if prot.other.execn:
+                prot_bit |= stat.S_IXOTH
+
+        st_mode = (type_bit | prot_bit)
+
+        return st_mode
+
+
+def trace_stat2str (buf):
+        gfid = uuid2str(buf.contents.ia_gfid)
+        mode = st_mode_from_ia(buf.contents.ia_prot, buf.contents.ia_type)
+        atime_buf = strftime("[%b %d %H:%M:%S]",
+                             localtime(buf.contents.ia_atime))
+        mtime_buf = strftime("[%b %d %H:%M:%S]",
+                             localtime(buf.contents.ia_mtime))
+        ctime_buf = strftime("[%b %d %H:%M:%S]",
+                             localtime(buf.contents.ia_ctime))
+        return ("(gfid={0:s}, ino={1:d}, mode={2:o}, nlink={3:d}, uid ={4:d}, "+
+                "gid ={5:d}, size={6:d}, blocks={7:d}, atime={8:s}, mtime={9:s}, "+
+                "ctime={10:s})").format(gfid, buf.contents.ia_no, mode,
+                                        buf.contents.ia_nlink,
+                                        buf.contents.ia_uid,
+                                        buf.contents.ia_gid,
+                                        buf.contents.ia_size,
+                                        buf.contents.ia_blocks,
+                                        atime_buf, mtime_buf,
+                                        ctime_buf)
+
+class xlator(Translator):
+
+        def __init__(self, c_this):
+                Translator.__init__(self, c_this)
+                self.gfids = {}
+
+        def lookup_fop(self, frame, this, loc, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.gfid)
+                print("GLUPY TRACE LOOKUP FOP- {0:d}: gfid={1:s}; " +
+                      "path={2:s}").format(unique, gfid, loc.contents.path)
+                self.gfids[key] = gfid
+                dl.wind_lookup(frame, POINTER(xlator_t)(), loc, xdata)
+                return 0
+
+        def lookup_cbk(self, frame, cookie, this, op_ret, op_errno,
+                       inode, buf, xdata, postparent):
+                unique =dl.get_rootunique(frame)
+                key =dl.get_id(frame)
+                if op_ret == 0:
+                        gfid = uuid2str(buf.contents.ia_gfid)
+                        statstr = trace_stat2str(buf)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE LOOKUP CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; *buf={3:s}; " +
+                              "*postparent={4:s}").format(unique, gfid,
+                                                          op_ret, statstr,
+                                                          postparentstr)
+                else:
+                        gfid = self.gfids[key]
+                        print("GLUPY TRACE LOOKUP CBK - {0:d}: gfid={1:s};" +
+                              " op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                      gfid,
+                                                                      op_ret,
+                                                                      op_errno)
+                del self.gfids[key]
+                dl.unwind_lookup(frame, cookie, this, op_ret, op_errno,
+                                 inode, buf, xdata, postparent)
+                return 0
+
+        def create_fop(self, frame, this, loc, flags, mode, umask, fd,
+                       xdata):
+                unique = dl.get_rootunique(frame)
+                gfid = uuid2str(loc.contents.gfid)
+                print("GLUPY TRACE CREATE FOP- {0:d}: gfid={1:s}; path={2:s}; " +
+                      "fd={3:s}; flags=0{4:o}; mode=0{5:o}; " +
+                      "umask=0{6:o}").format(unique, gfid, loc.contents.path,
+                                             fd, flags, mode, umask)
+                dl.wind_create(frame, POINTER(xlator_t)(), loc, flags,mode,
+                               umask, fd, xdata)
+                return 0
+
+        def create_cbk(self, frame, cookie, this, op_ret, op_errno, fd,
+                       inode, buf, preparent, postparent, xdata):
+                unique = dl.get_rootunique(frame)
+                if op_ret >= 0:
+                        gfid = uuid2str(inode.contents.gfid)
+                        statstr = trace_stat2str(buf)
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE CREATE CBK- {0:d}: gfid={1:s};" +
+                              " op_ret={2:d}; fd={3:s}; *stbuf={4:s}; " +
+                              "*preparent={5:s};" +
+                              " *postparent={6:s}").format(unique, gfid, op_ret,
+                                                           fd, statstr,
+                                                           preparentstr,
+                                                           postparentstr)
+                else:
+                        print ("GLUPY TRACE CREATE CBK- {0:d}: op_ret={1:d}; " +
+                              "op_errno={2:d}").format(unique, op_ret, op_errno)
+                dl.unwind_create(frame, cookie, this, op_ret, op_errno, fd,
+                                 inode, buf, preparent, postparent, xdata)
+                return 0
+
+        def open_fop(self, frame, this, loc, flags, fd, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE OPEN FOP- {0:d}: gfid={1:s}; path={2:s}; "+
+                      "flags={3:d}; fd={4:s}").format(unique, gfid,
+                                                      loc.contents.path, flags,
+                                                      fd)
+                self.gfids[key] = gfid
+                dl.wind_open(frame, POINTER(xlator_t)(), loc, flags, fd, xdata)
+                return 0
+
+        def open_cbk(self, frame, cookie, this, op_ret, op_errno, fd, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE OPEN CBK- {0:d}: gfid={1:s}; op_ret={2:d}; "
+                      "op_errno={3:d}; *fd={4:s}").format(unique, gfid,
+                                                          op_ret, op_errno, fd)
+                del self.gfids[key]
+                dl.unwind_open(frame, cookie, this, op_ret, op_errno, fd,
+                               xdata)
+                return 0
+
+        def readv_fop(self, frame, this, fd, size, offset, flags, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE READV FOP- {0:d}: gfid={1:s}; "+
+                      "fd={2:s}; size ={3:d}; offset={4:d}; " +
+                      "flags=0{5:x}").format(unique, gfid, fd, size, offset,
+                                             flags)
+                self.gfids[key] = gfid
+                dl.wind_readv (frame, POINTER(xlator_t)(), fd, size, offset,
+                               flags, xdata)
+                return 0
+
+        def readv_cbk(self, frame, cookie, this, op_ret, op_errno, vector,
+                      count, buf, iobref, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret >= 0:
+                        statstr = trace_stat2str(buf)
+                        print("GLUPY TRACE READV CBK- {0:d}: gfid={1:s}, "+
+                              "op_ret={2:d}; *buf={3:s};").format(unique, gfid,
+                                                                  op_ret,
+                                                                  statstr)
+
+                else:
+                        print("GLUPY TRACE READV CBK- {0:d}: gfid={1:s}, "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_readv (frame, cookie, this, op_ret, op_errno,
+                                 vector, count, buf, iobref, xdata)
+                return 0
+
+        def writev_fop(self, frame, this, fd, vector, count, offset, flags,
+                       iobref, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE  WRITEV FOP- {0:d}: gfid={1:s}; " +
+                      "fd={2:s}; count={3:d}; offset={4:d}; " +
+                      "flags=0{5:x}").format(unique, gfid, fd, count, offset,
+                                             flags)
+                self.gfids[key] = gfid
+                dl.wind_writev(frame, POINTER(xlator_t)(), fd, vector, count,
+                               offset, flags, iobref, xdata)
+                return 0
+
+        def writev_cbk(self, frame, cookie, this, op_ret, op_errno, prebuf,
+                       postbuf, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                if op_ret >= 0:
+                        preopstr = trace_stat2str(prebuf)
+                        postopstr = trace_stat2str(postbuf)
+                        print("GLUPY TRACE WRITEV CBK- {0:d}: op_ret={1:d}; " +
+                              "*prebuf={2:s}; " +
+                              "*postbuf={3:s}").format(unique, op_ret, preopstr,
+                                                       postopstr)
+                else:
+                        gfid = self.gfids[key]
+                        print("GLUPY TRACE WRITEV CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_writev (frame, cookie, this, op_ret, op_errno,
+                                  prebuf, postbuf, xdata)
+                return 0
+
+        def opendir_fop(self, frame, this, loc, fd, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE OPENDIR FOP- {0:d}: gfid={1:s}; path={2:s}; "+
+                      "fd={3:s}").format(unique, gfid, loc.contents.path, fd)
+                self.gfids[key] = gfid
+                dl.wind_opendir(frame, POINTER(xlator_t)(), loc, fd, xdata)
+                return 0
+
+        def opendir_cbk(self, frame, cookie, this, op_ret, op_errno, fd,
+                        xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE OPENDIR CBK- {0:d}: gfid={1:s}; op_ret={2:d};"+
+                      " op_errno={3:d}; fd={4:s}").format(unique, gfid, op_ret,
+                                                          op_errno, fd)
+                del self.gfids[key]
+                dl.unwind_opendir(frame, cookie, this, op_ret, op_errno,
+                                  fd, xdata)
+                return 0
+
+        def readdir_fop(self, frame, this, fd, size, offset, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE READDIR FOP- {0:d}:  gfid={1:s}; fd={2:s}; " +
+                      "size={3:d}; offset={4:d}").format(unique, gfid, fd, size,
+                                                         offset)
+                self.gfids[key] = gfid
+                dl.wind_readdir(frame, POINTER(xlator_t)(), fd, size, offset,
+                                xdata)
+                return 0
+
+        def readdir_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
+                        xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE READDIR CBK- {0:d}: gfid={1:s}; op_ret={2:d};"+
+                      " op_errno={3:d}").format(unique, gfid, op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_readdir(frame, cookie, this, op_ret, op_errno, buf,
+                                  xdata)
+                return 0
+
+        def readdirp_fop(self, frame, this, fd, size, offset, dictionary):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE READDIRP FOP- {0:d}: gfid={1:s}; fd={2:s}; "+
+                      " size={3:d}; offset={4:d}").format(unique, gfid, fd, size,
+                                                          offset)
+                self.gfids[key] = gfid
+                dl.wind_readdirp(frame, POINTER(xlator_t)(), fd, size, offset,
+                                 dictionary)
+                return 0
+
+        def readdirp_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
+                         xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE READDIRP CBK- {0:d}: gfid={1:s}; "+
+                      "op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
+                                                             op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_readdirp(frame, cookie, this, op_ret, op_errno, buf,
+                                  xdata)
+                return 0
+
+        def mkdir_fop(self, frame, this, loc, mode, umask, xdata):
+                unique = dl.get_rootunique(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE MKDIR FOP- {0:d}: gfid={1:s}; path={2:s}; " +
+                      "mode={3:d}; umask=0{4:o}").format(unique, gfid,
+                                                         loc.contents.path, mode,
+                                                         umask)
+                dl.wind_mkdir(frame, POINTER(xlator_t)(), loc, mode, umask,
+                              xdata)
+                return 0
+
+        def mkdir_cbk(self, frame, cookie, this, op_ret, op_errno, inode, buf,
+                      preparent, postparent,  xdata):
+                unique = dl.get_rootunique(frame)
+                if op_ret == 0:
+                        gfid = uuid2str(inode.contents.gfid)
+                        statstr = trace_stat2str(buf)
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE MKDIR CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; *stbuf={3:s}; *prebuf={4:s}; "+
+                              "*postbuf={5:s} ").format(unique, gfid, op_ret,
+                                                        statstr,
+                                                        preparentstr,
+                                                        postparentstr)
+                else:
+                        print("GLUPY TRACE MKDIR CBK- {0:d}:  op_ret={1:d}; "+
+                              "op_errno={2:d}").format(unique, op_ret, op_errno)
+                dl.unwind_mkdir(frame, cookie, this, op_ret, op_errno, inode,
+                                buf, preparent, postparent, xdata)
+                return 0
+
+        def rmdir_fop(self, frame, this, loc, flags, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE RMDIR FOP- {0:d}: gfid={1:s}; path={2:s}; "+
+                      "flags={3:d}").format(unique, gfid, loc.contents.path,
+                                            flags)
+                self.gfids[key] = gfid
+                dl.wind_rmdir(frame, POINTER(xlator_t)(), loc, flags, xdata)
+                return 0
+
+        def rmdir_cbk(self, frame, cookie, this, op_ret, op_errno, preparent,
+                      postparent, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE RMDIR CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; *prebuf={3:s}; "+
+                              "*postbuf={4:s}").format(unique, gfid, op_ret,
+                                                       preparentstr,
+                                                       postparentstr)
+                else:
+                        print("GLUPY TRACE RMDIR CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_rmdir(frame, cookie, this, op_ret, op_errno,
+                                preparent, postparent, xdata)
+                return 0
+
+        def stat_fop(self, frame, this, loc, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE STAT FOP- {0:d}: gfid={1:s}; " +
+                      " path={2:s}").format(unique, gfid, loc.contents.path)
+                self.gfids[key] = gfid
+                dl.wind_stat(frame, POINTER(xlator_t)(), loc, xdata)
+                return 0
+
+        def stat_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
+                     xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        statstr = trace_stat2str(buf)
+                        print("GLUPY TRACE STAT CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d};  *buf={3:s};").format(unique,
+                                                                   gfid,
+                                                                   op_ret,
+                                                                   statstr)
+                else:
+                        print("GLUPY TRACE STAT CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_stat(frame, cookie, this, op_ret, op_errno,
+                               buf, xdata)
+                return 0
+
+        def fstat_fop(self, frame, this, fd, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE FSTAT FOP- {0:d}:  gfid={1:s}; " +
+                      "fd={2:s}").format(unique, gfid, fd)
+                self.gfids[key] = gfid
+                dl.wind_fstat(frame, POINTER(xlator_t)(), fd, xdata)
+                return 0
+
+        def fstat_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
+                      xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        statstr = trace_stat2str(buf)
+                        print("GLUPY TRACE FSTAT CBK- {0:d}: gfid={1:s} "+
+                              " op_ret={2:d}; *buf={3:s}").format(unique,
+                                                                  gfid,
+                                                                  op_ret,
+                                                                  statstr)
+                else:
+                        print("GLUPY TRACE FSTAT CBK- {0:d}: gfid={1:s} "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique.
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_fstat(frame, cookie, this, op_ret, op_errno,
+                                buf, xdata)
+                return 0
+
+        def statfs_fop(self, frame, this, loc, xdata):
+                unique = dl.get_rootunique(frame)
+                if loc.contents.inode:
+                        gfid = uuid2str(loc.contents.inode.contents.gfid)
+                else:
+                        gfid = "0"
+                print("GLUPY TRACE STATFS FOP- {0:d}: gfid={1:s}; "+
+                      "path={2:s}").format(unique, gfid, loc.contents.path)
+                dl.wind_statfs(frame, POINTER(xlator_t)(), loc, xdata)
+                return 0
+
+        def statfs_cbk(self, frame, cookie, this, op_ret, op_errno, buf,
+                       xdata):
+                unique = dl.get_rootunique(frame)
+                if op_ret == 0:
+                        #TBD: print buf (pointer to an iovec type object)
+                        print("GLUPY TRACE STATFS CBK {0:d}: "+
+                              "op_ret={1:d}").format(unique, op_ret)
+                else:
+                        print("GLUPY TRACE STATFS CBK-  {0:d}"+
+                              "op_ret={1:d}; op_errno={2:d}").format(unique,
+                                                                     op_ret,
+                                                                     op_errno)
+                dl.unwind_statfs(frame, cookie, this, op_ret, op_errno,
+                                 buf, xdata)
+                return 0
+
+        def getxattr_fop(self, frame, this, loc, name, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE GETXATTR FOP- {0:d}: gfid={1:s}; path={2:s};"+
+                      " name={3:s}").format(unique, gfid, loc.contents.path,
+                                            name)
+                self.gfids[key]=gfid
+                dl.wind_getxattr(frame, POINTER(xlator_t)(), loc, name, xdata)
+                return 0
+
+        def getxattr_cbk(self, frame, cookie, this, op_ret, op_errno,
+                         dictionary, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE GETXATTR CBK- {0:d}: gfid={1:s}; "+
+                      "op_ret={2:d}; op_errno={3:d}; "+
+                      " dictionary={4:s}").format(unique, gfid, op_ret, op_errno,
+                                                  dictionary)
+                del self.gfids[key]
+                dl.unwind_getxattr(frame, cookie, this, op_ret, op_errno,
+                                   dictionary, xdata)
+                return 0
+
+        def fgetxattr_fop(self, frame, this, fd, name, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE FGETXATTR FOP- {0:d}: gfid={1:s}; fd={2:s}; "+
+                      "name={3:s}").format(unique, gfid, fd, name)
+                self.gfids[key] = gfid
+                dl.wind_fgetxattr(frame, POINTER(xlator_t)(), fd, name, xdata)
+                return 0
+
+        def fgetxattr_cbk(self, frame, cookie, this, op_ret, op_errno,
+                          dictionary, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE FGETXATTR CBK- {0:d}: gfid={1:s}; "+
+                      "op_ret={2:d}; op_errno={3:d};"+
+                      " dictionary={4:s}").format(unique, gfid, op_ret,
+                                                  op_errno, dictionary)
+                del self.gfids[key]
+                dl.unwind_fgetxattr(frame, cookie, this, op_ret, op_errno,
+                                    dictionary, xdata)
+                return 0
+
+        def setxattr_fop(self, frame, this, loc, dictionary, flags, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE SETXATTR FOP- {0:d}:  gfid={1:s}; path={2:s};"+
+                      " flags={3:d}").format(unique, gfid, loc.contents.path,
+                                             flags)
+                self.gfids[key] = gfid
+                dl.wind_setxattr(frame, POINTER(xlator_t)(), loc, dictionary,
+                                 flags, xdata)
+                return 0
+
+        def setxattr_cbk(self, frame, cookie, this, op_ret, op_errno, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE SETXATTR CBK- {0:d}: gfid={1:s}; "+
+                      "op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
+                                                             op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_setxattr(frame, cookie, this, op_ret, op_errno,
+                                   xdata)
+                return 0
+
+        def fsetxattr_fop(self, frame, this, fd, dictionary, flags, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(fd.contents.inode.contents.gfid)
+                print("GLUPY TRACE FSETXATTR FOP- {0:d}: gfid={1:s}; fd={2:p}; "+
+                      "flags={3:d}").format(unique, gfid, fd, flags)
+                self.gfids[key] = gfid
+                dl.wind_fsetxattr(frame, POINTER(xlator_t)(), fd, dictionary,
+                                  flags, xdata)
+                return 0
+
+        def fsetxattr_cbk(self, frame, cookie, this, op_ret, op_errno, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE FSETXATTR CBK- {0:d}: gfid={1:s};  "+
+                      "op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
+                                                             op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_fsetxattr(frame, cookie, this, op_ret, op_errno,
+                                   xdata)
+                return 0
+
+        def removexattr_fop(self, frame, this, loc, name, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE REMOVEXATTR FOP- {0:d}:  gfid={1:s}; "+
+                      "path={2:s}; name={3:s}").format(unique, gfid,
+                                                       loc.contents.path,
+                                                       name)
+                self.gfids[key] = gfid
+                dl.wind_removexattr(frame, POINTER(xlator_t)(), loc, name,
+                                    xdata)
+                return 0
+
+        def removexattr_cbk(self, frame, cookie, this, op_ret, op_errno,
+                            xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                print("GLUPY TRACE REMOVEXATTR CBK- {0:d}: gfid={1:s} "+
+                      " op_ret={2:d}; op_errno={3:d}").format(unique, gfid,
+                                                              op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_removexattr(frame, cookie, this, op_ret, op_errno,
+                                      xdata)
+                return 0
+
+        def link_fop(self, frame, this, oldloc, newloc, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                if (newloc.contents.inode):
+                        newgfid = uuid2str(newloc.contents.inode.contents.gfid)
+                else:
+                        newgfid = "0"
+                oldgfid = uuid2str(oldloc.contents.inode.contents.gfid)
+                print("GLUPY TRACE LINK FOP-{0:d}: oldgfid={1:s}; oldpath={2:s};"+
+                      "newgfid={3:s};"+
+                      "newpath={4:s}").format(unique, oldgfid,
+                                              oldloc.contents.path,
+                                              newgfid,
+                                              newloc.contents.path)
+                self.gfids[key] =  oldgfid
+                dl.wind_link(frame, POINTER(xlator_t)(), oldloc, newloc,
+                             xdata)
+                return 0
+
+        def link_cbk(self, frame, cookie, this, op_ret, op_errno, inode, buf,
+                     preparent, postparent, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        statstr = trace_stat2str(buf)
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE LINK CBK- {0:d}: op_ret={1:d} "+
+                              "*stbuf={2:s}; *prebuf={3:s}; "+
+                              "*postbuf={4:s} ").format(unique, op_ret, statstr,
+                                                        preparentstr,
+                                                        postparentstr)
+                else:
+                        print("GLUPY TRACE LINK CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; "+
+                              "op_errno={3:d}").format(unique, gfid,
+                                                       op_ret, op_errno)
+                del self.gfids[key]
+                dl.unwind_link(frame, cookie, this, op_ret, op_errno, inode,
+                               buf, preparent, postparent, xdata)
+                return 0
+
+        def unlink_fop(self, frame, this, loc, xflag, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE UNLINK FOP- {0:d}; gfid={1:s}; path={2:s}; "+
+                      "flag={3:d}").format(unique, gfid, loc.contents.path,
+                                           xflag)
+                self.gfids[key] = gfid
+                dl.wind_unlink(frame, POINTER(xlator_t)(), loc, xflag,
+                               xdata)
+                return 0
+
+        def unlink_cbk(self, frame, cookie, this, op_ret, op_errno,
+                       preparent, postparent, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE UNLINK CBK- {0:d}: gfid ={1:s}; "+
+                              "op_ret={2:d}; *prebuf={3:s}; "+
+                              "*postbuf={4:s} ").format(unique, gfid, op_ret,
+                                                        preparentstr,
+                                                        postparentstr)
+                else:
+                        print("GLUPY TRACE UNLINK CBK: {0:d}: gfid ={1:s}; "+
+                              "op_ret={2:d}; "+
+                              "op_errno={3:d}").format(unique, gfid, op_ret,
+                                                       op_errno)
+                del self.gfids[key]
+                dl.unwind_unlink(frame, cookie, this, op_ret, op_errno,
+                                 preparent, postparent, xdata)
+                return 0
+
+        def readlink_fop(self, frame, this, loc, size, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE READLINK FOP- {0:d}:  gfid={1:s}; path={2:s};"+
+                      " size={3:d}").format(unique, gfid, loc.contents.path,
+                                            size)
+                self.gfids[key] = gfid
+                dl.wind_readlink(frame, POINTER(xlator_t)(), loc, size,
+                               xdata)
+                return 0
+
+        def readlink_cbk(self, frame, cookie, this, op_ret, op_errno,
+                         buf, stbuf, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid  = self.gfids[key]
+                if op_ret == 0:
+                        statstr = trace_stat2str(stbuf)
+                        print("GLUPY TRACE READLINK CBK- {0:d}: gfid={1:s} "+
+                              " op_ret={2:d}; op_errno={3:d}; *prebuf={4:s}; "+
+                              "*postbuf={5:s} ").format(unique, gfid,
+                                                        op_ret, op_errno,
+                                                        buf, statstr)
+                else:
+                        print("GLUPY TRACE READLINK CBK- {0:d}: gfid={1:s} "+
+                              " op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                      gfid,
+                                                                      op_ret,
+                                                                      op_errno)
+                del self.gfids[key]
+                dl.unwind_readlink(frame, cookie, this, op_ret, op_errno, buf,
+                                   stbuf, xdata)
+                return 0
+
+        def symlink_fop(self, frame, this, linkpath, loc, umask, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = uuid2str(loc.contents.inode.contents.gfid)
+                print("GLUPY TRACE SYMLINK FOP- {0:d}: gfid={1:s}; "+
+                      "linkpath={2:s}; path={3:s};"+
+                      "umask=0{4:o}").format(unique, gfid, linkpath,
+                                             loc.contents.path, umask)
+                self.gfids[key] = gfid
+                dl.wind_symlink(frame, POINTER(xlator_t)(), linkpath, loc,
+                                umask, xdata)
+                return 0
+
+        def symlink_cbk(self, frame, cookie, this, op_ret, op_errno,
+                        inode, buf, preparent, postparent, xdata):
+                unique = dl.get_rootunique(frame)
+                key = dl.get_id(frame)
+                gfid = self.gfids[key]
+                if op_ret == 0:
+                        statstr = trace_stat2str(buf)
+                        preparentstr = trace_stat2str(preparent)
+                        postparentstr = trace_stat2str(postparent)
+                        print("GLUPY TRACE SYMLINK CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; *stbuf={3:s}; *preparent={4:s}; "+
+                              "*postparent={5:s}").format(unique, gfid,
+                                                          op_ret, statstr,
+                                                          preparentstr,
+                                                          postparentstr)
+                else:
+                        print("GLUPY TRACE SYMLINK CBK- {0:d}: gfid={1:s}; "+
+                              "op_ret={2:d}; op_errno={3:d}").format(unique,
+                                                                     gfid,
+                                                                     op_ret,
+                                                                     op_errno)
+                del self.gfids[key]
+                dl.unwind_symlink(frame, cookie, this, op_ret, op_errno,
+                                  inode, buf, preparent, postparent, xdata)
+                return 0
diff --git a/xlators/features/glupy/examples/helloworld.py b/xlators/features/glupy/examples/helloworld.py
new file mode 100644
index 000000000..b565a4e5b
--- /dev/null
+++ b/xlators/features/glupy/examples/helloworld.py
@@ -0,0 +1,19 @@
+import sys
+from gluster.glupy import *
+
+class xlator (Translator):
+
+    def __init__(self, c_this):
+        Translator.__init__(self, c_this)
+
+    def lookup_fop(self, frame, this, loc, xdata):
+        print "Python xlator: Hello!"
+        dl.wind_lookup(frame, POINTER(xlator_t)(), loc, xdata)
+        return 0
+
+    def lookup_cbk(self, frame, cookie, this, op_ret, op_errno, inode, buf,
+                   xdata, postparent):
+        print "Python xlator: Hello again!"
+        dl.unwind_lookup(frame, cookie, this, op_ret, op_errno, inode, buf,
+                         xdata, postparent)
+        return 0
diff --git a/xlators/features/glupy/examples/negative.py b/xlators/features/glupy/examples/negative.py
new file mode 100644
index 000000000..e7a4fc07c
--- /dev/null
+++ b/xlators/features/glupy/examples/negative.py
@@ -0,0 +1,91 @@
+import sys
+from uuid import UUID
+from gluster.glupy import *
+
+# Negative-lookup-caching example.  If a file wasn't there the last time we
+# looked, it's probably still not there.  This translator keeps track of
+# those failed lookups for us, and returns ENOENT without needing to pass the
+# call any further for repeated requests.
+
+# If we were doing this for real, we'd need separate caches for each xlator
+# instance.  The easiest way to do this would be to have xlator.__init__
+# "register" each instance in a module-global dict, with the key as the C
+# translator address and the value as the xlator object itself.  For testing
+# and teaching, it's sufficient just to have one cache.  The keys are parent
+# GFIDs, and the entries are lists of names within that parent that we know
+# don't exist.
+cache = {}
+
+# TBD: we need a better way of handling per-request data (frame->local in C).
+dl.get_id.restype = c_long
+dl.get_id.argtypes = [ POINTER(call_frame_t) ]
+
+def uuid2str (gfid):
+    return str(UUID(''.join(map("{0:02x}".format, gfid))))
+
+class xlator (Translator):
+
+    def __init__ (self, c_this):
+        self.requests = {}
+        Translator.__init__(self,c_this)
+
+    def lookup_fop (self, frame, this, loc, xdata):
+        pargfid = uuid2str(loc.contents.pargfid)
+        print "lookup FOP: %s:%s" % (pargfid, loc.contents.name)
+        # Check the cache.
+        if cache.has_key(pargfid):
+            if loc.contents.name in cache[pargfid]:
+                print "short-circuiting for %s:%s" % (pargfid,
+                    loc.contents.name)
+                dl.unwind_lookup(frame,0,this,-1,2,None,None,None,None)
+                return 0
+        key = dl.get_id(frame)
+        self.requests[key] = (pargfid, loc.contents.name[:])
+        # TBD: get real child xl from init, pass it here
+        dl.wind_lookup(frame,POINTER(xlator_t)(),loc,xdata)
+        return 0
+
+    def lookup_cbk (self, frame, cookie, this, op_ret, op_errno, inode, buf,
+                    xdata, postparent):
+        print "lookup CBK: %d (%d)" % (op_ret, op_errno)
+        key = dl.get_id(frame)
+        pargfid, name = self.requests[key]
+        # Update the cache.
+        if op_ret == 0:
+            print "found %s, removing from cache" % name
+            if cache.has_key(pargfid):
+                cache[pargfid].discard(name)
+        elif op_errno == 2:    # ENOENT
+            print "failed to find %s, adding to cache" % name
+            if cache.has_key(pargfid):
+                cache[pargfid].add(name)
+            else:
+                cache[pargfid] = set([name])
+        del self.requests[key]
+        dl.unwind_lookup(frame,cookie,this,op_ret,op_errno,
+                         inode,buf,xdata,postparent)
+        return 0
+
+    def create_fop (self, frame, this, loc, flags, mode, umask, fd, xdata):
+        pargfid = uuid2str(loc.contents.pargfid)
+        print "create FOP: %s:%s" % (pargfid, loc.contents.name)
+        key = dl.get_id(frame)
+        self.requests[key] = (pargfid, loc.contents.name[:])
+        # TBD: get real child xl from init, pass it here
+        dl.wind_create(frame,POINTER(xlator_t)(),loc,flags,mode,umask,fd,xdata)
+        return 0
+
+    def create_cbk (self, frame, cookie, this, op_ret, op_errno, fd, inode,
+                    buf, preparent, postparent, xdata):
+        print "create CBK: %d (%d)" % (op_ret, op_errno)
+        key = dl.get_id(frame)
+        pargfid, name = self.requests[key]
+        # Update the cache.
+        if op_ret == 0:
+            print "created %s, removing from cache" % name
+            if cache.has_key(pargfid):
+                cache[pargfid].discard(name)
+        del self.requests[key]
+        dl.unwind_create(frame,cookie,this,op_ret,op_errno,fd,inode,buf,
+                         preparent,postparent,xdata)
+        return 0
diff --git a/xlators/features/glupy/src/Makefile.am b/xlators/features/glupy/src/Makefile.am
new file mode 100644
index 000000000..ae7b6d14d
--- /dev/null
+++ b/xlators/features/glupy/src/Makefile.am
@@ -0,0 +1,21 @@
+xlator_LTLIBRARIES = glupy.la
+
+# Ensure GLUSTER_PYTHON_PATH is passed to glupy.so
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+glupydir = $(xlatordir)/glupy
+AM_CPPFLAGS = $(PYTHONDEV_CPPFLAGS) $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src -isystem $(BUILD_PYTHON_INC)
+AM_CFLAGS = $(PYTHONDEV_CFLAGS) -Wall -fno-strict-aliasing -DGLUSTER_PYTHON_PATH=\"$(glupydir)\" $(GF_CFLAGS)
+
+# Flags to build glupy.so with
+glupy_la_LDFLAGS = $(PYTHONDEV_LDFLAGS) -module -avoid-version -shared -nostartfiles
+glupy_la_SOURCES = glupy.c
+glupy_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la \
+	 -lpthread -l$(BUILD_PYTHON_LIB)
+
+noinst_HEADERS = glupy.h
+
+# Install glupy.py into the Python site-packages area
+pyglupydir = $(pythondir)/gluster
+pyglupy_PYTHON = glupy.py
+
+CLEANFILES =
diff --git a/xlators/features/glupy/src/glupy.c b/xlators/features/glupy/src/glupy.c
new file mode 100644
index 000000000..7492124dd
--- /dev/null
+++ b/xlators/features/glupy/src/glupy.c
@@ -0,0 +1,2490 @@
+/*
+  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
+  This file is part of GlusterFS.
+
+  GlusterFS is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published
+  by the Free Software Foundation; either version 3 of the License,
+  or (at your option) any later version.
+
+  GlusterFS is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with this program.  If not, see
+  <http://www.gnu.org/licenses/>.
+*/
+
+#include <ctype.h>
+#include <sys/uio.h>
+#include <Python.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "xlator.h"
+#include "logging.h"
+#include "defaults.h"
+
+#include "glupy.h"
+
+/* UTILITY FUNCTIONS FOR FOP-SPECIFIC CODE */
+
+pthread_key_t gil_init_key;
+
+PyGILState_STATE
+glupy_enter (void)
+{
+#if 0
+        if (!pthread_getspecific(gil_init_key)) {
+                PyEval_ReleaseLock();
+                (void)pthread_setspecific(gil_init_key,(void *)1);
+        }
+#endif
+
+        return PyGILState_Ensure();
+}
+
+void
+glupy_leave (PyGILState_STATE gstate)
+{
+        PyGILState_Release(gstate);
+}
+
+/* FOP: LOOKUP */
+
+int32_t
+glupy_lookup_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, inode_t *inode,
+                  struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_LOOKUP]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_lookup_cbk_t)(priv->cbks[GLUPY_LOOKUP]))(
+                frame, cookie, this, op_ret, op_errno,
+                inode, buf, xdata, postparent);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno, inode, buf,
+                             xdata, postparent);
+        return 0;
+}
+
+int32_t
+glupy_lookup (call_frame_t *frame, xlator_t *this, loc_t *loc,
+              dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_LOOKUP]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_lookup_t)(priv->fops[GLUPY_LOOKUP]))(
+                frame, this, loc, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, loc, xdata);
+        return 0;
+}
+
+void
+wind_lookup (call_frame_t *frame, xlator_t *xl, loc_t *loc, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_lookup_cbk,xl,xl->fops->lookup,loc,xdata);
+}
+
+void
+unwind_lookup (call_frame_t *frame, long cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, inode_t *inode,
+               struct iatt *buf, dict_t *xdata, struct iatt *postparent)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(lookup,frame,op_ret,op_errno,
+                            inode,buf,xdata,postparent);
+}
+
+void
+set_lookup_fop (long py_this, fop_lookup_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_LOOKUP] = (long)fop;
+}
+
+void
+set_lookup_cbk (long py_this, fop_lookup_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_LOOKUP] = (long)cbk;
+}
+
+/* FOP: CREATE */
+
+int32_t
+glupy_create_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+                  struct iatt *buf, struct iatt *preparent,
+                  struct iatt *postparent, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_CREATE]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_create_cbk_t)(priv->cbks[GLUPY_CREATE]))(
+                frame, cookie, this, op_ret, op_errno,
+                fd, inode, buf, preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_create (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
+              mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_CREATE]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_create_t)(priv->fops[GLUPY_CREATE]))(
+                frame, this, loc, flags, mode, umask, fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_create_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->create, loc, flags, mode, umask,
+                    fd, xdata);
+        return 0;
+}
+
+void
+wind_create (call_frame_t *frame, xlator_t *xl, loc_t *loc, int32_t flags,
+             mode_t mode, mode_t umask, fd_t *fd, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_create_cbk,xl, xl->fops->create,
+                    loc, flags, mode, umask, fd, xdata);
+}
+
+void
+unwind_create (call_frame_t *frame, long cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, fd_t *fd, inode_t *inode,
+               struct iatt *buf, struct iatt *preparent,
+               struct iatt *postparent, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
+                             preparent, postparent, xdata);
+}
+
+void
+set_create_fop (long py_this, fop_create_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_CREATE] = (long)fop;
+}
+
+void
+set_create_cbk (long py_this, fop_create_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_CREATE] = (long)cbk;
+}
+
+/* FOP: OPEN */
+
+int32_t
+glupy_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_OPEN]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_open_cbk_t)(priv->cbks[GLUPY_OPEN]))(
+                frame, cookie, this, op_ret, op_errno,
+                fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+        return 0;
+}
+
+int32_t
+glupy_open (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            int32_t flags, fd_t *fd, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_OPEN]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_open_t)(priv->fops[GLUPY_OPEN]))(
+                frame, this, loc, flags, fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_open_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->open, loc, flags, fd, xdata);
+        return 0;
+}
+
+void
+wind_open (call_frame_t *frame, xlator_t *xl, loc_t *loc, int32_t flags,
+           fd_t *fd, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_open_cbk, xl, xl->fops->open, loc, flags,
+                    fd, xdata);
+}
+
+void
+unwind_open (call_frame_t *frame, long cookie, xlator_t *this,
+             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
+}
+
+void
+set_open_fop (long py_this, fop_open_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->fops[GLUPY_OPEN] = (long)fop;
+}
+
+void
+set_open_cbk (long py_this, fop_open_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->cbks[GLUPY_OPEN] = (long)cbk;
+}
+
+/* FOP: READV */
+
+int32_t
+glupy_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iovec *vector,
+                 int32_t count, struct iatt *stbuf, struct iobref *iobref,
+                 dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_READV]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_readv_cbk_t)(priv->cbks[GLUPY_READV]))(
+                frame, cookie, this, op_ret, op_errno,
+                vector, count, stbuf, iobref, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector,
+                             count, stbuf, iobref, xdata);
+        return 0;
+}
+
+int32_t
+glupy_readv (call_frame_t *frame, xlator_t *this, fd_t *fd,
+             size_t size, off_t offset, uint32_t flags, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_READV]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_readv_t)(priv->fops[GLUPY_READV]))(
+                frame, this, fd, size, offset, flags, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_readv_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readv, fd, size, offset,
+                    flags, xdata);
+        return 0;
+}
+
+void
+wind_readv (call_frame_t *frame, xlator_t *xl, fd_t *fd, size_t size,
+            off_t offset, uint32_t flags, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_readv_cbk, xl, xl->fops->readv, fd, size,
+                    offset, flags, xdata);
+}
+
+void
+unwind_readv (call_frame_t *frame, long cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, struct iovec *vector,
+              int32_t count, struct iatt *stbuf, struct iobref *iobref,
+              dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno, vector,
+                             count, stbuf, iobref, xdata);
+}
+
+void
+set_readv_fop (long py_this, fop_readv_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->fops[GLUPY_READV] = (long)fop;
+}
+
+void
+set_readv_cbk (long py_this, fop_readv_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->cbks[GLUPY_READV] = (long)cbk;
+}
+
+/* FOP: WRITEV */
+
+int32_t
+glupy_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+                  struct iatt *postbuf, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_WRITEV]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_writev_cbk_t)(priv->cbks[GLUPY_WRITEV]))(
+                frame, cookie, this, op_ret, op_errno,
+                prebuf, postbuf, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+        return 0;
+}
+
+int32_t
+glupy_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              struct iovec *vector, int32_t count, off_t offset,
+              uint32_t flags, struct iobref *iobref, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_WRITEV]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_writev_t)(priv->fops[GLUPY_WRITEV]))(
+                frame, this, fd, vector, count, offset, flags,
+                iobref, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_writev_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->writev, fd, vector, count,
+                    offset, flags, iobref, xdata);
+        return 0;
+}
+
+void
+wind_writev (call_frame_t *frame, xlator_t *xl, fd_t *fd, struct iovec *vector,
+             int32_t count, off_t offset, uint32_t flags, struct iobref *iobref,
+             dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_writev_cbk, xl, xl->fops->writev, fd, vector,
+                    count, offset, flags, iobref, xdata);
+}
+
+void
+unwind_writev (call_frame_t *frame, long cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
+               struct iatt *postbuf, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf,
+                             postbuf, xdata);
+}
+
+void
+set_writev_fop (long py_this, fop_writev_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->fops[GLUPY_WRITEV] = (long)fop;
+}
+
+void
+set_writev_cbk (long py_this, fop_writev_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+        priv->cbks[GLUPY_WRITEV] = (long)cbk;
+}
+
+
+/* FOP: OPENDIR */
+
+int32_t
+glupy_opendir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, fd_t *fd,
+                   dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_OPENDIR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_opendir_cbk_t)(priv->cbks[GLUPY_OPENDIR]))(
+                frame, cookie, this, op_ret, op_errno,
+                fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (opendir, frame, op_ret, op_errno, fd, xdata);
+        return 0;
+}
+
+int32_t
+glupy_opendir (call_frame_t *frame, xlator_t *this, loc_t *loc,
+               fd_t *fd, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_OPENDIR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_opendir_t)(priv->fops[GLUPY_OPENDIR]))(
+                frame, this, loc, fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_opendir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->opendir, loc, fd, xdata);
+        return 0;
+}
+
+void
+wind_opendir (call_frame_t *frame, xlator_t *xl, loc_t *loc, fd_t *fd, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_opendir_cbk,xl,xl->fops->opendir,loc,fd,xdata);
+}
+
+void
+unwind_opendir (call_frame_t *frame, long cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(opendir,frame,op_ret,op_errno,
+                            fd,xdata);
+}
+
+void
+set_opendir_fop (long py_this, fop_opendir_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_OPENDIR] = (long)fop;
+}
+
+void
+set_opendir_cbk (long py_this, fop_opendir_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_OPENDIR] = (long)cbk;
+}
+
+/* FOP: READDIR */
+
+int32_t
+glupy_readdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                   dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_READDIR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_readdir_cbk_t)(priv->cbks[GLUPY_READDIR]))(
+                frame, cookie, this, op_ret, op_errno,
+                entries,  xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, entries,
+                             xdata);
+        return 0;
+}
+
+int32_t
+glupy_readdir (call_frame_t *frame, xlator_t *this, fd_t *fd,
+               size_t size, off_t offset, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_READDIR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_readdir_t)(priv->fops[GLUPY_READDIR]))(
+                frame, this, fd, size, offset, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_readdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readdir,fd, size, offset, xdata);
+        return 0;
+}
+
+void
+wind_readdir(call_frame_t *frame, xlator_t *xl, fd_t *fd, size_t size,
+             off_t offset, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_readdir_cbk,xl,xl->fops->readdir,fd,size,offset,xdata);
+}
+
+void
+unwind_readdir (call_frame_t *frame, long cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(readdir,frame,op_ret,op_errno,
+                            entries, xdata);
+}
+
+void
+set_readdir_fop (long py_this, fop_readdir_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_READDIR] = (long)fop;
+}
+
+void
+set_readdir_cbk (long py_this, fop_readdir_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_READDIR] = (long)cbk;
+}
+
+
+/* FOP: READDIRP */
+
+int32_t
+glupy_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                    dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_READDIRP]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_readdirp_cbk_t)(priv->cbks[GLUPY_READDIRP]))(
+                frame, cookie, this, op_ret, op_errno,
+                entries,  xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries,
+                             xdata);
+        return 0;
+}
+
+int32_t
+glupy_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                size_t size, off_t offset, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_READDIRP]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_readdirp_t)(priv->fops[GLUPY_READDIRP]))(
+                frame, this, fd, size, offset, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_readdirp_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readdirp,fd, size, offset, xdata);
+        return 0;
+}
+
+void
+wind_readdirp (call_frame_t *frame, xlator_t *xl, fd_t *fd, size_t size,
+               off_t offset, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_readdirp_cbk,xl,xl->fops->readdirp,fd,size,offset,xdata);
+}
+
+void
+unwind_readdirp (call_frame_t *frame, long cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, gf_dirent_t *entries,
+                 dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(readdirp,frame,op_ret,op_errno,
+                            entries, xdata);
+}
+
+void
+set_readdirp_fop (long py_this, fop_readdirp_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_READDIRP] = (long)fop;
+}
+
+void
+set_readdirp_cbk (long py_this, fop_readdirp_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_READDIRP] = (long)cbk;
+}
+
+
+/* FOP:STAT */
+
+int32_t
+glupy_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_STAT]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_stat_cbk_t)(priv->cbks[GLUPY_STAT]))(
+                frame, cookie, this, op_ret, op_errno,
+                buf, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (stat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+int32_t
+glupy_stat (call_frame_t *frame, xlator_t *this, loc_t *loc,
+            dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_STAT]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_stat_t)(priv->fops[GLUPY_STAT]))(
+                frame, this, loc, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_stat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->stat, loc, xdata);
+        return 0;
+}
+
+void
+wind_stat (call_frame_t *frame, xlator_t *xl, loc_t *loc, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_stat_cbk,xl,xl->fops->stat,loc,xdata);
+}
+
+void
+unwind_stat (call_frame_t *frame, long cookie, xlator_t *this,
+             int32_t op_ret, int32_t op_errno, struct iatt *buf,
+             dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(stat,frame,op_ret,op_errno,
+                            buf,xdata);
+}
+
+void
+set_stat_fop (long py_this, fop_stat_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_STAT] = (long)fop;
+}
+
+void
+set_stat_cbk (long py_this, fop_stat_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_STAT] = (long)cbk;
+}
+
+
+/* FOP: FSTAT */
+
+int32_t
+glupy_fstat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, struct iatt *buf, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_FSTAT]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_fstat_cbk_t)(priv->cbks[GLUPY_FSTAT]))(
+                frame, cookie, this, op_ret, op_errno,
+                buf, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fstat, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+int32_t
+glupy_fstat (call_frame_t *frame, xlator_t *this, fd_t *fd,
+             dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_FSTAT]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_fstat_t)(priv->fops[GLUPY_FSTAT]))(
+                frame, this, fd, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_fstat_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
+        return 0;
+}
+
+void
+wind_fstat (call_frame_t *frame, xlator_t *xl, fd_t *fd, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_fstat_cbk,xl,xl->fops->fstat,fd,xdata);
+}
+
+void
+unwind_fstat (call_frame_t *frame, long cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, struct iatt *buf,
+              dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(fstat,frame,op_ret,op_errno,
+                            buf,xdata);
+}
+
+void
+set_fstat_fop (long py_this, fop_fstat_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_FSTAT] = (long)fop;
+}
+
+void
+set_fstat_cbk (long py_this, fop_fstat_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_FSTAT] = (long)cbk;
+}
+
+/* FOP:STATFS */
+
+int32_t
+glupy_statfs_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, struct statvfs *buf, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_STATFS]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_statfs_cbk_t)(priv->cbks[GLUPY_STATFS]))(
+                frame, cookie, this, op_ret, op_errno,
+                buf, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (statfs, frame, op_ret, op_errno, buf, xdata);
+        return 0;
+}
+
+int32_t
+glupy_statfs (call_frame_t *frame, xlator_t *this, loc_t *loc,
+              dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_STATFS]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_statfs_t)(priv->fops[GLUPY_STATFS]))(
+                frame, this, loc, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_statfs_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->statfs, loc, xdata);
+        return 0;
+}
+
+void
+wind_statfs (call_frame_t *frame, xlator_t *xl, loc_t *loc, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND(frame,glupy_statfs_cbk,xl,xl->fops->statfs,loc,xdata);
+}
+
+void
+unwind_statfs (call_frame_t *frame, long cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno, struct statvfs *buf,
+               dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT(statfs,frame,op_ret,op_errno,
+                            buf,xdata);
+}
+
+void
+set_statfs_fop (long py_this, fop_statfs_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_STATFS] = (long)fop;
+}
+
+void
+set_statfs_cbk (long py_this, fop_statfs_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_STATFS] = (long)cbk;
+}
+
+
+/* FOP: SETXATTR */
+
+int32_t
+glupy_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_SETXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_setxattr_cbk_t)(priv->cbks[GLUPY_SETXATTR]))(
+                frame, cookie, this, op_ret, op_errno,
+                xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+glupy_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_SETXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_setxattr_t)(priv->fops[GLUPY_SETXATTR]))(
+                frame, this, loc, dict, flags, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_setxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setxattr, loc, dict,
+                    flags, xdata);
+        return 0;
+}
+
+void
+wind_setxattr (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+               dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_setxattr_cbk, xl, xl->fops->setxattr,
+                    loc, dict, flags, xdata);
+}
+
+
+void
+unwind_setxattr (call_frame_t *frame, long cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+
+}
+
+void
+set_setxattr_fop (long py_this, fop_setxattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_SETXATTR] = (long)fop;
+}
+
+void
+set_setxattr_cbk (long py_this, fop_setxattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_SETXATTR] = (long)cbk;
+}
+
+/* FOP: GETXATTR */
+
+int32_t
+glupy_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *dict,
+                    dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_GETXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_getxattr_cbk_t)(priv->cbks[GLUPY_GETXATTR]))(
+                frame, cookie, this, op_ret, op_errno, dict,
+                xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict,
+                             xdata);
+        return 0;
+}
+
+int32_t
+glupy_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                const char *name, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_GETXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_getxattr_t)(priv->fops[GLUPY_GETXATTR]))(
+                frame, this, loc, name, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr, loc, name,
+                    xdata);
+        return 0;
+}
+
+void
+wind_getxattr (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+               const char *name, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_getxattr_cbk, xl, xl->fops->getxattr,
+                    loc, name, xdata);
+}
+
+
+void
+unwind_getxattr (call_frame_t *frame, long cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict,
+                 dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict,
+                             xdata);
+
+}
+
+
+void
+set_getxattr_fop (long py_this, fop_getxattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_GETXATTR] = (long)fop;
+}
+
+
+void
+set_getxattr_cbk (long py_this, fop_getxattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_GETXATTR] = (long)cbk;
+}
+
+/* FOP: FSETXATTR */
+
+int32_t
+glupy_fsetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_FSETXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_fsetxattr_cbk_t)(priv->cbks[GLUPY_FSETXATTR]))(
+                frame, cookie, this, op_ret, op_errno,
+                xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+glupy_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_FSETXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_fsetxattr_t)(priv->fops[GLUPY_FSETXATTR]))(
+                frame, this, fd, dict, flags, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_fsetxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict,
+                    flags, xdata);
+        return 0;
+}
+
+void
+wind_fsetxattr (call_frame_t *frame, xlator_t *xl, fd_t *fd,
+                dict_t *dict, int32_t flags, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_fsetxattr_cbk, xl, xl->fops->fsetxattr,
+                    fd, dict, flags, xdata);
+}
+
+
+void
+unwind_fsetxattr (call_frame_t *frame, long cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, xdata);
+
+}
+
+void
+set_fsetxattr_fop (long py_this, fop_fsetxattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_FSETXATTR] = (long)fop;
+}
+
+void
+set_fsetxattr_cbk (long py_this, fop_fsetxattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_FSETXATTR] = (long)cbk;
+}
+
+/* FOP: FGETXATTR */
+
+int32_t
+glupy_fgetxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *dict,
+                     dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_FGETXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_fgetxattr_cbk_t)(priv->cbks[GLUPY_FGETXATTR]))(
+                frame, cookie, this, op_ret, op_errno, dict,
+                xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict,
+                             xdata);
+        return 0;
+}
+
+int32_t
+glupy_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                 const char *name, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_FGETXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_fgetxattr_t)(priv->fops[GLUPY_FGETXATTR]))(
+                frame, this, fd, name, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_fgetxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr, fd, name,
+                    xdata);
+        return 0;
+}
+
+void
+wind_fgetxattr (call_frame_t *frame, xlator_t *xl, fd_t *fd,
+                const char *name, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_fgetxattr_cbk, xl, xl->fops->fgetxattr,
+                    fd, name, xdata);
+}
+
+
+void
+unwind_fgetxattr (call_frame_t *frame, long cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno, dict_t *dict,
+                  dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict,
+                             xdata);
+
+}
+
+
+void
+set_fgetxattr_fop (long py_this, fop_fgetxattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_FGETXATTR] = (long)fop;
+}
+
+
+void
+set_fgetxattr_cbk (long py_this, fop_fgetxattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_FGETXATTR] = (long)cbk;
+}
+
+/* FOP:REMOVEXATTR */
+
+int32_t
+glupy_removexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_REMOVEXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_removexattr_cbk_t)(priv->cbks[GLUPY_REMOVEXATTR]))(
+                frame, cookie, this, op_ret, op_errno, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+glupy_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                   const char *name, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_REMOVEXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_removexattr_t)(priv->fops[GLUPY_REMOVEXATTR]))(
+                frame, this, loc, name, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_removexattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr, loc, name,
+                    xdata);
+        return 0;
+}
+
+void
+wind_removexattr (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+                  const char *name, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_removexattr_cbk, xl, xl->fops->removexattr,
+                    loc, name, xdata);
+}
+
+
+void
+unwind_removexattr (call_frame_t *frame, long cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (removexattr, frame, op_ret, op_errno, xdata);
+
+}
+
+void
+set_removexattr_fop (long py_this, fop_removexattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_REMOVEXATTR] = (long)fop;
+}
+
+void
+set_removexattr_cbk (long py_this, fop_removexattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_REMOVEXATTR] = (long)cbk;
+}
+
+
+/* FOP:FREMOVEXATTR */
+
+int32_t
+glupy_fremovexattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_FREMOVEXATTR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_fremovexattr_cbk_t)(priv->cbks[GLUPY_FREMOVEXATTR]))(
+                frame, cookie, this, op_ret, op_errno, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
+
+int32_t
+glupy_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   const char *name, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_FREMOVEXATTR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_fremovexattr_t)(priv->fops[GLUPY_FREMOVEXATTR]))(
+                frame, this, fd, name, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_fremovexattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fremovexattr, fd, name,
+                    xdata);
+        return 0;
+}
+
+void
+wind_fremovexattr (call_frame_t *frame, xlator_t *xl, fd_t *fd,
+                   const char *name, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_fremovexattr_cbk, xl, xl->fops->fremovexattr,
+                    fd, name, xdata);
+}
+
+
+void
+unwind_fremovexattr (call_frame_t *frame, long cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (fremovexattr, frame, op_ret, op_errno, xdata);
+
+}
+
+void
+set_fremovexattr_fop (long py_this, fop_fremovexattr_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_FREMOVEXATTR] = (long)fop;
+}
+
+void
+set_fremovexattr_cbk (long py_this, fop_fremovexattr_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_FREMOVEXATTR] = (long)cbk;
+}
+
+
+/* FOP: LINK*/
+int32_t
+glupy_link_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_LINK]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_link_cbk_t)(priv->cbks[GLUPY_LINK]))(
+                frame, cookie, this, op_ret, op_errno,
+                inode, buf, preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_link(call_frame_t *frame, xlator_t *this, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_LINK]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_link_t)(priv->fops[GLUPY_LINK]))(
+                frame, this, oldloc, newloc, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_link_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->link, oldloc, newloc,
+                    xdata);
+        return 0;
+}
+
+void
+wind_link (call_frame_t *frame, xlator_t *xl, loc_t *oldloc, loc_t *newloc,
+           dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_link_cbk, xl, xl->fops->link,
+                    oldloc, newloc, xdata);
+}
+
+void
+unwind_link (call_frame_t *frame, long cookie, xlator_t *this,
+             int32_t op_ret, int32_t op_errno, inode_t *inode,
+             struct iatt *buf, struct iatt *preparent,
+             struct iatt *postparent, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (link, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+}
+
+void
+set_link_fop (long py_this, fop_link_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_LINK] = (long)fop;
+}
+
+void
+set_link_cbk (long py_this, fop_link_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_LINK] = (long)cbk;
+}
+
+/* FOP: SYMLINK*/
+int32_t
+glupy_symlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, inode_t *inode,
+                   struct iatt *buf, struct iatt *preparent,
+                   struct iatt *postparent, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_SYMLINK]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_symlink_cbk_t)(priv->cbks[GLUPY_SYMLINK]))(
+                frame, cookie, this, op_ret, op_errno,
+                inode, buf, preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_symlink(call_frame_t *frame, xlator_t *this, const char *linkname,
+              loc_t *loc, mode_t umask, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_SYMLINK]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_symlink_t)(priv->fops[GLUPY_SYMLINK]))(
+                frame, this, linkname, loc, umask, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_symlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->symlink, linkname, loc,
+                    umask, xdata);
+        return 0;
+}
+
+void
+wind_symlink (call_frame_t *frame, xlator_t *xl, const char *linkname,
+              loc_t *loc, mode_t umask, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_symlink_cbk, xl, xl->fops->symlink,
+                    linkname, loc, umask, xdata);
+}
+
+void
+unwind_symlink (call_frame_t *frame, long cookie, xlator_t *this,
+                int32_t op_ret, int32_t op_errno, inode_t *inode,
+                struct iatt *buf, struct iatt *preparent,
+                struct iatt *postparent, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (symlink, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+}
+
+void
+set_symlink_fop (long py_this, fop_symlink_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_SYMLINK] = (long)fop;
+}
+
+void
+set_symlink_cbk (long py_this, fop_symlink_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_SYMLINK] = (long)cbk;
+}
+
+
+/* FOP: READLINK */
+int32_t
+glupy_readlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, const char *path,
+                    struct iatt *buf, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_READLINK]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_readlink_cbk_t)(priv->cbks[GLUPY_READLINK]))(
+                frame, cookie, this, op_ret, op_errno,
+                path, buf, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path,
+                             buf, xdata);
+        return 0;
+}
+
+int32_t
+glupy_readlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                size_t size, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_READLINK]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_readlink_t)(priv->fops[GLUPY_READLINK]))(
+                frame, this, loc, size, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_readlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readlink, loc,
+                    size, xdata);
+        return 0;
+}
+
+void
+wind_readlink (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+               size_t size, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_readlink_cbk, xl, xl->fops->readlink,
+                    loc, size, xdata);
+}
+
+void
+unwind_readlink (call_frame_t *frame, long cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, const char *path,
+                 struct iatt *buf, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readlink, frame, op_ret, op_errno, path, buf,
+                             xdata);
+}
+
+void
+set_readlink_fop (long py_this, fop_readlink_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_READLINK] = (long)fop;
+}
+
+void
+set_readlink_cbk (long py_this, fop_readlink_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_READLINK] = (long)cbk;
+}
+
+
+/* FOP: UNLINK */
+
+int32_t
+glupy_unlink_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                  int32_t op_ret, int32_t op_errno,
+                  struct iatt *preparent, struct iatt *postparent,
+                  dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_UNLINK]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_unlink_cbk_t)(priv->cbks[GLUPY_UNLINK]))(
+                frame, cookie, this, op_ret, op_errno,
+                preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno, preparent,
+                             postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc,
+              int xflags, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_UNLINK]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_unlink_t)(priv->fops[GLUPY_UNLINK]))(
+                frame, this, loc, xflags, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_unlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->unlink, loc,
+                    xflags, xdata);
+        return 0;
+}
+
+void
+wind_unlink (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+             int xflags, dict_t *xdata)
+{
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_unlink_cbk, xl, xl->fops->unlink,
+                    loc, xflags, xdata);
+}
+
+void
+unwind_unlink (call_frame_t *frame, long cookie, xlator_t *this,
+               int32_t op_ret, int32_t op_errno,
+               struct iatt *preparent, struct iatt *postparent,
+               dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (unlink, frame, op_ret, op_errno,
+                             preparent, postparent, xdata);
+}
+
+void
+set_unlink_fop (long py_this, fop_unlink_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_UNLINK] = (long)fop;
+}
+
+void
+set_unlink_cbk (long py_this, fop_unlink_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_UNLINK] = (long)cbk;
+}
+
+
+/* FOP: MKDIR */
+
+int32_t
+glupy_mkdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, inode_t *inode,
+                 struct iatt *buf, struct iatt *preparent,
+                 struct iatt *postparent, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_MKDIR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_mkdir_cbk_t)(priv->cbks[GLUPY_MKDIR]))(
+                frame, cookie, this, op_ret, op_errno,
+                inode, buf, preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_mkdir (call_frame_t *frame, xlator_t *this, loc_t *loc, mode_t mode,
+             mode_t umask, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_MKDIR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_mkdir_t)(priv->fops[GLUPY_MKDIR]))(
+                frame, this, loc, mode, umask, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_mkdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->mkdir, loc, mode, umask,
+                    xdata);
+        return 0;
+}
+
+void
+wind_mkdir (call_frame_t *frame, xlator_t *xl, loc_t *loc, mode_t mode,
+            mode_t umask, dict_t *xdata)
+{
+
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_mkdir_cbk, xl, xl->fops->mkdir,
+                    loc, mode, umask, xdata);
+}
+
+void
+unwind_mkdir (call_frame_t *frame, long cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, inode_t *inode,
+              struct iatt *buf, struct iatt *preparent,
+              struct iatt *postparent, dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (mkdir, frame, op_ret, op_errno, inode, buf,
+                             preparent, postparent, xdata);
+}
+
+void
+set_mkdir_fop (long py_this, fop_mkdir_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_MKDIR] = (long)fop;
+}
+
+void
+set_mkdir_cbk (long py_this, fop_mkdir_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_MKDIR] = (long)cbk;
+}
+
+
+/* FOP: RMDIR */
+
+int32_t
+glupy_rmdir_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno,
+                 struct iatt *preparent, struct iatt *postparent,
+                 dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+
+        if (!priv->cbks[GLUPY_RMDIR]) {
+                goto unwind;
+        }
+
+        gstate = glupy_enter();
+        ret = ((fop_rmdir_cbk_t)(priv->cbks[GLUPY_RMDIR]))(
+                frame, cookie, this, op_ret, op_errno,
+                preparent, postparent, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno, preparent,
+                             postparent, xdata);
+        return 0;
+}
+
+int32_t
+glupy_rmdir (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             int xflags, dict_t *xdata)
+{
+        glupy_private_t *priv = this->private;
+        PyGILState_STATE gstate;
+        int32_t ret;
+        static long next_id = 0;
+
+        if (!priv->fops[GLUPY_RMDIR]) {
+                goto wind;
+        }
+
+        gstate = glupy_enter();
+        frame->local = (void *)++next_id;
+        ret = ((fop_rmdir_t)(priv->fops[GLUPY_RMDIR]))(
+                frame, this, loc, xflags, xdata);
+        glupy_leave(gstate);
+
+        return ret;
+
+wind:
+        STACK_WIND (frame, glupy_rmdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->rmdir, loc,
+                    xflags, xdata);
+        return 0;
+}
+
+void
+wind_rmdir (call_frame_t *frame, xlator_t *xl, loc_t *loc,
+            int xflags, dict_t *xdata)
+{
+
+        xlator_t        *this = THIS;
+
+        if (!xl || (xl == this)) {
+                xl = FIRST_CHILD(this);
+        }
+
+        STACK_WIND (frame, glupy_rmdir_cbk, xl, xl->fops->rmdir,
+                    loc, xflags, xdata);
+}
+
+void
+unwind_rmdir (call_frame_t *frame, long cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno,
+              struct iatt *preparent, struct iatt *postparent,
+              dict_t *xdata)
+{
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (rmdir, frame, op_ret, op_errno,
+                             preparent, postparent, xdata);
+}
+
+void
+set_rmdir_fop (long py_this, fop_rmdir_t fop)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->fops[GLUPY_RMDIR] = (long)fop;
+}
+
+void
+set_rmdir_cbk (long py_this, fop_rmdir_cbk_t cbk)
+{
+        glupy_private_t *priv   = ((xlator_t *)py_this)->private;
+
+        priv->cbks[GLUPY_RMDIR] = (long)cbk;
+}
+
+
+/* NON-FOP-SPECIFIC CODE */
+
+
+long
+get_id (call_frame_t *frame)
+{
+        return (long)(frame->local);
+}
+
+uint64_t
+get_rootunique (call_frame_t *frame)
+{
+        return frame->root->unique;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        if (!this)
+                return ret;
+
+        ret = xlator_mem_acct_init (this, gf_glupy_mt_end);
+
+        if (ret != 0) {
+                gf_log(this->name, GF_LOG_ERROR, "Memory accounting init"
+                       " failed");
+                return ret;
+        }
+
+        return ret;
+}
+
+int32_t
+init (xlator_t *this)
+{
+        glupy_private_t         *priv           = NULL;
+        char                    *module_name    = NULL;
+        PyObject                *py_mod_name    = NULL;
+        PyObject                *py_init_func   = NULL;
+        PyObject                *py_args        = NULL;
+        PyObject                *syspath        = NULL;
+	PyObject                *path           = NULL;
+	static gf_boolean_t      py_inited      = _gf_false;
+        void *                   err_cleanup    = &&err_return;
+
+        if (dict_get_str(this->options,"module-name",&module_name) != 0) {
+                gf_log (this->name, GF_LOG_ERROR, "missing module-name");
+                return -1;
+        }
+
+	priv = GF_CALLOC (1, sizeof (glupy_private_t), gf_glupy_mt_priv);
+        if (!priv) {
+                goto *err_cleanup;
+        }
+        this->private = priv;
+        err_cleanup = &&err_free_priv;
+
+	if (!py_inited) {
+                Py_Initialize();
+                PyEval_InitThreads();
+#if 0
+                (void)pthread_key_create(&gil_init_key,NULL);
+                (void)pthread_setspecific(gil_init_key,(void *)1);
+#endif
+                /* PyEval_InitThreads takes this "for" us.  No thanks. */
+                PyEval_ReleaseLock();
+                py_inited = _gf_true;
+        }
+
+	/* Adjust python's path */
+	syspath = PySys_GetObject("path");
+	path = PyString_FromString(GLUSTER_PYTHON_PATH);
+	PyList_Append(syspath, path);
+	Py_DECREF(path);
+
+	py_mod_name = PyString_FromString(module_name);
+        if (!py_mod_name) {
+                gf_log (this->name, GF_LOG_ERROR, "could not create name");
+                if (PyErr_Occurred()) {
+                        PyErr_Print();
+                }
+                goto *err_cleanup;
+        }
+
+        gf_log (this->name, GF_LOG_DEBUG, "py_mod_name = %s", module_name);
+        priv->py_module = PyImport_Import(py_mod_name);
+        Py_DECREF(py_mod_name);
+        if (!priv->py_module) {
+                gf_log (this->name, GF_LOG_ERROR, "Python import failed");
+                if (PyErr_Occurred()) {
+                        PyErr_Print();
+                }
+                goto *err_cleanup;
+        }
+        gf_log (this->name, GF_LOG_INFO, "Import of %s succeeded", module_name);
+        err_cleanup = &&err_deref_module;
+
+        py_init_func = PyObject_GetAttrString(priv->py_module, "xlator");
+        if (!py_init_func || !PyCallable_Check(py_init_func)) {
+                gf_log (this->name, GF_LOG_ERROR, "missing init func");
+                if (PyErr_Occurred()) {
+                        PyErr_Print();
+                }
+                goto *err_cleanup;
+        }
+        err_cleanup = &&err_deref_init;
+
+        py_args = PyTuple_New(1);
+        if (!py_args) {
+                gf_log (this->name, GF_LOG_ERROR, "could not create args");
+                if (PyErr_Occurred()) {
+                        PyErr_Print();
+                }
+                goto *err_cleanup;
+        }
+        PyTuple_SetItem(py_args,0,PyLong_FromLong((long)this));
+
+        /* TBD: pass in list of children */
+        priv->py_xlator = PyObject_CallObject(py_init_func, py_args);
+        Py_DECREF(py_args);
+        if (!priv->py_xlator) {
+                gf_log (this->name, GF_LOG_ERROR, "Python init failed");
+                if (PyErr_Occurred()) {
+                        PyErr_Print();
+                }
+                goto *err_cleanup;
+        }
+        gf_log (this->name, GF_LOG_DEBUG, "init returned %p", priv->py_xlator);
+
+        return 0;
+
+err_deref_init:
+        Py_DECREF(py_init_func);
+err_deref_module:
+        Py_DECREF(priv->py_module);
+err_free_priv:
+        GF_FREE(priv);
+err_return:
+        return -1;
+}
+
+void
+fini (xlator_t *this)
+{
+        glupy_private_t *priv = this->private;
+
+        if (!priv)
+                return;
+        Py_DECREF(priv->py_xlator);
+        Py_DECREF(priv->py_module);
+        this->private = NULL;
+        GF_FREE (priv);
+
+        return;
+}
+
+struct xlator_fops fops = {
+        .lookup       = glupy_lookup,
+        .create       = glupy_create,
+        .open         = glupy_open,
+        .readv        = glupy_readv,
+        .writev       = glupy_writev,
+        .opendir      = glupy_opendir,
+        .readdir      = glupy_readdir,
+        .stat         = glupy_stat,
+        .fstat        = glupy_fstat,
+        .setxattr     = glupy_setxattr,
+        .getxattr     = glupy_getxattr,
+        .fsetxattr    = glupy_fsetxattr,
+        .fgetxattr    = glupy_fgetxattr,
+        .removexattr  = glupy_removexattr,
+        .fremovexattr = glupy_fremovexattr,
+        .link         = glupy_link,
+        .unlink       = glupy_unlink,
+        .readlink     = glupy_readlink,
+        .symlink      = glupy_symlink,
+        .mkdir        = glupy_mkdir,
+        .rmdir        = glupy_rmdir,
+        .statfs       = glupy_statfs,
+        .readdirp     = glupy_readdirp
+};
+
+struct xlator_cbks cbks = {
+};
+
+struct volume_options options[] = {
+        { .key  = {NULL} },
+};
diff --git a/xlators/features/glupy/src/glupy.h b/xlators/features/glupy/src/glupy.h
new file mode 100644
index 000000000..8661fce88
--- /dev/null
+++ b/xlators/features/glupy/src/glupy.h
@@ -0,0 +1,69 @@
+/*
+   Copyright (c) 2006-2011 Gluster, Inc. <http://www.gluster.com>
+   This file is part of GlusterFS.
+
+   GlusterFS is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published
+   by the Free Software Foundation; either version 3 of the License,
+   or (at your option) any later version.
+
+   GlusterFS is distributed in the hope that it will be useful, but
+   WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see
+   <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef __GLUPY_H__
+#define __GLUPY_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+#include "mem-types.h"
+
+enum {
+        GLUPY_LOOKUP = 0,
+        GLUPY_CREATE,
+        GLUPY_OPEN,
+        GLUPY_READV,
+        GLUPY_WRITEV,
+        GLUPY_OPENDIR,
+        GLUPY_READDIR,
+        GLUPY_READDIRP,
+        GLUPY_STAT,
+        GLUPY_FSTAT,
+        GLUPY_STATFS,
+        GLUPY_SETXATTR,
+        GLUPY_GETXATTR,
+        GLUPY_FSETXATTR,
+        GLUPY_FGETXATTR,
+        GLUPY_REMOVEXATTR,
+        GLUPY_FREMOVEXATTR,
+        GLUPY_LINK,
+        GLUPY_UNLINK,
+        GLUPY_READLINK,
+        GLUPY_SYMLINK,
+        GLUPY_MKNOD,
+        GLUPY_MKDIR,
+        GLUPY_RMDIR,
+        GLUPY_N_FUNCS
+};
+
+typedef struct {
+        PyObject        *py_module;
+        PyObject        *py_xlator;
+        long            fops[GLUPY_N_FUNCS];
+        long            cbks[GLUPY_N_FUNCS];
+} glupy_private_t;
+
+enum gf_glupy_mem_types_ {
+        gf_glupy_mt_priv = gf_common_mt_end + 1,
+        gf_glupy_mt_end
+};
+
+#endif /* __GLUPY_H__ */
diff --git a/xlators/features/glupy/src/glupy.py b/xlators/features/glupy/src/glupy.py
new file mode 100644
index 000000000..a5daa77d3
--- /dev/null
+++ b/xlators/features/glupy/src/glupy.py
@@ -0,0 +1,841 @@
+import sys
+from ctypes import *
+
+dl = CDLL("",RTLD_GLOBAL)
+
+
+class call_frame_t (Structure):
+        pass
+
+class dev_t (Structure):
+        pass
+
+
+class dict_t (Structure):
+        pass
+
+
+class gf_dirent_t (Structure):
+        pass
+
+
+class iobref_t (Structure):
+	pass
+
+
+class iovec_t (Structure):
+        pass
+
+
+class list_head (Structure):
+        pass
+
+list_head._fields_ = [
+                ("next", POINTER(list_head)),
+                ("prev", POINTER(list_head))
+        ]
+
+
+class rwxperm_t (Structure):
+        _fields_ = [
+                ("read", c_uint8, 1),
+                ("write", c_uint8, 1),
+                ("execn", c_uint8, 1)
+        ]
+
+
+class statvfs_t (Structure):
+        pass
+
+
+class xlator_t (Structure):
+        pass
+
+
+class ia_prot_t (Structure):
+        _fields_ = [
+                ("suid", c_uint8, 1),
+                ("sgid", c_uint8, 1),
+                ("sticky", c_uint8, 1),
+                ("owner", rwxperm_t),
+                ("group", rwxperm_t),
+                ("other", rwxperm_t)
+        ]
+
+# For checking file type.
+(IA_INVAL, IA_IFREG, IA_IFDIR, IA_IFLNK, IA_IFBLK, IA_IFCHR, IA_IFIFO,
+ IA_IFSOCK) = xrange(8)
+
+
+class iatt_t (Structure):
+        _fields_ = [
+                ("ia_no", c_uint64),
+                ("ia_gfid", c_ubyte * 16),
+                ("ia_dev", c_uint64),
+                ("ia_type", c_uint),
+                ("ia_prot", ia_prot_t),
+                ("ia_nlink", c_uint32),
+                ("ia_uid", c_uint32),
+                ("ia_gid", c_uint32),
+                ("ia_rdev", c_uint64),
+                ("ia_size", c_uint64),
+                ("ia_blksize", c_uint32),
+                ("ia_blocks", c_uint64),
+                ("ia_atime", c_uint32 ),
+                ("ia_atime_nsec", c_uint32),
+                ("ia_mtime", c_uint32),
+                ("ia_mtime_nsec", c_uint32),
+                ("ia_ctime", c_uint32),
+                ("ia_ctime_nsec", c_uint32)
+        ]
+
+
+class mem_pool (Structure):
+        _fields_ = [
+                ("list", list_head),
+                ("hot_count", c_int),
+                ("cold_count", c_int),
+                ("lock", c_void_p),
+                ("padded_sizeof_type", c_ulong),
+                ("pool", c_void_p),
+                ("pool_end", c_void_p),
+                ("real_sizeof_type", c_int),
+                ("alloc_count", c_uint64),
+                ("pool_misses", c_uint64),
+                ("max_alloc", c_int),
+                ("curr_stdalloc", c_int),
+                ("max_stdalloc", c_int),
+                ("name", c_char_p),
+                ("global_list", list_head)
+        ]
+
+
+class U_ctx_key_inode (Union):
+        _fields_ = [
+                ("key", c_uint64),
+                ("xl_key", POINTER(xlator_t))
+        ]
+
+
+class U_ctx_value1 (Union):
+        _fields_ = [
+                ("value1", c_uint64),
+                ("ptr1", c_void_p)
+        ]
+
+
+class U_ctx_value2 (Union):
+        _fields_ = [
+                ("value2", c_uint64),
+                ("ptr2", c_void_p)
+        ]
+
+class inode_ctx (Structure):
+        _anonymous_ = ("u_key","u_value1","u_value2",)
+        _fields_ = [
+                ("u_key", U_ctx_key_inode),
+                ("u_value1", U_ctx_value1),
+                ("u_value2", U_ctx_value2)
+        ]
+
+class inode_t (Structure):
+        pass
+
+class inode_table_t (Structure):
+        _fields_ = [
+                ("lock", c_void_p),
+                ("hashsize", c_size_t),
+                ("name", c_char_p),
+                ("root", POINTER(inode_t)),
+                ("xl", POINTER(xlator_t)),
+                ("lru_limit", c_uint32),
+                ("inode_hash", POINTER(list_head)),
+                ("name_hash", POINTER(list_head)),
+                ("active", list_head),
+                ("active_size", c_uint32),
+                ("lru", list_head),
+                ("lru_size", c_uint32),
+                ("purge", list_head),
+                ("purge_size", c_uint32),
+                ("inode_pool", POINTER(mem_pool)),
+                ("dentry_pool", POINTER(mem_pool)),
+                ("fd_mem_pool", POINTER(mem_pool))
+        ]
+
+inode_t._fields_ = [
+                ("table", POINTER(inode_table_t)),
+                ("gfid", c_ubyte * 16),
+                ("lock", c_void_p),
+                ("nlookup", c_uint64),
+                ("fd_count", c_uint32),
+                ("ref", c_uint32),
+                ("ia_type", c_uint),
+                ("fd_list", list_head),
+                ("dentry_list", list_head),
+                ("hashv", list_head),
+                ("listv", list_head),
+                ("ctx", POINTER(inode_ctx))
+        ]
+
+
+
+class U_ctx_key_fd (Union):
+        _fields_ = [
+                ("key", c_uint64),
+                ("xl_key", c_void_p)
+        ]
+
+class fd_lk_ctx (Structure):
+        _fields_ = [
+                ("lk_list", list_head),
+                ("ref", c_int),
+                ("lock", c_void_p)
+        ]
+
+class fd_ctx (Structure):
+        _anonymous_ = ("u_key","u_value1")
+        _fields_ = [
+                ("u_key", U_ctx_key_fd),
+                ("u_value1", U_ctx_value1)
+        ]
+
+class fd_t (Structure):
+        _fields_ = [
+                ("pid", c_uint64),
+                ("flags", c_int32),
+                ("refcount", c_int32),
+                ("inode_list", list_head),
+                ("inode", POINTER(inode_t)),
+                ("lock", c_void_p),
+                ("ctx", POINTER(fd_ctx)),
+                ("xl_count", c_int),
+                ("lk_ctx", POINTER(fd_lk_ctx)),
+                ("anonymous", c_uint)
+        ]
+
+class loc_t (Structure):
+        _fields_ = [
+                ("path", c_char_p),
+                ("name", c_char_p),
+                ("inode", POINTER(inode_t)),
+                ("parent", POINTER(inode_t)),
+                ("gfid", c_ubyte * 16),
+                ("pargfid", c_ubyte * 16),
+        ]
+
+
+
+def _init_op (a_class, fop, cbk, wind, unwind):
+        # Decorators, used by translators. We could pass the signatures as
+        # parameters, but it's actually kind of nice to keep them around for
+        # inspection.
+        a_class.fop_type = apply(CFUNCTYPE,a_class.fop_sig)
+        a_class.cbk_type = apply(CFUNCTYPE,a_class.cbk_sig)
+        # Dispatch-function registration.
+        fop.restype = None
+        fop.argtypes = [ c_long, a_class.fop_type ]
+        # Callback-function registration.
+        cbk.restype = None
+        cbk.argtypes = [ c_long, a_class.cbk_type ]
+        # STACK_WIND function.
+        wind.restype = None
+        wind.argtypes = list(a_class.fop_sig[1:])
+        # STACK_UNWIND function.
+        unwind.restype = None
+        unwind.argtypes = list(a_class.cbk_sig[1:])
+
+class OpLookup:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
+                   POINTER(dict_t), POINTER(iatt_t))
+_init_op (OpLookup, dl.set_lookup_fop, dl.set_lookup_cbk,
+                    dl.wind_lookup,    dl.unwind_lookup)
+
+class OpCreate:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_int, c_uint, c_uint, POINTER(fd_t),
+                   POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(fd_t), POINTER(inode_t),
+                   POINTER(iatt_t), POINTER(iatt_t), POINTER(iatt_t),
+                   POINTER(dict_t))
+_init_op (OpCreate, dl.set_create_fop, dl.set_create_cbk,
+                    dl.wind_create,    dl.unwind_create)
+
+class OpOpen:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_int, POINTER(fd_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(fd_t), POINTER(dict_t))
+_init_op (OpOpen, dl.set_open_fop, dl.set_open_cbk,
+                  dl.wind_open,    dl.unwind_open)
+
+class OpReadv:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), c_size_t, c_long, c_uint32, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iovec_t), c_int, POINTER(iatt_t),
+                   POINTER(iobref_t), POINTER(dict_t))
+_init_op (OpReadv, dl.set_readv_fop, dl.set_readv_cbk,
+                   dl.wind_readv,    dl.unwind_readv)
+class OpWritev:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), POINTER(iovec_t), c_int, c_long, c_uint32,
+                   POINTER(iobref_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iatt_t), POINTER(iatt_t),
+                   POINTER(dict_t))
+_init_op (OpWritev, dl.set_writev_fop, dl.set_writev_cbk,
+                    dl.wind_writev,    dl.unwind_writev)
+
+class OpOpendir:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(fd_t) ,POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(fd_t), POINTER(dict_t))
+_init_op (OpOpendir, dl.set_opendir_fop, dl.set_opendir_cbk,
+                     dl.wind_opendir,    dl.unwind_opendir)
+
+class OpReaddir:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), c_size_t, c_long, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(gf_dirent_t), POINTER(dict_t))
+_init_op (OpReaddir, dl.set_readdir_fop, dl.set_readdir_cbk,
+                     dl.wind_readdir,    dl.unwind_readdir)
+
+class OpReaddirp:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), c_size_t, c_long, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(gf_dirent_t), POINTER(dict_t))
+_init_op (OpReaddirp, dl.set_readdirp_fop, dl.set_readdirp_cbk,
+                      dl.wind_readdirp,    dl.unwind_readdirp)
+
+class OpStat:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpStat, dl.set_stat_fop, dl.set_stat_cbk,
+                  dl.wind_stat,    dl.unwind_stat)
+
+class OpFstat:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpFstat, dl.set_fstat_fop, dl.set_fstat_cbk,
+                   dl.wind_fstat,    dl.unwind_fstat)
+
+class OpStatfs:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(statvfs_t), POINTER(dict_t))
+_init_op (OpStatfs, dl.set_statfs_fop, dl.set_statfs_cbk,
+                    dl.wind_statfs,    dl.unwind_statfs)
+
+
+class OpSetxattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(dict_t), c_int32,
+                   POINTER (dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t))
+_init_op (OpSetxattr, dl.set_setxattr_fop, dl.set_setxattr_cbk,
+                      dl.wind_setxattr,    dl.unwind_setxattr)
+
+class OpGetxattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_char_p, POINTER (dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t), POINTER(dict_t))
+_init_op (OpGetxattr, dl.set_getxattr_fop, dl.set_getxattr_cbk,
+                      dl.wind_getxattr,    dl.unwind_getxattr)
+
+class OpFsetxattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), POINTER(dict_t), c_int32,
+                   POINTER (dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t))
+_init_op (OpFsetxattr, dl.set_fsetxattr_fop, dl.set_fsetxattr_cbk,
+                       dl.wind_fsetxattr,    dl.unwind_fsetxattr)
+
+class OpFgetxattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), c_char_p, POINTER (dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t), POINTER(dict_t))
+_init_op (OpFgetxattr, dl.set_fgetxattr_fop, dl.set_fgetxattr_cbk,
+                       dl.wind_fgetxattr,    dl.unwind_fgetxattr)
+
+class OpRemovexattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_char_p, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t))
+_init_op (OpRemovexattr, dl.set_removexattr_fop, dl.set_removexattr_cbk,
+                         dl.wind_removexattr,    dl.unwind_removexattr)
+
+
+class OpFremovexattr:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(fd_t), c_char_p, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(dict_t))
+_init_op (OpFremovexattr, dl.set_fremovexattr_fop, dl.set_fremovexattr_cbk,
+                          dl.wind_fremovexattr,    dl.unwind_fremovexattr)
+
+class OpLink:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), POINTER(loc_t), POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
+                   POINTER(iatt_t), POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpLink, dl.set_link_fop, dl.set_link_cbk,
+                  dl.wind_link,    dl.unwind_link)
+
+class OpSymlink:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   c_char_p, POINTER(loc_t), c_uint, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
+                   POINTER(iatt_t), POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpSymlink, dl.set_symlink_fop, dl.set_symlink_cbk,
+                     dl.wind_symlink,    dl.unwind_symlink)
+
+class OpUnlink:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_int, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iatt_t), POINTER(iatt_t),
+                   POINTER(dict_t))
+_init_op (OpUnlink, dl.set_unlink_fop, dl.set_unlink_cbk,
+                    dl.wind_unlink,    dl.unwind_unlink)
+
+class OpReadlink:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_size_t, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, c_char_p, POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpReadlink, dl.set_readlink_fop, dl.set_readlink_cbk,
+                      dl.wind_readlink,    dl.unwind_readlink)
+
+class OpMkdir:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_uint, c_uint, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(inode_t), POINTER(iatt_t),
+                   POINTER(iatt_t), POINTER(iatt_t), POINTER(dict_t))
+_init_op (OpMkdir, dl.set_mkdir_fop, dl.set_mkdir_cbk,
+                   dl.wind_mkdir,    dl.unwind_mkdir)
+
+class OpRmdir:
+        fop_sig = (c_int, POINTER(call_frame_t), POINTER(xlator_t),
+                   POINTER(loc_t), c_int, POINTER(dict_t))
+        cbk_sig = (c_int, POINTER(call_frame_t), c_long, POINTER(xlator_t),
+                   c_int, c_int, POINTER(iatt_t), POINTER(iatt_t),
+                   POINTER(dict_t))
+_init_op (OpRmdir, dl.set_rmdir_fop, dl.set_rmdir_cbk,
+                   dl.wind_rmdir,    dl.unwind_rmdir)
+
+
+class Translator:
+        def __init__ (self, c_this):
+                # This is only here to keep references to the stubs we create,
+                # because ctypes doesn't and glupy.so can't because it doesn't
+                # get a pointer to the actual Python object. It's a dictionary
+                # instead of a list in case we ever allow changing fops/cbks
+                # after initialization and need to look them up.
+                self.stub_refs = {}
+                funcs = dir(self.__class__)
+                if "lookup_fop" in funcs:
+                        @OpLookup.fop_type
+                        def stub (frame, this, loc, xdata, s=self):
+                                return s.lookup_fop (frame, this, loc, xdata)
+                        self.stub_refs["lookup_fop"] = stub
+                        dl.set_lookup_fop(c_this,stub)
+                if "lookup_cbk" in funcs:
+                        @OpLookup.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, inode,
+                                  buf, xdata, postparent, s=self):
+                                return s.lookup_cbk(frame, cookie, this, op_ret,
+                                                    op_errno, inode, buf, xdata,
+                                                    postparent)
+                        self.stub_refs["lookup_cbk"] = stub
+                        dl.set_lookup_cbk(c_this,stub)
+                if "create_fop" in funcs:
+                        @OpCreate.fop_type
+                        def stub (frame, this, loc, flags, mode, umask, fd,
+                                  xdata, s=self):
+                                return s.create_fop (frame, this, loc, flags,
+                                                     mode, umask, fd, xdata)
+                        self.stub_refs["create_fop"] = stub
+                        dl.set_create_fop(c_this,stub)
+                if "create_cbk" in funcs:
+                        @OpCreate.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, fd,
+                                  inode, buf, preparent, postparent, xdata,
+                                  s=self):
+                                return s.create_cbk (frame, cookie, this,
+                                                     op_ret, op_errno, fd,
+                                                     inode, buf, preparent,
+                                                     postparent, xdata)
+                        self.stub_refs["create_cbk"] = stub
+                        dl.set_create_cbk(c_this,stub)
+                if "open_fop" in funcs:
+                        @OpOpen.fop_type
+                        def stub (frame, this, loc, flags, fd,
+                                  xdata, s=self):
+                                return s.open_fop (frame, this, loc, flags,
+                                                   fd, xdata)
+                        self.stub_refs["open_fop"] = stub
+                        dl.set_open_fop(c_this,stub)
+                if "open_cbk" in funcs:
+                        @OpOpen.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, fd,
+                                  xdata, s=self):
+                                return s.open_cbk (frame, cookie, this,
+                                                   op_ret, op_errno, fd,
+                                                   xdata)
+                        self.stub_refs["open_cbk"] = stub
+                        dl.set_open_cbk(c_this,stub)
+                if "readv_fop" in funcs:
+                        @OpReadv.fop_type
+                        def stub (frame, this, fd, size, offset, flags,
+                                  xdata, s=self):
+                                return s.readv_fop (frame, this, fd, size,
+                                                    offset, flags, xdata)
+                        self.stub_refs["readv_fop"] = stub
+                        dl.set_readv_fop(c_this,stub)
+                if "readv_cbk" in funcs:
+                        @OpReadv.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  vector, count, stbuf, iobref, xdata,
+                                  s=self):
+                                return s.readv_cbk (frame, cookie, this,
+                                                    op_ret, op_errno, vector,
+                                                    count, stbuf, iobref,
+                                                    xdata)
+                        self.stub_refs["readv_cbk"] = stub
+                        dl.set_readv_cbk(c_this,stub)
+                if "writev_fop" in funcs:
+                        @OpWritev.fop_type
+                        def stub (frame, this, fd, vector, count,
+                                  offset, flags, iobref, xdata, s=self):
+                                return s.writev_fop (frame, this, fd, vector,
+                                                     count, offset, flags,
+                                                     iobref, xdata)
+                        self.stub_refs["writev_fop"] = stub
+                        dl.set_writev_fop(c_this,stub)
+                if "writev_cbk" in funcs:
+                        @OpWritev.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  prebuf, postbuf, xdata, s=self):
+                                return s.writev_cbk (frame, cookie, this,
+                                                     op_ret, op_errno, prebuf,
+                                                     postbuf, xdata)
+                        self.stub_refs["writev_cbk"] = stub
+                        dl.set_writev_cbk(c_this,stub)
+                if "opendir_fop" in funcs:
+                        @OpOpendir.fop_type
+                        def stub (frame, this, loc, fd, xdata, s=self):
+                                return s.opendir_fop (frame, this, loc, fd,
+                                                      xdata)
+                        self.stub_refs["opendir_fop"] = stub
+                        dl.set_opendir_fop(c_this,stub)
+                if "opendir_cbk" in funcs:
+                        @OpOpendir.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, fd,
+                                  xdata, s=self):
+                                return s.opendir_cbk(frame, cookie, this,
+                                                     op_ret, op_errno, fd,
+                                                     xdata)
+                        self.stub_refs["opendir_cbk"] = stub
+                        dl.set_opendir_cbk(c_this,stub)
+                if "readdir_fop" in funcs:
+                        @OpReaddir.fop_type
+                        def stub (frame, this, fd, size, offset, xdata, s=self):
+                                return s.readdir_fop (frame, this, fd, size,
+                                                      offset, xdata)
+                        self.stub_refs["readdir_fop"] = stub
+                        dl.set_readdir_fop(c_this,stub)
+                if "readdir_cbk" in funcs:
+                        @OpReaddir.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  entries, xdata, s=self):
+                                return s.readdir_cbk(frame, cookie, this,
+                                                     op_ret, op_errno, entries,
+                                                     xdata)
+                        self.stub_refs["readdir_cbk"] = stub
+                        dl.set_readdir_cbk(c_this,stub)
+                if "readdirp_fop" in funcs:
+                        @OpReaddirp.fop_type
+                        def stub (frame, this, fd, size, offset, xdata, s=self):
+                                return s.readdirp_fop (frame, this, fd, size,
+                                                       offset, xdata)
+                        self.stub_refs["readdirp_fop"] = stub
+                        dl.set_readdirp_fop(c_this,stub)
+                if "readdirp_cbk" in funcs:
+                        @OpReaddirp.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  entries, xdata, s=self):
+                                return s.readdirp_cbk (frame, cookie, this,
+                                                       op_ret, op_errno,
+                                                       entries, xdata)
+                        self.stub_refs["readdirp_cbk"] = stub
+                        dl.set_readdirp_cbk(c_this,stub)
+                if "stat_fop" in funcs:
+                        @OpStat.fop_type
+                        def stub (frame, this, loc, xdata, s=self):
+                                return s.stat_fop (frame, this, loc, xdata)
+                        self.stub_refs["stat_fop"] = stub
+                        dl.set_stat_fop(c_this,stub)
+                if "stat_cbk" in funcs:
+                        @OpStat.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, buf,
+                                  xdata, s=self):
+                                return s.stat_cbk(frame, cookie, this, op_ret,
+                                                  op_errno, buf, xdata)
+                        self.stub_refs["stat_cbk"] = stub
+                        dl.set_stat_cbk(c_this,stub)
+                if "fstat_fop" in funcs:
+                        @OpFstat.fop_type
+                        def stub (frame, this, fd, xdata, s=self):
+                                return s.fstat_fop (frame, this, fd, xdata)
+                        self.stub_refs["fstat_fop"] = stub
+                        dl.set_fstat_fop(c_this,stub)
+                if "fstat_cbk" in funcs:
+                        @OpFstat.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, buf,
+                                  xdata, s=self):
+                                return s.fstat_cbk(frame, cookie, this, op_ret,
+                                                   op_errno, buf, xdata)
+                        self.stub_refs["fstat_cbk"] = stub
+                        dl.set_fstat_cbk(c_this,stub)
+                if "statfs_fop" in funcs:
+                        @OpStatfs.fop_type
+                        def stub (frame, this, loc, xdata, s=self):
+                                return s.statfs_fop (frame, this, loc, xdata)
+                        self.stub_refs["statfs_fop"] = stub
+                        dl.set_statfs_fop(c_this,stub)
+                if "statfs_cbk" in funcs:
+                        @OpStatfs.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, buf,
+                                  xdata, s=self):
+                                return s.statfs_cbk (frame, cookie, this,
+                                                     op_ret, op_errno, buf,
+                                                     xdata)
+                        self.stub_refs["statfs_cbk"] = stub
+                        dl.set_statfs_cbk(c_this,stub)
+                if "setxattr_fop" in funcs:
+                        @OpSetxattr.fop_type
+                        def stub (frame, this, loc, dictionary, flags, xdata,
+                                  s=self):
+                                return s.setxattr_fop (frame, this, loc,
+                                                       dictionary, flags,
+                                                       xdata)
+                        self.stub_refs["setxattr_fop"] = stub
+                        dl.set_setxattr_fop(c_this,stub)
+                if "setxattr_cbk" in funcs:
+                        @OpSetxattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, xdata,
+                                  s=self):
+                                return s.setxattr_cbk(frame, cookie, this,
+                                                      op_ret, op_errno, xdata)
+                        self.stub_refs["setxattr_cbk"] = stub
+                        dl.set_setxattr_cbk(c_this,stub)
+                if "getxattr_fop" in funcs:
+                        @OpGetxattr.fop_type
+                        def stub (frame, this, loc, name, xdata, s=self):
+                                return s.getxattr_fop (frame, this, loc, name,
+                                                       xdata)
+                        self.stub_refs["getxattr_fop"] = stub
+                        dl.set_getxattr_fop(c_this,stub)
+                if "getxattr_cbk" in funcs:
+                        @OpGetxattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  dictionary, xdata, s=self):
+                                return s.getxattr_cbk(frame, cookie, this,
+                                                      op_ret, op_errno,
+                                                      dictionary, xdata)
+                        self.stub_refs["getxattr_cbk"] = stub
+                        dl.set_getxattr_cbk(c_this,stub)
+                if "fsetxattr_fop" in funcs:
+                        @OpFsetxattr.fop_type
+                        def stub (frame, this, fd, dictionary, flags, xdata,
+                                  s=self):
+                                return s.fsetxattr_fop (frame, this, fd,
+                                                        dictionary, flags,
+                                                        xdata)
+                        self.stub_refs["fsetxattr_fop"] = stub
+                        dl.set_fsetxattr_fop(c_this,stub)
+                if "fsetxattr_cbk" in funcs:
+                        @OpFsetxattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, xdata,
+                                  s=self):
+                                return s.fsetxattr_cbk(frame, cookie, this,
+                                                       op_ret, op_errno, xdata)
+                        self.stub_refs["fsetxattr_cbk"] = stub
+                        dl.set_fsetxattr_cbk(c_this,stub)
+                if "fgetxattr_fop" in funcs:
+                        @OpFgetxattr.fop_type
+                        def stub (frame, this, fd, name, xdata, s=self):
+                                return s.fgetxattr_fop (frame, this, fd, name,
+                                                        xdata)
+                        self.stub_refs["fgetxattr_fop"] = stub
+                        dl.set_fgetxattr_fop(c_this,stub)
+                if "fgetxattr_cbk" in funcs:
+                        @OpFgetxattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  dictionary, xdata, s=self):
+                                return s.fgetxattr_cbk(frame, cookie, this,
+                                                       op_ret, op_errno,
+                                                       dictionary, xdata)
+                        self.stub_refs["fgetxattr_cbk"] = stub
+                        dl.set_fgetxattr_cbk(c_this,stub)
+                if "removexattr_fop" in funcs:
+                        @OpRemovexattr.fop_type
+                        def stub (frame, this, loc, name, xdata, s=self):
+                                return s.removexattr_fop (frame, this, loc,
+                                                          name, xdata)
+                        self.stub_refs["removexattr_fop"] = stub
+                        dl.set_removexattr_fop(c_this,stub)
+                if "removexattr_cbk" in funcs:
+                        @OpRemovexattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  xdata, s=self):
+                                return s.removexattr_cbk(frame, cookie, this,
+                                                         op_ret, op_errno,
+                                                         xdata)
+                        self.stub_refs["removexattr_cbk"] = stub
+                        dl.set_removexattr_cbk(c_this,stub)
+                if "fremovexattr_fop" in funcs:
+                        @OpFremovexattr.fop_type
+                        def stub (frame, this, fd, name, xdata, s=self):
+                                return s.fremovexattr_fop (frame, this, fd,
+                                                           name, xdata)
+                        self.stub_refs["fremovexattr_fop"] = stub
+                        dl.set_fremovexattr_fop(c_this,stub)
+                if "fremovexattr_cbk" in funcs:
+                        @OpFremovexattr.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  xdata, s=self):
+                                return s.fremovexattr_cbk(frame, cookie, this,
+                                                          op_ret, op_errno,
+                                                          xdata)
+                        self.stub_refs["fremovexattr_cbk"] = stub
+                        dl.set_fremovexattr_cbk(c_this,stub)
+                if "link_fop" in funcs:
+                        @OpLink.fop_type
+                        def stub (frame, this, oldloc, newloc,
+                                  xdata, s=self):
+                                return s.link_fop (frame, this, oldloc,
+                                                   newloc, xdata)
+                        self.stub_refs["link_fop"] = stub
+                        dl.set_link_fop(c_this,stub)
+                if "link_cbk" in funcs:
+                        @OpLink.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  inode, buf, preparent, postparent, xdata,
+                                  s=self):
+                                return s.link_cbk (frame, cookie, this,
+                                                   op_ret, op_errno, inode,
+                                                   buf, preparent,
+                                                   postparent, xdata)
+                        self.stub_refs["link_cbk"] = stub
+                        dl.set_link_cbk(c_this,stub)
+                if "symlink_fop" in funcs:
+                        @OpSymlink.fop_type
+                        def stub (frame, this, linkname, loc,
+                                  umask, xdata, s=self):
+                                return s.symlink_fop (frame, this, linkname,
+                                                      loc, umask, xdata)
+                        self.stub_refs["symlink_fop"] = stub
+                        dl.set_symlink_fop(c_this,stub)
+                if "symlink_cbk" in funcs:
+                        @OpSymlink.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  inode, buf, preparent, postparent, xdata,
+                                  s=self):
+                                return s.symlink_cbk (frame, cookie, this,
+                                                      op_ret, op_errno, inode,
+                                                      buf, preparent,
+                                                      postparent, xdata)
+                        self.stub_refs["symlink_cbk"] = stub
+                        dl.set_symlink_cbk(c_this,stub)
+                if "unlink_fop" in funcs:
+                        @OpUnlink.fop_type
+                        def stub (frame, this, loc, xflags,
+                                  xdata, s=self):
+                                return s.unlink_fop (frame, this, loc,
+                                                     xflags, xdata)
+                        self.stub_refs["unlink_fop"] = stub
+                        dl.set_unlink_fop(c_this,stub)
+                if "unlink_cbk" in funcs:
+                        @OpUnlink.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  preparent, postparent, xdata, s=self):
+                                return s.unlink_cbk (frame, cookie, this,
+                                                     op_ret, op_errno,
+                                                     preparent, postparent,
+                                                     xdata)
+                        self.stub_refs["unlink_cbk"] = stub
+                        dl.set_unlink_cbk(c_this,stub)
+                if "readlink_fop" in funcs:
+                        @OpReadlink.fop_type
+                        def stub (frame, this, loc, size,
+                                  xdata, s=self):
+                                return s.readlink_fop (frame, this, loc,
+                                                       size, xdata)
+                        self.stub_refs["readlink_fop"] = stub
+                        dl.set_readlink_fop(c_this,stub)
+                if "readlink_cbk" in funcs:
+                        @OpReadlink.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  path, buf, xdata, s=self):
+                                return s.readlink_cbk (frame, cookie, this,
+                                                       op_ret, op_errno,
+                                                       path, buf, xdata)
+                        self.stub_refs["readlink_cbk"] = stub
+                        dl.set_readlink_cbk(c_this,stub)
+                if "mkdir_fop" in funcs:
+                        @OpMkdir.fop_type
+                        def stub (frame, this, loc, mode, umask, xdata,
+                                  s=self):
+                                return s.mkdir_fop (frame, this, loc, mode,
+                                                    umask, xdata)
+                        self.stub_refs["mkdir_fop"] = stub
+                        dl.set_mkdir_fop(c_this,stub)
+                if "mkdir_cbk" in funcs:
+                        @OpMkdir.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno, inode,
+                                  buf, preparent, postparent, xdata, s=self):
+                                return s.mkdir_cbk (frame, cookie, this,
+                                                    op_ret, op_errno, inode,
+                                                    buf, preparent,
+                                                    postparent, xdata)
+                        self.stub_refs["mkdir_cbk"] = stub
+                        dl.set_mkdir_cbk(c_this,stub)
+                if "rmdir_fop" in funcs:
+                        @OpRmdir.fop_type
+                        def stub (frame, this, loc, xflags,
+                                  xdata, s=self):
+                                return s.rmdir_fop (frame, this, loc,
+                                                    xflags, xdata)
+                        self.stub_refs["rmdir_fop"] = stub
+                        dl.set_rmdir_fop(c_this,stub)
+                if "rmdir_cbk" in funcs:
+                        @OpRmdir.cbk_type
+                        def stub (frame, cookie, this, op_ret, op_errno,
+                                  preparent, postparent, xdata, s=self):
+                                return s.rmdir_cbk (frame, cookie, this,
+                                                    op_ret, op_errno,
+                                                    preparent, postparent,
+                                                    xdata)
+                        self.stub_refs["rmdir_cbk"] = stub
+                        dl.set_rmdir_cbk(c_this,stub)
diff --git a/xlators/features/glupy/src/setup.py.in b/xlators/features/glupy/src/setup.py.in
new file mode 100644
index 000000000..b9ee02c2b
--- /dev/null
+++ b/xlators/features/glupy/src/setup.py.in
@@ -0,0 +1,24 @@
+from distutils.core import setup
+
+DESC = """GlusterFS is a clustered file-system capable of scaling to
+several petabytes. It aggregates various storage bricks over Infiniband
+RDMA or TCP/IP interconnect into one large parallel network file system.
+GlusterFS is one of the most sophisticated file systems in terms of
+features and extensibility.  It borrows a powerful concept called
+Translators from GNU Hurd kernel. Much of the code in GlusterFS is in
+user space and easily manageable.
+
+This package contains Glupy, the Python translator interface for GlusterFS."""
+
+setup(
+    name='glusterfs-glupy',
+    version='@PACKAGE_VERSION@',
+    description='Glupy is the Python translator interface for GlusterFS',
+    long_description=DESC,
+    author='Gluster Community',
+    author_email='gluster-devel@gluster.org',
+    license='LGPLv3',
+    url='http://gluster.org/',
+    package_dir={'gluster':''},
+    packages=['gluster']
+)
diff --git a/xlators/features/index/Makefile.am b/xlators/features/index/Makefile.am
new file mode 100644
index 000000000..a985f42a8
--- /dev/null
+++ b/xlators/features/index/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/features/index/src/Makefile.am b/xlators/features/index/src/Makefile.am
new file mode 100644
index 000000000..73bb8972e
--- /dev/null
+++ b/xlators/features/index/src/Makefile.am
@@ -0,0 +1,17 @@
+xlator_LTLIBRARIES = index.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+index_la_LDFLAGS = -module -avoid-version
+
+index_la_SOURCES = index.c
+index_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = index.h index-mem-types.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) \
+	-I$(top_srcdir)/libglusterfs/src -I$(top_srcdir)/rpc/xdr/src \
+	-I$(top_srcdir)/rpc/rpc-lib/src
+
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/index/src/index-mem-types.h b/xlators/features/index/src/index-mem-types.h
new file mode 100644
index 000000000..553d492df
--- /dev/null
+++ b/xlators/features/index/src/index-mem-types.h
@@ -0,0 +1,22 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __QUIESCE_MEM_TYPES_H__
+#define __QUIESCE_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_index_mem_types_ {
+        gf_index_mt_priv_t = gf_common_mt_end + 1,
+        gf_index_inode_ctx_t = gf_common_mt_end + 2,
+        gf_index_fd_ctx_t = gf_common_mt_end + 3,
+        gf_index_mt_end
+};
+#endif
diff --git a/xlators/features/index/src/index.c b/xlators/features/index/src/index.c
new file mode 100644
index 000000000..4b2eb2e29
--- /dev/null
+++ b/xlators/features/index/src/index.c
@@ -0,0 +1,1275 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "index.h"
+#include "options.h"
+#include "glusterfs3-xdr.h"
+#include "syscall.h"
+
+#define XATTROP_SUBDIR "xattrop"
+
+call_stub_t *
+__index_dequeue (struct list_head *callstubs)
+{
+        call_stub_t *stub = NULL;
+
+        if (!list_empty (callstubs)) {
+                stub = list_entry (callstubs->next, call_stub_t, list);
+                list_del_init (&stub->list);
+        }
+
+        return stub;
+}
+
+inline static void
+__index_enqueue (struct list_head *callstubs, call_stub_t *stub)
+{
+        list_add_tail (&stub->list, callstubs);
+}
+
+static void
+worker_enqueue (xlator_t *this, call_stub_t *stub)
+{
+        index_priv_t    *priv = NULL;
+
+        priv = this->private;
+        pthread_mutex_lock (&priv->mutex);
+        {
+                __index_enqueue (&priv->callstubs, stub);
+                pthread_cond_signal (&priv->cond);
+        }
+        pthread_mutex_unlock (&priv->mutex);
+}
+
+void *
+index_worker (void *data)
+{
+        index_priv_t     *priv = NULL;
+        xlator_t         *this = NULL;
+        call_stub_t      *stub = NULL;
+        int               ret = 0;
+
+        THIS = data;
+        this = data;
+        priv = this->private;
+
+        for (;;) {
+                pthread_mutex_lock (&priv->mutex);
+                {
+                        while (list_empty (&priv->callstubs)) {
+                                ret = pthread_cond_wait (&priv->cond,
+                                                         &priv->mutex);
+                        }
+
+                        stub = __index_dequeue (&priv->callstubs);
+                }
+                pthread_mutex_unlock (&priv->mutex);
+
+                if (stub) /* guard against spurious wakeups */
+                        call_resume (stub);
+        }
+
+        return NULL;
+}
+int
+__index_inode_ctx_get (inode_t *inode, xlator_t *this, index_inode_ctx_t **ctx)
+{
+        int               ret = 0;
+        index_inode_ctx_t *ictx = NULL;
+        uint64_t          tmpctx = 0;
+
+        ret = __inode_ctx_get (inode, this, &tmpctx);
+        if (!ret) {
+                ictx = (index_inode_ctx_t*) (long) tmpctx;
+                goto out;
+        }
+        ictx = GF_CALLOC (1, sizeof (*ictx), gf_index_inode_ctx_t);
+        if (!ictx) {
+                ret = -1;
+                goto out;
+        }
+
+        INIT_LIST_HEAD (&ictx->callstubs);
+        ret = __inode_ctx_put (inode, this, (uint64_t)ictx);
+        if (ret) {
+                GF_FREE (ictx);
+                ictx = NULL;
+                goto out;
+        }
+out:
+        if (ictx)
+                *ctx = ictx;
+        return ret;
+}
+
+int
+index_inode_ctx_get (inode_t *inode, xlator_t *this, index_inode_ctx_t **ctx)
+{
+        int               ret = 0;
+
+        LOCK (&inode->lock);
+        {
+                ret = __index_inode_ctx_get (inode, this, ctx);
+        }
+        UNLOCK (&inode->lock);
+
+        return ret;
+}
+
+static void
+make_index_dir_path (char *base, const char *subdir,
+                     char *index_dir, size_t len)
+{
+        snprintf (index_dir, len, "%s/%s", base, subdir);
+}
+
+int
+index_dir_create (xlator_t *this, const char *subdir)
+{
+        int          ret = 0;
+        struct       stat st = {0};
+        char         fullpath[PATH_MAX] = {0};
+        char         path[PATH_MAX] = {0};
+        char         *dir = NULL;
+        index_priv_t *priv = NULL;
+        size_t       len = 0;
+        size_t       pathlen = 0;
+
+        priv = this->private;
+        make_index_dir_path (priv->index_basepath, subdir, fullpath,
+                             sizeof (fullpath));
+        ret = stat (fullpath, &st);
+        if (!ret) {
+                if (!S_ISDIR (st.st_mode))
+                        ret = -2;
+                goto out;
+        }
+
+        pathlen = strlen (fullpath);
+        if ((pathlen > 1) && fullpath[pathlen - 1] == '/')
+                fullpath[pathlen - 1] = '\0';
+        dir = strchr (fullpath, '/');
+        while (dir) {
+                dir = strchr (dir + 1, '/');
+                if (dir)
+                        len = pathlen - strlen (dir);
+                else
+                        len = pathlen;
+                strncpy (path, fullpath, len);
+                path[len] = '\0';
+                ret = mkdir (path, 0600);
+                if (ret && (errno != EEXIST))
+                        goto out;
+        }
+        ret = 0;
+out:
+        if (ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR, "%s/%s: Failed to "
+                        "create (%s)", priv->index_basepath, subdir,
+                        strerror (errno));
+        } else if (ret == -2) {
+                gf_log (this->name, GF_LOG_ERROR, "%s/%s: Failed to create, "
+                        "path exists, not a directory ", priv->index_basepath,
+                        subdir);
+        }
+        return ret;
+}
+
+void
+index_get_index (index_priv_t *priv, uuid_t index)
+{
+        LOCK (&priv->lock);
+        {
+                uuid_copy (index, priv->index);
+        }
+        UNLOCK (&priv->lock);
+}
+
+void
+index_generate_index (index_priv_t *priv, uuid_t index)
+{
+        LOCK (&priv->lock);
+        {
+                //To prevent duplicate generates.
+                //This method fails if number of contending threads is greater
+                //than MAX_LINK count of the fs
+                if (!uuid_compare (priv->index, index))
+                        uuid_generate (priv->index);
+                uuid_copy (index, priv->index);
+        }
+        UNLOCK (&priv->lock);
+}
+
+static void
+make_index_path (char *base, const char *subdir, uuid_t index,
+                 char *index_path, size_t len)
+{
+        make_index_dir_path (base, subdir, index_path, len);
+        snprintf (index_path + strlen (index_path), len - strlen (index_path),
+                  "/%s-%s", subdir, uuid_utoa (index));
+}
+
+static void
+make_gfid_path (char *base, const char *subdir, uuid_t gfid,
+                char *gfid_path, size_t len)
+{
+        make_index_dir_path (base, subdir, gfid_path, len);
+        snprintf (gfid_path + strlen (gfid_path), len - strlen (gfid_path),
+                  "/%s", uuid_utoa (gfid));
+}
+
+static void
+make_file_path (char *base, const char *subdir, const char *filename,
+                char *file_path, size_t len)
+{
+        make_index_dir_path (base, subdir, file_path, len);
+        snprintf (file_path + strlen (file_path), len - strlen (file_path),
+                  "/%s", filename);
+}
+
+static void
+check_delete_stale_index_file (xlator_t *this, char *filename)
+{
+        int             ret = 0;
+        struct stat     st = {0};
+        char            filepath[PATH_MAX] = {0};
+        index_priv_t    *priv = NULL;
+
+        priv = this->private;
+        make_file_path (priv->index_basepath, XATTROP_SUBDIR,
+                        filename, filepath, sizeof (filepath));
+        ret = stat (filepath, &st);
+        if (!ret && st.st_nlink == 1)
+                unlink (filepath);
+}
+
+static int
+index_fill_readdir (fd_t *fd, DIR *dir, off_t off,
+                    size_t size, gf_dirent_t *entries)
+{
+        off_t     in_case = -1;
+        size_t    filled = 0;
+        int       count = 0;
+        char      entrybuf[sizeof(struct dirent) + 256 + 8];
+        struct dirent  *entry          = NULL;
+        int32_t              this_size      = -1;
+        gf_dirent_t          *this_entry     = NULL;
+        xlator_t             *this = NULL;
+
+        this = THIS;
+        if (!off) {
+                rewinddir (dir);
+        } else {
+                seekdir (dir, off);
+        }
+
+        while (filled <= size) {
+                in_case = telldir (dir);
+
+                if (in_case == -1) {
+                        gf_log (THIS->name, GF_LOG_ERROR,
+                                "telldir failed on dir=%p: %s",
+                                dir, strerror (errno));
+                        goto out;
+                }
+
+                errno = 0;
+                entry = NULL;
+                readdir_r (dir, (struct dirent *)entrybuf, &entry);
+
+                if (!entry) {
+                        if (errno == EBADF) {
+                                gf_log (THIS->name, GF_LOG_WARNING,
+                                        "readdir failed on dir=%p: %s",
+                                        dir, strerror (errno));
+                                goto out;
+                        }
+                        break;
+                }
+
+                if (!strncmp (entry->d_name, XATTROP_SUBDIR"-",
+                              strlen (XATTROP_SUBDIR"-"))) {
+                        check_delete_stale_index_file (this, entry->d_name);
+                        continue;
+                }
+
+                this_size = max (sizeof (gf_dirent_t),
+                                 sizeof (gfs3_dirplist))
+                        + strlen (entry->d_name) + 1;
+
+                if (this_size + filled > size) {
+                        seekdir (dir, in_case);
+                        break;
+                }
+
+                this_entry = gf_dirent_for_name (entry->d_name);
+
+                if (!this_entry) {
+                        gf_log (THIS->name, GF_LOG_ERROR,
+                                "could not create gf_dirent for entry %s: (%s)",
+                                entry->d_name, strerror (errno));
+                        goto out;
+                }
+                this_entry->d_off = telldir (dir);
+                this_entry->d_ino = entry->d_ino;
+
+                list_add_tail (&this_entry->list, &entries->list);
+
+                filled += this_size;
+                count ++;
+        }
+
+        if ((!readdir (dir) && (errno == 0)))
+                /* Indicate EOF */
+                errno = ENOENT;
+out:
+        return count;
+}
+
+int
+index_add (xlator_t *this, uuid_t gfid, const char *subdir)
+{
+        int32_t           op_errno = 0;
+        char              gfid_path[PATH_MAX] = {0};
+        char              index_path[PATH_MAX] = {0};
+        int               ret = 0;
+        uuid_t            index = {0};
+        index_priv_t      *priv = NULL;
+        struct stat       st = {0};
+        int               fd = 0;
+
+        priv = this->private;
+        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name, !uuid_is_null (gfid),
+                                       out, op_errno, EINVAL);
+
+        make_gfid_path (priv->index_basepath, subdir, gfid,
+                        gfid_path, sizeof (gfid_path));
+
+        ret = stat (gfid_path, &st);
+        if (!ret)
+                goto out;
+        index_get_index (priv, index);
+        make_index_path (priv->index_basepath, subdir,
+                         index, index_path, sizeof (index_path));
+        ret = sys_link (index_path, gfid_path);
+        if (!ret || (errno == EEXIST))  {
+                ret = 0;
+                goto out;
+        }
+
+        op_errno = errno;
+        if (op_errno == ENOENT) {
+                ret = index_dir_create (this, subdir);
+                if (ret)
+                        goto out;
+        } else if (op_errno == EMLINK) {
+                index_generate_index (priv, index);
+                make_index_path (priv->index_basepath, subdir,
+                                 index, index_path, sizeof (index_path));
+        } else {
+                goto out;
+        }
+
+        fd = creat (index_path, 0);
+        if ((fd < 0) && (errno != EEXIST)) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR, "%s: Not able to "
+                        "create index (%s)", uuid_utoa (gfid),
+                        strerror (errno));
+                goto out;
+        }
+
+        if (fd >= 0)
+                close (fd);
+
+        ret = sys_link (index_path, gfid_path);
+        if (ret && (errno != EEXIST)) {
+                gf_log (this->name, GF_LOG_ERROR, "%s: Not able to "
+                        "add to index (%s)", uuid_utoa (gfid),
+                        strerror (errno));
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int
+index_del (xlator_t *this, uuid_t gfid, const char *subdir)
+{
+        int32_t      op_errno __attribute__((unused)) = 0;
+        index_priv_t *priv = NULL;
+        int          ret = 0;
+        char         gfid_path[PATH_MAX] = {0};
+
+        priv = this->private;
+        GF_ASSERT_AND_GOTO_WITH_ERROR (this->name, !uuid_is_null (gfid),
+                                       out, op_errno, EINVAL);
+        make_gfid_path (priv->index_basepath, subdir, gfid,
+                        gfid_path, sizeof (gfid_path));
+        ret = unlink (gfid_path);
+        if (ret && (errno != ENOENT)) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "%s: failed to delete from index (%s)",
+                        gfid_path, strerror (errno));
+                ret = -errno;
+                goto out;
+        }
+        ret = 0;
+out:
+        return ret;
+}
+
+static int
+_check_key_is_zero_filled (dict_t *d, char *k, data_t *v,
+                           void *tmp)
+{
+        if (mem_0filled ((const char*)v->data, v->len)) {
+                /* -1 means, no more iterations, treat as 'break' */
+                return -1;
+        }
+        return 0;
+}
+
+void
+_index_action (xlator_t *this, inode_t *inode, gf_boolean_t zero_xattr)
+{
+        int               ret  = 0;
+        index_inode_ctx_t *ctx = NULL;
+
+        ret = index_inode_ctx_get (inode, this, &ctx);
+        if (ret) {
+                gf_log (this->name, GF_LOG_ERROR, "Not able to %s %s -> index",
+                        zero_xattr?"del":"add", uuid_utoa (inode->gfid));
+                goto out;
+        }
+        if (zero_xattr) {
+                if (ctx->state == NOTIN)
+                        goto out;
+                ret = index_del (this, inode->gfid, XATTROP_SUBDIR);
+                if (!ret)
+                        ctx->state = NOTIN;
+        } else {
+                if (ctx->state == IN)
+                        goto out;
+                ret = index_add (this, inode->gfid, XATTROP_SUBDIR);
+                if (!ret)
+                        ctx->state = IN;
+        }
+out:
+        return;
+}
+
+void
+_xattrop_index_action (xlator_t *this, inode_t *inode,  dict_t *xattr)
+{
+        gf_boolean_t      zero_xattr = _gf_true;
+        int               ret = 0;
+
+        ret = dict_foreach (xattr, _check_key_is_zero_filled, NULL);
+        if (ret == -1)
+                zero_xattr = _gf_false;
+        _index_action (this, inode, zero_xattr);
+        return;
+}
+
+void
+fop_xattrop_index_action (xlator_t *this, inode_t *inode, dict_t *xattr)
+{
+        _xattrop_index_action (this, inode, xattr);
+}
+
+void
+fop_fxattrop_index_action (xlator_t *this, inode_t *inode, dict_t *xattr)
+{
+        _xattrop_index_action (this, inode, xattr);
+}
+
+static inline gf_boolean_t
+index_xattrop_track (loc_t *loc, gf_xattrop_flags_t flags, dict_t *dict)
+{
+        return (flags == GF_XATTROP_ADD_ARRAY);
+}
+
+static inline gf_boolean_t
+index_fxattrop_track (fd_t *fd, gf_xattrop_flags_t flags, dict_t *dict)
+{
+        return (flags == GF_XATTROP_ADD_ARRAY);
+}
+
+int
+__index_fd_ctx_get (fd_t *fd, xlator_t *this, index_fd_ctx_t **ctx)
+{
+        int               ret = 0;
+        index_fd_ctx_t    *fctx = NULL;
+        uint64_t          tmpctx = 0;
+        char              index_dir[PATH_MAX] = {0};
+        index_priv_t      *priv = NULL;
+
+        priv = this->private;
+        if (uuid_compare (fd->inode->gfid, priv->xattrop_vgfid)) {
+                ret = -EINVAL;
+                goto out;
+        }
+
+        ret = __fd_ctx_get (fd, this, &tmpctx);
+        if (!ret) {
+                fctx = (index_fd_ctx_t*) (long) tmpctx;
+                goto out;
+        }
+
+        fctx = GF_CALLOC (1, sizeof (*fctx), gf_index_fd_ctx_t);
+        if (!fctx) {
+                ret = -ENOMEM;
+                goto out;
+        }
+
+        make_index_dir_path (priv->index_basepath, XATTROP_SUBDIR,
+                             index_dir, sizeof (index_dir));
+        fctx->dir = opendir (index_dir);
+        if (!fctx->dir) {
+                ret = -errno;
+                GF_FREE (fctx);
+                fctx = NULL;
+                goto out;
+        }
+
+        ret = __fd_ctx_set (fd, this, (uint64_t)(long)fctx);
+        if (ret) {
+                GF_FREE (fctx);
+                fctx = NULL;
+                ret = -EINVAL;
+                goto out;
+        }
+out:
+        if (fctx)
+                *ctx = fctx;
+        return ret;
+}
+
+int
+index_fd_ctx_get (fd_t *fd, xlator_t *this, index_fd_ctx_t **ctx)
+{
+        int     ret = 0;
+        LOCK (&fd->lock);
+        {
+                ret = __index_fd_ctx_get (fd, this, ctx);
+        }
+        UNLOCK (&fd->lock);
+        return ret;
+}
+
+//new - Not NULL means start a fop
+//new - NULL means done processing the fop
+void
+index_queue_process (xlator_t *this, inode_t *inode, call_stub_t *new)
+{
+        call_stub_t       *stub = NULL;
+        index_inode_ctx_t *ctx = NULL;
+        int               ret = 0;
+        call_frame_t      *frame = NULL;
+
+        LOCK (&inode->lock);
+        {
+                ret = __index_inode_ctx_get (inode, this, &ctx);
+                if (ret)
+                        goto unlock;
+
+                if (new) {
+                        __index_enqueue (&ctx->callstubs, new);
+                        new = NULL;
+                } else {
+                        ctx->processing = _gf_false;
+                }
+
+                if (!ctx->processing) {
+                        stub = __index_dequeue (&ctx->callstubs);
+                        if (stub)
+                                ctx->processing = _gf_true;
+                        else
+                                ctx->processing = _gf_false;
+                }
+        }
+unlock:
+        UNLOCK (&inode->lock);
+
+        if (ret && new) {
+                frame = new->frame;
+                if (new->fop == GF_FOP_XATTROP) {
+                        INDEX_STACK_UNWIND (xattrop, frame, -1, ENOMEM,
+                                            NULL, NULL);
+                } else if (new->fop == GF_FOP_FXATTROP) {
+                        INDEX_STACK_UNWIND (fxattrop, frame, -1, ENOMEM,
+                                            NULL, NULL);
+                }
+                call_stub_destroy (new);
+        } else if (stub) {
+                call_resume (stub);
+        }
+        return;
+}
+
+int32_t
+index_xattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                   int32_t op_ret, int32_t op_errno, dict_t *xattr, dict_t *xdata)
+{
+        inode_t *inode = NULL;
+
+        inode = inode_ref (frame->local);
+        if (op_ret < 0)
+                goto out;
+        fop_xattrop_index_action (this, frame->local, xattr);
+out:
+        INDEX_STACK_UNWIND (xattrop, frame, op_ret, op_errno, xattr, xdata);
+        index_queue_process (this, inode, NULL);
+        inode_unref (inode);
+
+        return 0;
+}
+
+int32_t
+index_fxattrop_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xattr,
+                    dict_t *xdata)
+{
+        inode_t *inode = NULL;
+
+        inode = inode_ref (frame->local);
+        if (op_ret < 0)
+                goto out;
+
+        fop_fxattrop_index_action (this, frame->local, xattr);
+out:
+        INDEX_STACK_UNWIND (fxattrop, frame, op_ret, op_errno, xattr, xdata);
+        index_queue_process (this, inode, NULL);
+        inode_unref (inode);
+
+        return 0;
+}
+
+int
+index_xattrop_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                       gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+        //In wind phase bring the gfid into index. This way if the brick crashes
+        //just after posix performs xattrop before _cbk reaches index xlator
+        //we will still have the gfid in index.
+        _index_action (this, frame->local, _gf_false);
+
+        STACK_WIND (frame, index_xattrop_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->xattrop, loc, optype, xattr,
+                    xdata);
+        return 0;
+}
+
+int
+index_fxattrop_wrapper (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                        gf_xattrop_flags_t optype, dict_t *xattr, dict_t *xdata)
+{
+        //In wind phase bring the gfid into index. This way if the brick crashes
+        //just after posix performs xattrop before _cbk reaches index xlator
+        //we will still have the gfid in index.
+        _index_action (this, frame->local, _gf_false);
+        STACK_WIND (frame, index_fxattrop_cbk, FIRST_CHILD (this),
+                    FIRST_CHILD (this)->fops->fxattrop, fd, optype, xattr,
+                    xdata);
+        return 0;
+}
+
+int32_t
+index_xattrop (call_frame_t *frame, xlator_t *this, loc_t *loc,
+	       gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+        call_stub_t     *stub = NULL;
+
+        if (!index_xattrop_track (loc, flags, dict))
+                goto out;
+
+        frame->local = inode_ref (loc->inode);
+        stub = fop_xattrop_stub (frame, index_xattrop_wrapper,
+                                 loc, flags, dict, xdata);
+        if (!stub) {
+                INDEX_STACK_UNWIND (xattrop, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+
+        index_queue_process (this, loc->inode, stub);
+        return 0;
+out:
+        STACK_WIND (frame, default_xattrop_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->xattrop, loc, flags, dict, xdata);
+        return 0;
+}
+
+int32_t
+index_fxattrop (call_frame_t *frame, xlator_t *this, fd_t *fd,
+		gf_xattrop_flags_t flags, dict_t *dict, dict_t *xdata)
+{
+        call_stub_t    *stub = NULL;
+
+        if (!index_fxattrop_track (fd, flags, dict))
+                goto out;
+
+        frame->local = inode_ref (fd->inode);
+        stub = fop_fxattrop_stub (frame, index_fxattrop_wrapper,
+                                  fd, flags, dict, xdata);
+        if (!stub) {
+                INDEX_STACK_UNWIND (fxattrop, frame, -1, ENOMEM, NULL, xdata);
+                return 0;
+        }
+
+        index_queue_process (this, fd->inode, stub);
+        return 0;
+out:
+        STACK_WIND (frame, default_fxattrop_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fxattrop, fd, flags, dict, xdata);
+        return 0;
+}
+
+uint64_t
+index_entry_count (xlator_t *this, char *subdir)
+{
+	index_priv_t *priv = NULL;
+	char index_dir[PATH_MAX];
+	DIR *dirp = NULL;
+	uint64_t count = 0;
+	struct dirent buf;
+	struct dirent *entry = NULL;
+
+	priv = this->private;
+
+	make_index_dir_path (priv->index_basepath, subdir,
+			     index_dir, sizeof (index_dir));
+
+	dirp = opendir (index_dir);
+	if (!dirp)
+		return 0;
+
+	while (readdir_r (dirp, &buf, &entry) == 0) {
+		if (!entry)
+			break;
+		if (!strcmp (entry->d_name, ".") ||
+		    !strcmp (entry->d_name, ".."))
+			continue;
+                if (!strncmp (entry->d_name, subdir, strlen (subdir)))
+			continue;
+		count++;
+	}
+	closedir (dirp);
+
+	return count;
+}
+
+
+int32_t
+index_getxattr_wrapper (call_frame_t *frame, xlator_t *this,
+                        loc_t *loc, const char *name, dict_t *xdata)
+{
+        index_priv_t    *priv = NULL;
+        dict_t          *xattr = NULL;
+        int             ret = 0;
+	uint64_t        count = 0;
+
+        priv = this->private;
+
+        xattr = dict_new ();
+        if (!xattr) {
+                ret = -ENOMEM;
+                goto done;
+        }
+
+	if (strcmp (name, GF_XATTROP_INDEX_GFID) == 0) {
+		ret = dict_set_static_bin (xattr, (char*)name, priv->xattrop_vgfid,
+					   sizeof (priv->xattrop_vgfid));
+		if (ret) {
+			ret = -ENOMEM;
+			gf_log (this->name, GF_LOG_ERROR, "xattrop index "
+				"gfid set failed");
+			goto done;
+		}
+	} else if (strcmp (name, GF_XATTROP_INDEX_COUNT) == 0) {
+		count = index_entry_count (this, XATTROP_SUBDIR);
+
+		ret = dict_set_uint64 (xattr, (char *)name, count);
+		if (ret) {
+			ret = -ENOMEM;
+			gf_log (this->name, GF_LOG_ERROR, "xattrop index "
+				"count set failed");
+			goto done;
+		}
+	}
+done:
+        if (ret)
+                STACK_UNWIND_STRICT (getxattr, frame, -1, -ret, xattr, xdata);
+        else
+                STACK_UNWIND_STRICT (getxattr, frame, 0, 0, xattr, xdata);
+
+        if (xattr)
+                dict_unref (xattr);
+
+        return 0;
+}
+
+int32_t
+index_lookup_wrapper (call_frame_t *frame, xlator_t *this,
+                      loc_t *loc, dict_t *xattr_req)
+{
+        index_priv_t    *priv = NULL;
+        struct stat     lstatbuf = {0};
+        int             ret = 0;
+        int32_t         op_errno = EINVAL;
+        int32_t         op_ret = -1;
+        char            path[PATH_MAX] = {0};
+        struct iatt     stbuf        = {0, };
+        struct iatt     postparent = {0,};
+        dict_t          *xattr = NULL;
+        gf_boolean_t    is_dir = _gf_false;
+
+        priv = this->private;
+
+        VALIDATE_OR_GOTO (loc, done);
+        if (!uuid_compare (loc->gfid, priv->xattrop_vgfid)) {
+                make_index_dir_path (priv->index_basepath, XATTROP_SUBDIR,
+                                     path, sizeof (path));
+                is_dir = _gf_true;
+        } else if (!uuid_compare (loc->pargfid, priv->xattrop_vgfid)) {
+                make_file_path (priv->index_basepath, XATTROP_SUBDIR,
+                                loc->name, path, sizeof (path));
+        }
+
+        ret = lstat (path, &lstatbuf);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG, "Stat failed on index dir "
+                        "(%s)", strerror (errno));
+                op_errno = errno;
+                goto done;
+        } else if (!S_ISDIR (lstatbuf.st_mode) && is_dir) {
+                gf_log (this->name, GF_LOG_DEBUG, "Stat failed on index dir, "
+                        "not a directory");
+                op_errno = ENOENT;
+                goto done;
+        }
+        xattr = dict_new ();
+        if (!xattr) {
+                op_errno = ENOMEM;
+                goto done;
+        }
+
+        iatt_from_stat (&stbuf, &lstatbuf);
+        if (is_dir)
+                uuid_copy (stbuf.ia_gfid, priv->xattrop_vgfid);
+        else
+                uuid_generate (stbuf.ia_gfid);
+        stbuf.ia_ino = -1;
+        op_ret = 0;
+done:
+        STACK_UNWIND_STRICT (lookup, frame, op_ret, op_errno,
+                             loc->inode, &stbuf, xattr, &postparent);
+        if (xattr)
+                dict_unref (xattr);
+        return 0;
+}
+
+int32_t
+index_readdir_wrapper (call_frame_t *frame, xlator_t *this,
+                       fd_t *fd, size_t size, off_t off, dict_t *xdata)
+{
+        index_fd_ctx_t       *fctx           = NULL;
+        DIR                  *dir            = NULL;
+        int                   ret            = -1;
+        int32_t               op_ret         = -1;
+        int32_t               op_errno       = 0;
+        int                   count          = 0;
+        gf_dirent_t           entries;
+
+        INIT_LIST_HEAD (&entries.list);
+
+        ret = index_fd_ctx_get (fd, this, &fctx);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "pfd is NULL, fd=%p", fd);
+                op_errno = -ret;
+                goto done;
+        }
+
+        dir = fctx->dir;
+
+        if (!dir) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dir is NULL for fd=%p", fd);
+                op_errno = EINVAL;
+                goto done;
+        }
+
+        count = index_fill_readdir (fd, dir, off, size, &entries);
+
+        /* pick ENOENT to indicate EOF */
+        op_errno = errno;
+        op_ret = count;
+done:
+        STACK_UNWIND_STRICT (readdir, frame, op_ret, op_errno, &entries, xdata);
+        gf_dirent_free (&entries);
+        return 0;
+}
+
+int
+index_unlink_wrapper (call_frame_t *frame, xlator_t *this, loc_t *loc, int flag,
+                      dict_t *xdata)
+{
+        index_priv_t    *priv = NULL;
+        int32_t         op_ret = 0;
+        int32_t         op_errno = 0;
+        int             ret = 0;
+        struct  iatt    preparent = {0};
+        struct  iatt    postparent = {0};
+        char            index_dir[PATH_MAX] = {0};
+        struct  stat    lstatbuf = {0};
+        uuid_t          gfid = {0};
+
+        priv = this->private;
+        make_index_dir_path (priv->index_basepath, XATTROP_SUBDIR,
+                             index_dir, sizeof (index_dir));
+        ret = lstat (index_dir, &lstatbuf);
+        if (ret < 0) {
+                op_ret = -1;
+                op_errno = errno;
+                goto done;
+        }
+
+        iatt_from_stat (&preparent, &lstatbuf);
+        uuid_copy (preparent.ia_gfid, priv->xattrop_vgfid);
+        preparent.ia_ino = -1;
+        uuid_parse (loc->name, gfid);
+        ret = index_del (this, gfid, XATTROP_SUBDIR);
+        if (ret < 0) {
+                op_ret = -1;
+                op_errno = -ret;
+                goto done;
+        }
+        memset (&lstatbuf, 0, sizeof (lstatbuf));
+        ret = lstat (index_dir, &lstatbuf);
+        if (ret < 0) {
+                op_ret = -1;
+                op_errno = errno;
+                goto done;
+        }
+        iatt_from_stat (&postparent, &lstatbuf);
+        uuid_copy (postparent.ia_gfid, priv->xattrop_vgfid);
+        postparent.ia_ino = -1;
+done:
+        INDEX_STACK_UNWIND (unlink, frame, op_ret, op_errno, &preparent,
+                            &postparent, xdata);
+        return 0;
+}
+
+int32_t
+index_getxattr (call_frame_t *frame, xlator_t *this,
+                loc_t *loc, const char *name, dict_t *xdata)
+{
+        call_stub_t     *stub = NULL;
+        index_priv_t    *priv = NULL;
+
+        priv = this->private;
+
+        if (!name || (strcmp (GF_XATTROP_INDEX_GFID, name) &&
+		      strcmp (GF_XATTROP_INDEX_COUNT, name)))
+                goto out;
+
+        stub = fop_getxattr_stub (frame, index_getxattr_wrapper, loc, name,
+                                  xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (getxattr, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+        worker_enqueue (this, stub);
+        return 0;
+out:
+        STACK_WIND (frame, default_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+        return 0;
+}
+
+int32_t
+index_lookup (call_frame_t *frame, xlator_t *this,
+              loc_t *loc, dict_t *xattr_req)
+{
+        call_stub_t     *stub = NULL;
+        index_priv_t    *priv = NULL;
+
+        priv = this->private;
+
+        if (uuid_compare (loc->gfid, priv->xattrop_vgfid) &&
+            uuid_compare (loc->pargfid, priv->xattrop_vgfid))
+                goto normal;
+
+        stub = fop_lookup_stub (frame, index_lookup_wrapper, loc, xattr_req);
+        if (!stub) {
+                STACK_UNWIND_STRICT (lookup, frame, -1, ENOMEM, loc->inode,
+                                     NULL, NULL, NULL);
+                return 0;
+        }
+        worker_enqueue (this, stub);
+        return 0;
+normal:
+        STACK_WIND (frame, default_lookup_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, loc, xattr_req);
+
+        return 0;
+}
+
+int32_t
+index_readdir (call_frame_t *frame, xlator_t *this,
+               fd_t *fd, size_t size, off_t off, dict_t *xdata)
+{
+        call_stub_t     *stub = NULL;
+        index_priv_t    *priv = NULL;
+
+        priv = this->private;
+        if (uuid_compare (fd->inode->gfid, priv->xattrop_vgfid))
+                goto out;
+        stub = fop_readdir_stub (frame, index_readdir_wrapper, fd, size, off,
+                                 xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (readdir, frame, -1, ENOMEM, NULL, NULL);
+                return 0;
+        }
+        worker_enqueue (this, stub);
+        return 0;
+out:
+        STACK_WIND (frame, default_readdir_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readdir, fd, size, off, xdata);
+        return 0;
+}
+
+int
+index_unlink (call_frame_t *frame, xlator_t *this, loc_t *loc, int xflag,
+              dict_t *xdata)
+{
+        call_stub_t     *stub = NULL;
+        index_priv_t    *priv = NULL;
+
+        priv = this->private;
+        if (uuid_compare (loc->pargfid, priv->xattrop_vgfid))
+                goto out;
+
+        stub = fop_unlink_stub (frame, index_unlink_wrapper, loc, xflag, xdata);
+        if (!stub) {
+                STACK_UNWIND_STRICT (unlink, frame, -1, ENOMEM, NULL, NULL,
+                                     NULL);
+                return 0;
+        }
+        worker_enqueue (this, stub);
+        return 0;
+out:
+        STACK_WIND (frame, default_unlink_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->unlink, loc, xflag, xdata);
+        return 0;
+}
+
+int32_t
+mem_acct_init (xlator_t *this)
+{
+        int     ret = -1;
+
+        ret = xlator_mem_acct_init (this, gf_index_mt_end + 1);
+
+        return ret;
+}
+
+int
+init (xlator_t *this)
+{
+        int ret = -1;
+        index_priv_t *priv = NULL;
+        pthread_t thread;
+        pthread_attr_t  w_attr;
+        gf_boolean_t    mutex_inited = _gf_false;
+        gf_boolean_t    cond_inited  = _gf_false;
+        gf_boolean_t    attr_inited  = _gf_false;
+
+	if (!this->children || this->children->next) {
+		gf_log (this->name, GF_LOG_ERROR,
+			"'index' not configured with exactly one child");
+                goto out;
+	}
+
+	if (!this->parents) {
+		gf_log (this->name, GF_LOG_WARNING,
+			"dangling volume. check volfile ");
+	}
+
+        priv = GF_CALLOC (1, sizeof (*priv), gf_index_mt_priv_t);
+        if (!priv)
+                goto out;
+
+        LOCK_INIT (&priv->lock);
+        if ((ret = pthread_cond_init(&priv->cond, NULL)) != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "pthread_cond_init failed (%d)", ret);
+                goto out;
+        }
+        cond_inited = _gf_true;
+
+        if ((ret = pthread_mutex_init(&priv->mutex, NULL)) != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "pthread_mutex_init failed (%d)", ret);
+                goto out;
+        }
+        mutex_inited = _gf_true;
+
+        if ((ret = pthread_attr_init (&w_attr)) != 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "pthread_attr_init failed (%d)", ret);
+                goto out;
+        }
+        attr_inited = _gf_true;
+
+        ret = pthread_attr_setstacksize (&w_attr, INDEX_THREAD_STACK_SIZE);
+        if (ret == EINVAL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Using default thread stack size");
+        }
+        GF_OPTION_INIT ("index-base", priv->index_basepath, path, out);
+        uuid_generate (priv->index);
+        uuid_generate (priv->xattrop_vgfid);
+        INIT_LIST_HEAD (&priv->callstubs);
+
+        this->private = priv;
+
+        ret = index_dir_create (this, XATTROP_SUBDIR);
+        if (ret < 0)
+                goto out;
+
+        ret = gf_thread_create (&thread, &w_attr, index_worker, this);
+        if (ret) {
+                gf_log (this->name, GF_LOG_WARNING, "Failed to create "
+                        "worker thread, aborting");
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (ret) {
+                if (cond_inited)
+                        pthread_cond_destroy (&priv->cond);
+                if (mutex_inited)
+                        pthread_mutex_destroy (&priv->mutex);
+                if (priv)
+                        GF_FREE (priv);
+                this->private = NULL;
+        }
+        if (attr_inited)
+                pthread_attr_destroy (&w_attr);
+        return ret;
+}
+
+void
+fini (xlator_t *this)
+{
+        index_priv_t *priv = NULL;
+
+        priv = this->private;
+        if (!priv)
+                goto out;
+        this->private = NULL;
+        LOCK_DESTROY (&priv->lock);
+        pthread_cond_destroy (&priv->cond);
+        pthread_mutex_destroy (&priv->mutex);
+        GF_FREE (priv);
+out:
+        return;
+}
+
+int
+index_forget (xlator_t *this, inode_t *inode)
+{
+        uint64_t tmp_cache = 0;
+        if (!inode_ctx_del (inode, this, &tmp_cache))
+                GF_FREE ((index_inode_ctx_t*) (long)tmp_cache);
+
+        return 0;
+}
+
+int32_t
+index_releasedir (xlator_t *this, fd_t *fd)
+{
+        index_fd_ctx_t *fctx = NULL;
+        uint64_t        ctx = 0;
+        int             ret = 0;
+
+        ret = fd_ctx_del (fd, this, &ctx);
+        if (ret < 0)
+                goto out;
+
+        fctx = (index_fd_ctx_t*) (long) ctx;
+        if (fctx->dir)
+                closedir (fctx->dir);
+
+        GF_FREE (fctx);
+out:
+        return 0;
+}
+
+int32_t
+index_release (xlator_t *this, fd_t *fd)
+{
+        index_fd_ctx_t *fctx = NULL;
+        uint64_t        ctx = 0;
+        int             ret = 0;
+
+        ret = fd_ctx_del (fd, this, &ctx);
+        if (ret < 0)
+                goto out;
+
+        fctx = (index_fd_ctx_t*) (long) ctx;
+        GF_FREE (fctx);
+out:
+        return 0;
+}
+
+int
+notify (xlator_t *this, int event, void *data, ...)
+{
+        int     ret = 0;
+        ret = default_notify (this, event, data);
+        return ret;
+}
+
+struct xlator_fops fops = {
+	.xattrop     = index_xattrop,
+	.fxattrop    = index_fxattrop,
+
+        //interface functions follow
+        .getxattr    = index_getxattr,
+        .lookup      = index_lookup,
+        .readdir     = index_readdir,
+        .unlink      = index_unlink
+};
+
+struct xlator_dumpops dumpops;
+
+struct xlator_cbks cbks = {
+        .forget         = index_forget,
+        .release        = index_release,
+        .releasedir     = index_releasedir
+};
+
+struct volume_options options[] = {
+        { .key  = {"index-base" },
+          .type = GF_OPTION_TYPE_PATH,
+          .description = "path where the index files need to be stored",
+        },
+        { .key  = {NULL} },
+};
diff --git a/xlators/features/index/src/index.h b/xlators/features/index/src/index.h
new file mode 100644
index 000000000..661dcdbc4
--- /dev/null
+++ b/xlators/features/index/src/index.h
@@ -0,0 +1,59 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __INDEX_H__
+#define __INDEX_H__
+
+#include "xlator.h"
+#include "call-stub.h"
+#include "defaults.h"
+#include "byte-order.h"
+#include "common-utils.h"
+#include "index-mem-types.h"
+
+#define INDEX_THREAD_STACK_SIZE   ((size_t)(1024*1024))
+
+typedef enum {
+        UNKNOWN,
+        IN,
+        NOTIN
+} index_state_t;
+
+typedef struct index_inode_ctx {
+        gf_boolean_t processing;
+        struct list_head callstubs;
+        index_state_t state;
+} index_inode_ctx_t;
+
+typedef struct index_fd_ctx {
+        DIR *dir;
+} index_fd_ctx_t;
+
+typedef struct index_priv {
+        char *index_basepath;
+        uuid_t index;
+        gf_lock_t lock;
+        uuid_t xattrop_vgfid;//virtual gfid of the xattrop index dir
+        struct list_head callstubs;
+        pthread_mutex_t mutex;
+        pthread_cond_t  cond;
+} index_priv_t;
+
+#define INDEX_STACK_UNWIND(fop, frame, params ...)      \
+do {                                                    \
+        if (frame) {                                    \
+                inode_t *_inode = frame->local;         \
+                frame->local = NULL;                    \
+                inode_unref (_inode);                   \
+        }                                               \
+        STACK_UNWIND_STRICT (fop, frame, params);       \
+} while (0)
+
+#endif
diff --git a/xlators/features/locks/src/Makefile.am b/xlators/features/locks/src/Makefile.am
index 53dd3aa5d..0f79731b4 100644
--- a/xlators/features/locks/src/Makefile.am
+++ b/xlators/features/locks/src/Makefile.am
@@ -1,15 +1,18 @@
 xlator_LTLIBRARIES = locks.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-locks_la_LDFLAGS = -module -avoidversion
+locks_la_LDFLAGS = -module -avoid-version
 
-locks_la_SOURCES = common.c posix.c entrylk.c inodelk.c reservelk.c
-locks_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la 
+locks_la_SOURCES = common.c posix.c entrylk.c inodelk.c reservelk.c \
+		   clear.c
+locks_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-noinst_HEADERS = locks.h common.h locks-mem-types.h
+noinst_HEADERS = locks.h common.h locks-mem-types.h clear.h
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -fno-strict-aliasing -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src $(GF_CFLAGS) -shared -nostartfiles
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
+
+
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
 
 CLEANFILES = 
 
@@ -17,4 +20,4 @@ uninstall-local:
 	rm -f $(DESTDIR)$(xlatordir)/posix-locks.so
 
 install-data-hook:
-	ln -sf locks.so $(DESTDIR)$(xlatordir)/posix-locks.so
-\ No newline at end of file
+	ln -sf locks.so $(DESTDIR)$(xlatordir)/posix-locks.so
diff --git a/xlators/features/locks/src/clear.c b/xlators/features/locks/src/clear.c
new file mode 100644
index 000000000..75593b898
--- /dev/null
+++ b/xlators/features/locks/src/clear.c
@@ -0,0 +1,423 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#include <unistd.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <pthread.h>
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "glusterfs.h"
+#include "compat.h"
+#include "xlator.h"
+#include "inode.h"
+#include "logging.h"
+#include "common-utils.h"
+
+#include "locks.h"
+#include "common.h"
+#include "statedump.h"
+#include "clear.h"
+
+int
+clrlk_get_kind (char *kind)
+{
+        char  *clrlk_kinds[CLRLK_KIND_MAX] = {"dummy", "blocked", "granted",
+                                              "all"};
+        int   ret_kind                     = CLRLK_KIND_MAX;
+        int   i                            = 0;
+
+        for (i = CLRLK_BLOCKED; i < CLRLK_KIND_MAX; i++) {
+                if (!strcmp (clrlk_kinds[i], kind)) {
+                        ret_kind = i;
+                        break;
+                }
+        }
+
+        return ret_kind;
+}
+
+int
+clrlk_get_type (char *type)
+{
+        char    *clrlk_types[CLRLK_TYPE_MAX]    = {"inode", "entry", "posix"};
+        int     ret_type                        = CLRLK_TYPE_MAX;
+        int     i                               = 0;
+
+        for (i = CLRLK_INODE; i < CLRLK_TYPE_MAX; i++) {
+                if (!strcmp (clrlk_types[i], type)) {
+                        ret_type = i;
+                        break;
+                }
+        }
+
+        return ret_type;
+}
+
+int
+clrlk_get_lock_range (char *range_str, struct gf_flock *ulock,
+                      gf_boolean_t *chk_range)
+{
+        int     ret     =       -1;
+
+        if (!chk_range)
+                goto out;
+
+        if (!range_str) {
+                ret = 0;
+                *chk_range = _gf_false;
+                goto out;
+        }
+
+        if (sscanf (range_str, "%hd,%"PRId64"-""%"PRId64, &ulock->l_whence,
+                    &ulock->l_start, &ulock->l_len) != 3) {
+                goto out;
+        }
+
+        ret = 0;
+        *chk_range = _gf_true;
+out:
+        return ret;
+}
+
+int
+clrlk_parse_args (const char* cmd, clrlk_args *args)
+{
+        char            *opts           = NULL;
+        char            *cur            = NULL;
+        char            *tok            = NULL;
+        char            *sptr           = NULL;
+        char            *free_ptr       = NULL;
+        char            kw[KW_MAX]     = {[KW_TYPE]     = 't',
+                                          [KW_KIND]     = 'k',
+                                          };
+        int             ret             = -1;
+        int             i               = 0;
+
+        GF_ASSERT (cmd);
+        free_ptr = opts = GF_CALLOC (1, strlen (cmd), gf_common_mt_char);
+        if (!opts)
+                goto out;
+
+        if (sscanf (cmd, GF_XATTR_CLRLK_CMD".%s", opts) < 1) {
+                ret = -1;
+                goto out;
+        }
+
+        /*clr_lk_prefix.ttype.kkind.args, args - type specific*/
+        cur = opts;
+        for (i = 0; i < KW_MAX && (tok = strtok_r (cur, ".", &sptr));
+             cur = NULL, i++) {
+                if (tok[0] != kw[i]) {
+                        ret = -1;
+                        goto out;
+                }
+                if (i == KW_TYPE)
+                        args->type = clrlk_get_type (tok+1);
+                if (i == KW_KIND)
+                        args->kind = clrlk_get_kind (tok+1);
+        }
+
+        if ((args->type == CLRLK_TYPE_MAX) || (args->kind == CLRLK_KIND_MAX))
+                goto out;
+
+        /*optional args, neither range nor basename can 'legally' contain
+         * "/" in them*/
+        tok = strtok_r (NULL, "/", &sptr);
+        if (tok)
+                args->opts = gf_strdup (tok);
+
+        ret = 0;
+out:
+        GF_FREE (free_ptr);
+        return ret;
+}
+
+int
+clrlk_clear_posixlk (xlator_t *this, pl_inode_t *pl_inode, clrlk_args *args,
+                     int *blkd, int *granted, int *op_errno)
+{
+        posix_lock_t            *plock          = NULL;
+        posix_lock_t            *tmp            = NULL;
+        struct gf_flock         ulock           = {0, };
+        int                     ret             = -1;
+        int                     bcount          = 0;
+        int                     gcount          = 0;
+        gf_boolean_t            chk_range       = _gf_false;
+
+        if (clrlk_get_lock_range (args->opts, &ulock, &chk_range)) {
+                *op_errno = EINVAL;
+                goto out;
+        }
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry_safe (plock, tmp, &pl_inode->ext_list,
+                                          list) {
+                        if ((plock->blocked &&
+                             !(args->kind & CLRLK_BLOCKED)) ||
+                            (!plock->blocked &&
+                             !(args->kind & CLRLK_GRANTED)))
+                                continue;
+
+                        if (chk_range &&
+                            (plock->user_flock.l_whence != ulock.l_whence
+                            || plock->user_flock.l_start != ulock.l_start
+                            || plock->user_flock.l_len != ulock.l_len))
+                                continue;
+
+                        list_del_init (&plock->list);
+                        if (plock->blocked) {
+                                bcount++;
+                                pl_trace_out (this, plock->frame, NULL, NULL,
+                                              F_SETLKW, &plock->user_flock,
+                                              -1, EAGAIN, NULL);
+
+                                STACK_UNWIND_STRICT (lk, plock->frame, -1, EAGAIN,
+                                                     &plock->user_flock, NULL);
+
+                        } else {
+                                gcount++;
+                        }
+                        GF_FREE (plock);
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+        grant_blocked_locks (this, pl_inode);
+        ret = 0;
+out:
+        *blkd    = bcount;
+        *granted = gcount;
+        return ret;
+}
+
+/* Returns 0 on success and -1 on failure */
+int
+clrlk_clear_inodelk (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
+                     clrlk_args *args, int *blkd, int *granted, int *op_errno)
+{
+        pl_inode_lock_t         *ilock          = NULL;
+        pl_inode_lock_t         *tmp            = NULL;
+        struct gf_flock         ulock           = {0, };
+        int                     ret             = -1;
+        int                     bcount          = 0;
+        int                     gcount          = 0;
+        gf_boolean_t            chk_range       = _gf_false;
+        struct list_head        released;
+
+        INIT_LIST_HEAD (&released);
+        if (clrlk_get_lock_range (args->opts, &ulock, &chk_range)) {
+                *op_errno = EINVAL;
+                goto out;
+        }
+
+        if (args->kind & CLRLK_BLOCKED)
+                goto blkd;
+
+        if (args->kind & CLRLK_GRANTED)
+                goto granted;
+
+blkd:
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry_safe (ilock, tmp, &dom->blocked_inodelks,
+                                          blocked_locks) {
+                        if (chk_range &&
+                            (ilock->user_flock.l_whence != ulock.l_whence
+                            || ilock->user_flock.l_start != ulock.l_start
+                            || ilock->user_flock.l_len != ulock.l_len))
+                                continue;
+
+                        bcount++;
+                        list_del_init (&ilock->blocked_locks);
+                        list_add (&ilock->blocked_locks, &released);
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        list_for_each_entry_safe (ilock, tmp, &released, blocked_locks) {
+                list_del_init (&ilock->blocked_locks);
+                pl_trace_out (this, ilock->frame, NULL, NULL, F_SETLKW,
+                              &ilock->user_flock, -1, EAGAIN,
+                              ilock->volume);
+                STACK_UNWIND_STRICT (inodelk, ilock->frame, -1,
+                                     EAGAIN, NULL);
+                //No need to take lock as the locks are only in one list
+                __pl_inodelk_unref (ilock);
+        }
+
+        if (!(args->kind & CLRLK_GRANTED)) {
+                ret = 0;
+                goto out;
+        }
+
+granted:
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry_safe (ilock, tmp, &dom->inodelk_list,
+                                          list) {
+                        if (chk_range &&
+                            (ilock->user_flock.l_whence != ulock.l_whence
+                            || ilock->user_flock.l_start != ulock.l_start
+                            || ilock->user_flock.l_len != ulock.l_len))
+                                continue;
+
+                        gcount++;
+                        list_del_init (&ilock->list);
+                        list_add (&ilock->list, &released);
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        list_for_each_entry_safe (ilock, tmp, &released, list) {
+                list_del_init (&ilock->list);
+                //No need to take lock as the locks are only in one list
+                __pl_inodelk_unref (ilock);
+        }
+
+        ret = 0;
+out:
+        grant_blocked_inode_locks (this, pl_inode, dom);
+        *blkd    = bcount;
+        *granted = gcount;
+        return ret;
+}
+
+/* Returns 0 on success and -1 on failure */
+int
+clrlk_clear_entrylk (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
+                     clrlk_args *args, int *blkd, int *granted, int *op_errno)
+{
+        pl_entry_lock_t         *elock          = NULL;
+        pl_entry_lock_t         *tmp            = NULL;
+        int                     bcount          = 0;
+        int                     gcount          = 0;
+        int                     ret             = -1;
+        struct list_head        removed;
+        struct list_head        released;
+
+        INIT_LIST_HEAD (&released);
+        if (args->kind & CLRLK_BLOCKED)
+                goto blkd;
+
+        if (args->kind & CLRLK_GRANTED)
+                goto granted;
+
+blkd:
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry_safe (elock, tmp, &dom->blocked_entrylks,
+                                          blocked_locks) {
+                        if (args->opts) {
+                                if (!elock->basename ||
+                                    strcmp (elock->basename, args->opts))
+                                        continue;
+                        }
+
+                        bcount++;
+
+                        list_del_init (&elock->blocked_locks);
+                        list_add_tail (&elock->blocked_locks, &released);
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        list_for_each_entry_safe (elock, tmp, &released, blocked_locks) {
+                list_del_init (&elock->blocked_locks);
+                entrylk_trace_out (this, elock->frame, elock->volume, NULL, NULL,
+                                   elock->basename, ENTRYLK_LOCK, elock->type,
+                                   -1, EAGAIN);
+                STACK_UNWIND_STRICT (entrylk, elock->frame, -1, EAGAIN, NULL);
+
+		__pl_entrylk_unref (elock);
+        }
+
+        if (!(args->kind & CLRLK_GRANTED)) {
+                ret = 0;
+                goto out;
+        }
+
+granted:
+        INIT_LIST_HEAD (&removed);
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry_safe (elock, tmp, &dom->entrylk_list,
+                                          domain_list) {
+                        if (args->opts) {
+                                if (!elock->basename ||
+                                    strcmp (elock->basename, args->opts))
+                                        continue;
+                        }
+
+                        gcount++;
+                        list_del_init (&elock->domain_list);
+                        list_add_tail (&elock->domain_list, &removed);
+
+			__pl_entrylk_unref (elock);
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+	grant_blocked_entry_locks (this, pl_inode, dom);
+
+        ret = 0;
+out:
+        *blkd    = bcount;
+        *granted = gcount;
+        return ret;
+}
+
+int
+clrlk_clear_lks_in_all_domains (xlator_t *this, pl_inode_t *pl_inode,
+                                clrlk_args *args, int *blkd, int *granted,
+                                int *op_errno)
+{
+        pl_dom_list_t   *dom            = NULL;
+        int             ret             = -1;
+        int             tmp_bcount      = 0;
+        int             tmp_gcount      = 0;
+
+        if (list_empty (&pl_inode->dom_list)) {
+                ret = 0;
+                goto out;
+        }
+
+        list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
+                tmp_bcount = tmp_gcount = 0;
+
+                switch (args->type)
+                {
+                case CLRLK_INODE:
+                        ret = clrlk_clear_inodelk (this, pl_inode, dom, args,
+                                             &tmp_bcount, &tmp_gcount,
+                                             op_errno);
+                        if (ret)
+                                goto out;
+                        break;
+                case CLRLK_ENTRY:
+                        ret = clrlk_clear_entrylk (this, pl_inode, dom, args,
+                                             &tmp_bcount, &tmp_gcount,
+                                             op_errno);
+                        if (ret)
+                                goto out;
+                        break;
+                }
+
+                *blkd    += tmp_bcount;
+                *granted += tmp_gcount;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
diff --git a/xlators/features/locks/src/clear.h b/xlators/features/locks/src/clear.h
new file mode 100644
index 000000000..511f3f74a
--- /dev/null
+++ b/xlators/features/locks/src/clear.h
@@ -0,0 +1,76 @@
+/*
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __CLEAR_H__
+#define __CLEAR_H__
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "compat-errno.h"
+#include "stack.h"
+#include "call-stub.h"
+#include "locks.h"
+
+typedef enum {
+        CLRLK_INODE,
+        CLRLK_ENTRY,
+        CLRLK_POSIX,
+        CLRLK_TYPE_MAX
+} clrlk_type;
+
+typedef enum {
+        CLRLK_BLOCKED = 1,
+        CLRLK_GRANTED,
+        CLRLK_ALL,
+        CLRLK_KIND_MAX
+} clrlk_kind;
+
+typedef enum {
+        KW_TYPE,
+        KW_KIND,
+        /*add new keywords here*/
+        KW_MAX
+} clrlk_opts;
+
+struct _clrlk_args;
+typedef struct _clrlk_args clrlk_args;
+
+struct _clrlk_args {
+        int type;
+        int kind;
+        char *opts;
+};
+
+int
+clrlk_get__kind (char *kind);
+int
+clrlk_get_type (char *type);
+int
+clrlk_get_lock_range (char *range_str, struct gf_flock *ulock,
+                      gf_boolean_t *chk_range);
+int
+clrlk_parse_args (const char* cmd, clrlk_args *args);
+
+int
+clrlk_clear_posixlk (xlator_t *this, pl_inode_t *pl_inode, clrlk_args *args,
+                     int *blkd, int *granted, int *op_errno);
+int
+clrlk_clear_inodelk (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
+                     clrlk_args *args, int *blkd, int *granted, int *op_errno);
+int
+clrlk_clear_entrylk (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom,
+                     clrlk_args *args, int *blkd, int *granted, int *op_errno);
+int
+clrlk_clear_lks_in_all_domains (xlator_t *this, pl_inode_t *pl_inode,
+                                clrlk_args *args, int *blkd, int *granted,
+                                int *op_errno);
+#endif /* __CLEAR_H__ */
diff --git a/xlators/features/locks/src/common.c b/xlators/features/locks/src/common.c
index e295e4fd5..f6c71c1cf 100644
--- a/xlators/features/locks/src/common.c
+++ b/xlators/features/locks/src/common.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006, 2007, 2008 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <unistd.h>
 #include <fcntl.h>
 #include <limits.h>
@@ -45,23 +35,20 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock);
 static int
 pl_send_prelock_unlock (xlator_t *this, pl_inode_t *pl_inode,
                         posix_lock_t *old_lock);
+
 static pl_dom_list_t *
-allocate_domain (const char *volume)
+__allocate_domain (const char *volume)
 {
         pl_dom_list_t *dom = NULL;
 
         dom = GF_CALLOC (1, sizeof (*dom),
                          gf_locks_mt_pl_dom_list_t);
         if (!dom)
-                return NULL;
-
+                goto out;
 
         dom->domain = gf_strdup(volume);
-	if (!dom->domain) {
-		gf_log ("posix-locks", GF_LOG_TRACE,
-			"Out of Memory");
-		return NULL;
-	}
+        if (!dom->domain)
+                goto out;
 
         gf_log ("posix-locks", GF_LOG_TRACE,
                 "New domain allocated: %s", dom->domain);
@@ -72,6 +59,12 @@ allocate_domain (const char *volume)
         INIT_LIST_HEAD (&dom->inodelk_list);
         INIT_LIST_HEAD (&dom->blocked_inodelks);
 
+out:
+        if (dom && (NULL == dom->domain)) {
+                GF_FREE (dom);
+                dom = NULL;
+        }
+
         return dom;
 }
 
@@ -83,19 +76,28 @@ get_domain (pl_inode_t *pl_inode, const char *volume)
 {
         pl_dom_list_t *dom = NULL;
 
-        list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
-                if (strcmp (dom->domain, volume) == 0)
-                        goto found;
+        GF_VALIDATE_OR_GOTO ("posix-locks", pl_inode, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", volume, out);
 
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
+                        if (strcmp (dom->domain, volume) == 0)
+                                goto unlock;
+                }
 
+                dom = __allocate_domain (volume);
+                if (dom)
+                        list_add (&dom->inode_list, &pl_inode->dom_list);
         }
-
-        dom = allocate_domain(volume);
-
-        if (dom)
-                list_add (&dom->inode_list, &pl_inode->dom_list);
-found:
-
+unlock:
+        pthread_mutex_unlock (&pl_inode->mutex);
+        if (dom) {
+                gf_log ("posix-locks", GF_LOG_TRACE, "Domain %s found", volume);
+        } else {
+                gf_log ("posix-locks", GF_LOG_TRACE, "Domain %s not found", volume);
+        }
+out:
         return dom;
 }
 
@@ -134,10 +136,10 @@ __pl_inode_is_empty (pl_inode_t *pl_inode)
 void
 pl_print_locker (char *str, int size, xlator_t *this, call_frame_t *frame)
 {
-        snprintf (str, size, "Pid=%llu, lk-owner=%llu, Transport=%p, Frame=%llu",
+        snprintf (str, size, "Pid=%llu, lk-owner=%s, Client=%p, Frame=%llu",
                   (unsigned long long) frame->root->pid,
-                  (unsigned long long) frame->root->lk_owner,
-                  (void *)frame->root->trans,
+                  lkowner_utoa (&frame->root->lk_owner),
+                  frame->root->client,
                   (unsigned long long) frame->root->unique);
 }
 
@@ -167,18 +169,17 @@ pl_print_lockee (char *str, int size, fd_t *fd, loc_t *loc)
                         ipath = NULL;
         }
 
-        snprintf (str, size, "ino=%llu, fd=%p, path=%s",
-                  (unsigned long long) inode->ino, fd,
+        snprintf (str, size, "gfid=%s, fd=%p, path=%s",
+                  uuid_utoa (inode->gfid), fd,
                   ipath ? ipath : "<nul>");
 
-        if (ipath)
-                GF_FREE (ipath);
+        GF_FREE (ipath);
 }
 
 
 void
 pl_print_lock (char *str, int size, int cmd,
-               struct gf_flock *flock, uint64_t owner)
+               struct gf_flock *flock, gf_lkowner_t *owner)
 {
         char *cmd_str = NULL;
         char *type_str = NULL;
@@ -226,11 +227,11 @@ pl_print_lock (char *str, int size, int cmd,
         }
 
         snprintf (str, size, "lock=FCNTL, cmd=%s, type=%s, "
-                  "start=%llu, len=%llu, pid=%llu, lk-owner=%llu",
+                  "start=%llu, len=%llu, pid=%llu, lk-owner=%s",
                   cmd_str, type_str, (unsigned long long) flock->l_start,
                   (unsigned long long) flock->l_len,
                   (unsigned long long) flock->l_pid,
-                  (unsigned long long) owner);
+                  lkowner_utoa (owner));
 }
 
 
@@ -253,9 +254,9 @@ pl_trace_in (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
         if (domain)
                 pl_print_inodelk (pl_lock, 256, cmd, flock, domain);
         else
-                pl_print_lock (pl_lock, 256, cmd, flock, frame->root->lk_owner);
+                pl_print_lock (pl_lock, 256, cmd, flock, &frame->root->lk_owner);
 
-        gf_log (this->name, GF_LOG_NORMAL,
+        gf_log (this->name, GF_LOG_INFO,
                 "[REQUEST] Locker = {%s} Lockee = {%s} Lock = {%s}",
                 pl_locker, pl_lockee, pl_lock);
 }
@@ -303,11 +304,11 @@ pl_trace_out (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
         if (domain)
                 pl_print_inodelk (pl_lock, 256, cmd, flock, domain);
         else
-                pl_print_lock (pl_lock, 256, cmd, flock, frame->root->lk_owner);
+                pl_print_lock (pl_lock, 256, cmd, flock, &frame->root->lk_owner);
 
         pl_print_verdict (verdict, 32, op_ret, op_errno);
 
-        gf_log (this->name, GF_LOG_NORMAL,
+        gf_log (this->name, GF_LOG_INFO,
                 "[%s] Locker = {%s} Lockee = {%s} Lock = {%s}",
                 verdict, pl_locker, pl_lockee, pl_lock);
 }
@@ -333,9 +334,9 @@ pl_trace_block (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
         if (domain)
                 pl_print_inodelk (pl_lock, 256, cmd, flock, domain);
         else
-                pl_print_lock (pl_lock, 256, cmd, flock, frame->root->lk_owner);
+                pl_print_lock (pl_lock, 256, cmd, flock, &frame->root->lk_owner);
 
-        gf_log (this->name, GF_LOG_NORMAL,
+        gf_log (this->name, GF_LOG_INFO,
                 "[BLOCKED] Locker = {%s} Lockee = {%s} Lock = {%s}",
                 pl_locker, pl_lockee, pl_lock);
 }
@@ -362,7 +363,7 @@ pl_trace_flush (xlator_t *this, call_frame_t *frame, fd_t *fd)
         pl_print_locker (pl_locker, 256, this, frame);
         pl_print_lockee (pl_lockee, 256, fd, NULL);
 
-        gf_log (this->name, GF_LOG_NORMAL,
+        gf_log (this->name, GF_LOG_INFO,
                 "[FLUSH] Locker = {%s} Lockee = {%s}",
                 pl_locker, pl_lockee);
 }
@@ -380,7 +381,7 @@ pl_trace_release (xlator_t *this, fd_t *fd)
 
         pl_print_lockee (pl_lockee, 256, fd, NULL);
 
-        gf_log (this->name, GF_LOG_NORMAL,
+        gf_log (this->name, GF_LOG_INFO,
                 "[RELEASE] Lockee = {%s}", pl_lockee);
 }
 
@@ -391,6 +392,7 @@ pl_update_refkeeper (xlator_t *this, inode_t *inode)
         pl_inode_t *pl_inode  = NULL;
         int         is_empty  = 0;
         int         need_unref = 0;
+        int         need_ref = 0;
 
         pl_inode = pl_inode_get (this, inode);
 
@@ -404,13 +406,17 @@ pl_update_refkeeper (xlator_t *this, inode_t *inode)
                 }
 
                 if (!is_empty && !pl_inode->refkeeper) {
-                        pl_inode->refkeeper = inode_ref (inode);
+                        need_ref = 1;
+                        pl_inode->refkeeper = inode;
                 }
         }
         pthread_mutex_unlock (&pl_inode->mutex);
 
         if (need_unref)
                 inode_unref (inode);
+
+        if (need_ref)
+                inode_ref (inode);
 }
 
 
@@ -418,78 +424,78 @@ pl_inode_t *
 pl_inode_get (xlator_t *this, inode_t *inode)
 {
         uint64_t    tmp_pl_inode = 0;
-	pl_inode_t *pl_inode = NULL;
-//	mode_t      st_mode = 0;
-	int         ret = 0;
-
-	ret = inode_ctx_get (inode, this,&tmp_pl_inode);
-	if (ret == 0) {
-                pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
-		goto out;
-        }
-	pl_inode = GF_CALLOC (1, sizeof (*pl_inode),
-                              gf_locks_mt_pl_inode_t);
-	if (!pl_inode) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		goto out;
-	}
+        pl_inode_t *pl_inode = NULL;
+        int         ret = 0;
 
-        gf_log (this->name, GF_LOG_TRACE,
-                "Allocating new pl inode");
+        LOCK (&inode->lock);
+        {
+                ret = __inode_ctx_get (inode, this, &tmp_pl_inode);
+                if (ret == 0) {
+                        pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
+                        goto unlock;
+                }
+                pl_inode = GF_CALLOC (1, sizeof (*pl_inode),
+                                      gf_locks_mt_pl_inode_t);
+                if (!pl_inode) {
+                        goto unlock;
+                }
 
-/*
-	st_mode  = inode->st_mode;
-	if ((st_mode & S_ISGID) && !(st_mode & S_IXGRP))
-		pl_inode->mandatory = 1;
-*/
+                gf_log (this->name, GF_LOG_TRACE,
+                        "Allocating new pl inode");
 
-	pthread_mutex_init (&pl_inode->mutex, NULL);
+                pthread_mutex_init (&pl_inode->mutex, NULL);
 
-	INIT_LIST_HEAD (&pl_inode->dom_list);
-	INIT_LIST_HEAD (&pl_inode->ext_list);
-	INIT_LIST_HEAD (&pl_inode->rw_list);
-	INIT_LIST_HEAD (&pl_inode->reservelk_list);
-	INIT_LIST_HEAD (&pl_inode->blocked_reservelks);
-        INIT_LIST_HEAD (&pl_inode->blocked_calls);
+                INIT_LIST_HEAD (&pl_inode->dom_list);
+                INIT_LIST_HEAD (&pl_inode->ext_list);
+                INIT_LIST_HEAD (&pl_inode->rw_list);
+                INIT_LIST_HEAD (&pl_inode->reservelk_list);
+                INIT_LIST_HEAD (&pl_inode->blocked_reservelks);
+                INIT_LIST_HEAD (&pl_inode->blocked_calls);
 
-	inode_ctx_put (inode, this, (uint64_t)(long)(pl_inode));
+                __inode_ctx_put (inode, this, (uint64_t)(long)(pl_inode));
+        }
+unlock:
+        UNLOCK (&inode->lock);
 
-out:
-	return pl_inode;
+        return pl_inode;
 }
 
 
 /* Create a new posix_lock_t */
 posix_lock_t *
-new_posix_lock (struct gf_flock *flock, void *transport, pid_t client_pid,
-                uint64_t owner, fd_t *fd)
+new_posix_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
+                gf_lkowner_t *owner, fd_t *fd)
 {
-	posix_lock_t *lock = NULL;
+        posix_lock_t *lock = NULL;
+
+        GF_VALIDATE_OR_GOTO ("posix-locks", flock, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", client, out);
+        GF_VALIDATE_OR_GOTO ("posix-locks", fd, out);
 
-	lock = GF_CALLOC (1, sizeof (posix_lock_t),
+        lock = GF_CALLOC (1, sizeof (posix_lock_t),
                           gf_locks_mt_posix_lock_t);
-	if (!lock) {
-		return NULL;
-	}
+        if (!lock) {
+                goto out;
+        }
 
-	lock->fl_start = flock->l_start;
-	lock->fl_type  = flock->l_type;
+        lock->fl_start = flock->l_start;
+        lock->fl_type  = flock->l_type;
 
-	if (flock->l_len == 0)
-		lock->fl_end = LLONG_MAX;
-	else
-		lock->fl_end = flock->l_start + flock->l_len - 1;
+        if (flock->l_len == 0)
+                lock->fl_end = LLONG_MAX;
+        else
+                lock->fl_end = flock->l_start + flock->l_len - 1;
 
-	lock->transport  = transport;
+        lock->client     = client;
         lock->fd_num     = fd_to_fdnum (fd);
         lock->fd         = fd;
-	lock->client_pid = client_pid;
-        lock->owner      = owner;
+        lock->client_pid = client_pid;
+        lock->owner      = *owner;
 
-	INIT_LIST_HEAD (&lock->list);
+        INIT_LIST_HEAD (&lock->list);
 
-	return lock;
+out:
+        return lock;
 }
 
 
@@ -497,7 +503,7 @@ new_posix_lock (struct gf_flock *flock, void *transport, pid_t client_pid,
 void
 __delete_lock (pl_inode_t *pl_inode, posix_lock_t *lock)
 {
-	list_del_init (&lock->list);
+        list_del_init (&lock->list);
 }
 
 
@@ -505,7 +511,7 @@ __delete_lock (pl_inode_t *pl_inode, posix_lock_t *lock)
 void
 __destroy_lock (posix_lock_t *lock)
 {
-	GF_FREE (lock);
+        GF_FREE (lock);
 }
 
 
@@ -513,24 +519,29 @@ __destroy_lock (posix_lock_t *lock)
 void
 posix_lock_to_flock (posix_lock_t *lock, struct gf_flock *flock)
 {
-	flock->l_pid   = lock->client_pid;
-	flock->l_type  = lock->fl_type;
-	flock->l_start = lock->fl_start;
+        flock->l_pid   = lock->client_pid;
+        flock->l_type  = lock->fl_type;
+        flock->l_start = lock->fl_start;
         flock->l_owner = lock->owner;
 
-	if (lock->fl_end == LLONG_MAX)
-		flock->l_len = 0;
-	else
-		flock->l_len = lock->fl_end - lock->fl_start + 1;
+        if (lock->fl_end == LLONG_MAX)
+                flock->l_len = 0;
+        else
+                flock->l_len = lock->fl_end - lock->fl_start + 1;
 }
 
 /* Insert the lock into the inode's lock list */
 static void
 __insert_lock (pl_inode_t *pl_inode, posix_lock_t *lock)
 {
+        if (lock->blocked)
+                gettimeofday (&lock->blkd_time, NULL);
+        else
+                gettimeofday (&lock->granted_time, NULL);
+
         list_add_tail (&lock->list, &pl_inode->ext_list);
 
-	return;
+        return;
 }
 
 
@@ -538,14 +549,14 @@ __insert_lock (pl_inode_t *pl_inode, posix_lock_t *lock)
 int
 locks_overlap (posix_lock_t *l1, posix_lock_t *l2)
 {
-	/* 
-	   Note:
-	   FUSE always gives us absolute offsets, so no need to worry 
-	   about SEEK_CUR or SEEK_END
-	*/
-
-	return ((l1->fl_end >= l2->fl_start) &&
-		(l2->fl_end >= l1->fl_start));
+        /*
+           Note:
+           FUSE always gives us absolute offsets, so no need to worry
+           about SEEK_CUR or SEEK_END
+        */
+
+        return ((l1->fl_end >= l2->fl_start) &&
+                (l2->fl_end >= l1->fl_start));
 }
 
 
@@ -554,8 +565,8 @@ int
 same_owner (posix_lock_t *l1, posix_lock_t *l2)
 {
 
-                return ((l1->owner == l2->owner) &&
-                        (l1->transport  == l2->transport));
+        return (is_same_lkowner (&l1->owner, &l2->owner) &&
+                (l1->client == l2->client));
 
 }
 
@@ -564,15 +575,15 @@ same_owner (posix_lock_t *l1, posix_lock_t *l2)
 void
 __delete_unlck_locks (pl_inode_t *pl_inode)
 {
-	posix_lock_t *l = NULL;
-	posix_lock_t *tmp = NULL;
-
-	list_for_each_entry_safe (l, tmp, &pl_inode->ext_list, list) {
-		if (l->fl_type == F_UNLCK) {
-			__delete_lock (pl_inode, l);
-			__destroy_lock (l);
-		}
-	}
+        posix_lock_t *l = NULL;
+        posix_lock_t *tmp = NULL;
+
+        list_for_each_entry_safe (l, tmp, &pl_inode->ext_list, list) {
+                if (l->fl_type == F_UNLCK) {
+                        __delete_lock (pl_inode, l);
+                        __destroy_lock (l);
+                }
+        }
 }
 
 
@@ -580,22 +591,22 @@ __delete_unlck_locks (pl_inode_t *pl_inode)
 static posix_lock_t *
 add_locks (posix_lock_t *l1, posix_lock_t *l2)
 {
-	posix_lock_t *sum = NULL;
+        posix_lock_t *sum = NULL;
 
-	sum = GF_CALLOC (1, sizeof (posix_lock_t),
+        sum = GF_CALLOC (1, sizeof (posix_lock_t),
                          gf_locks_mt_posix_lock_t);
-	if (!sum)
-		return NULL;
+        if (!sum)
+                return NULL;
 
-	sum->fl_start = min (l1->fl_start, l2->fl_start);
-	sum->fl_end   = max (l1->fl_end, l2->fl_end);
+        sum->fl_start = min (l1->fl_start, l2->fl_start);
+        sum->fl_end   = max (l1->fl_end, l2->fl_end);
 
-	return sum;
+        return sum;
 }
 
 /* Subtract two locks */
 struct _values {
-	posix_lock_t *locks[3];
+        posix_lock_t *locks[3];
 };
 
 /* {big} must always be contained inside {small} */
@@ -603,89 +614,88 @@ static struct _values
 subtract_locks (posix_lock_t *big, posix_lock_t *small)
 {
 
-	struct _values v = { .locks = {0, 0, 0} };
+        struct _values v = { .locks = {0, 0, 0} };
 
-	if ((big->fl_start == small->fl_start) &&
-	    (big->fl_end   == small->fl_end)) {
-		/* both edges coincide with big */
-		v.locks[0] = GF_CALLOC (1, sizeof (posix_lock_t),
+        if ((big->fl_start == small->fl_start) &&
+            (big->fl_end   == small->fl_end)) {
+                /* both edges coincide with big */
+                v.locks[0] = GF_CALLOC (1, sizeof (posix_lock_t),
                                         gf_locks_mt_posix_lock_t);
                 if (!v.locks[0])
                         goto out;
-		memcpy (v.locks[0], big, sizeof (posix_lock_t));
-		v.locks[0]->fl_type = small->fl_type;
+                memcpy (v.locks[0], big, sizeof (posix_lock_t));
+                v.locks[0]->fl_type = small->fl_type;
                 goto done;
-	}
+        }
 
-	if ((small->fl_start > big->fl_start) &&
+        if ((small->fl_start > big->fl_start) &&
             (small->fl_end   < big->fl_end)) {
-		/* both edges lie inside big */
-		v.locks[0] = GF_CALLOC (1, sizeof (posix_lock_t),
+                /* both edges lie inside big */
+                v.locks[0] = GF_CALLOC (1, sizeof (posix_lock_t),
                                         gf_locks_mt_posix_lock_t);
                 if (!v.locks[0])
                         goto out;
 
-		v.locks[1] = GF_CALLOC (1, sizeof (posix_lock_t),
+                v.locks[1] = GF_CALLOC (1, sizeof (posix_lock_t),
                                         gf_locks_mt_posix_lock_t);
                 if (!v.locks[1])
                         goto out;
 
-		v.locks[2] = GF_CALLOC (1, sizeof (posix_lock_t),
+                v.locks[2] = GF_CALLOC (1, sizeof (posix_lock_t),
                                         gf_locks_mt_posix_lock_t);
                 if (!v.locks[1])
                         goto out;
 
-		memcpy (v.locks[0], big, sizeof (posix_lock_t));
-		v.locks[0]->fl_end = small->fl_start - 1;
+                memcpy (v.locks[0], big, sizeof (posix_lock_t));
+                v.locks[0]->fl_end = small->fl_start - 1;
 
-		memcpy (v.locks[1], small, sizeof (posix_lock_t));
+                memcpy (v.locks[1], small, sizeof (posix_lock_t));
 
-		memcpy (v.locks[2], big, sizeof (posix_lock_t));
-		v.locks[2]->fl_start = small->fl_end + 1;
+                memcpy (v.locks[2], big, sizeof (posix_lock_t));
+                v.locks[2]->fl_start = small->fl_end + 1;
                 goto done;
 
-	}
+        }
 
-	/* one edge coincides with big */
+        /* one edge coincides with big */
         if (small->fl_start == big->fl_start) {
-		v.locks[0] = GF_CALLOC (1, sizeof (posix_lock_t),
+                v.locks[0] = GF_CALLOC (1, sizeof (posix_lock_t),
                                         gf_locks_mt_posix_lock_t);
                 if (!v.locks[0])
                         goto out;
 
-		v.locks[1] = GF_CALLOC (1, sizeof (posix_lock_t),
+                v.locks[1] = GF_CALLOC (1, sizeof (posix_lock_t),
                                         gf_locks_mt_posix_lock_t);
                 if (!v.locks[1])
                         goto out;
 
-		memcpy (v.locks[0], big, sizeof (posix_lock_t));
-		v.locks[0]->fl_start = small->fl_end + 1;
+                memcpy (v.locks[0], big, sizeof (posix_lock_t));
+                v.locks[0]->fl_start = small->fl_end + 1;
 
-		memcpy (v.locks[1], small, sizeof (posix_lock_t));
+                memcpy (v.locks[1], small, sizeof (posix_lock_t));
                 goto done;
-	}
+        }
 
-	if (small->fl_end  == big->fl_end) {
-		v.locks[0] = GF_CALLOC (1, sizeof (posix_lock_t),
+        if (small->fl_end  == big->fl_end) {
+                v.locks[0] = GF_CALLOC (1, sizeof (posix_lock_t),
                                         gf_locks_mt_posix_lock_t);
                 if (!v.locks[0])
                         goto out;
 
-		v.locks[1] = GF_CALLOC (1, sizeof (posix_lock_t),
+                v.locks[1] = GF_CALLOC (1, sizeof (posix_lock_t),
                                         gf_locks_mt_posix_lock_t);
                 if (!v.locks[1])
                         goto out;
 
-		memcpy (v.locks[0], big, sizeof (posix_lock_t));
-		v.locks[0]->fl_end = small->fl_start - 1;
+                memcpy (v.locks[0], big, sizeof (posix_lock_t));
+                v.locks[0]->fl_end = small->fl_start - 1;
 
-		memcpy (v.locks[1], small, sizeof (posix_lock_t));
+                memcpy (v.locks[1], small, sizeof (posix_lock_t));
                 goto done;
-	}
+        }
 
-        gf_log ("posix-locks", GF_LOG_ERROR,
-                "Unexpected case in subtract_locks. Please send "
-                "a bug report to gluster-devel@nongnu.org");
+        GF_ASSERT (0);
+        gf_log ("posix-locks", GF_LOG_ERROR, "Unexpected case in subtract_locks");
 
 out:
         if (v.locks[0]) {
@@ -705,6 +715,36 @@ done:
         return v;
 }
 
+static posix_lock_t *
+first_conflicting_overlap (pl_inode_t *pl_inode, posix_lock_t *lock)
+{
+        posix_lock_t *l = NULL;
+        posix_lock_t *conf = NULL;
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry (l, &pl_inode->ext_list, list) {
+                        if (l->blocked)
+                                continue;
+
+                        if (locks_overlap (l, lock)) {
+                                if (same_owner (l, lock))
+                                        continue;
+
+                                if ((l->fl_type == F_WRLCK) ||
+                                    (lock->fl_type == F_WRLCK)) {
+                                        conf = l;
+                                        goto unlock;
+                                }
+                        }
+                }
+        }
+unlock:
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        return conf;
+}
+
 /*
   Start searching from {begin}, and return the first lock that
   conflicts, NULL if no conflict
@@ -763,6 +803,8 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock)
         struct _values v = { .locks = {0, 0, 0} };
 
         list_for_each_entry_safe (conf, t, &pl_inode->ext_list, list) {
+                if (conf->blocked)
+                        continue;
                 if (!locks_overlap (conf, lock))
                         continue;
 
@@ -771,7 +813,7 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock)
                                 sum = add_locks (lock, conf);
 
                                 sum->fl_type    = lock->fl_type;
-                                sum->transport  = lock->transport;
+                                sum->client     = lock->client;
                                 sum->fd_num     = lock->fd_num;
                                 sum->client_pid = lock->client_pid;
                                 sum->owner      = lock->owner;
@@ -780,6 +822,8 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock)
                                 __destroy_lock (conf);
 
                                 __destroy_lock (lock);
+                                INIT_LIST_HEAD (&sum->list);
+                                posix_lock_to_flock (sum, &sum->user_flock);
                                 __insert_and_merge (pl_inode, sum);
 
                                 return;
@@ -787,7 +831,7 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock)
                                 sum = add_locks (lock, conf);
 
                                 sum->fl_type    = conf->fl_type;
-                                sum->transport  = conf->transport;
+                                sum->client     = conf->client;
                                 sum->fd_num     = conf->fd_num;
                                 sum->client_pid = conf->client_pid;
                                 sum->owner      = conf->owner;
@@ -807,6 +851,8 @@ __insert_and_merge (pl_inode_t *pl_inode, posix_lock_t *lock)
                                                 continue;
 
                                         INIT_LIST_HEAD (&v.locks[i]->list);
+                                        posix_lock_to_flock (v.locks[i],
+                                                       &v.locks[i]->user_flock);
                                         __insert_and_merge (pl_inode,
                                                             v.locks[i]);
                                 }
@@ -860,7 +906,7 @@ __grant_blocked_locks (xlator_t *this, pl_inode_t *pl_inode, struct list_head *g
                 list_del_init (&l->list);
 
                 if (__is_lock_grantable (pl_inode, l)) {
-			conf = GF_CALLOC (1, sizeof (*conf),
+                        conf = GF_CALLOC (1, sizeof (*conf),
                                           gf_locks_mt_posix_lock_t);
 
                         if (!conf) {
@@ -875,10 +921,9 @@ __grant_blocked_locks (xlator_t *this, pl_inode_t *pl_inode, struct list_head *g
                         posix_lock_to_flock (l, &conf->user_flock);
 
                         gf_log (this->name, GF_LOG_TRACE,
-                                "%s (pid=%d) lk-owner:%"PRIu64" %"PRId64" - %"PRId64" => Granted",
+                                "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => Granted",
                                 l->fl_type == F_UNLCK ? "Unlock" : "Lock",
-                                l->client_pid,
-                                l->owner,
+                                l->client_pid, lkowner_utoa (&l->owner),
                                 l->user_flock.l_start,
                                 l->user_flock.l_len);
 
@@ -914,7 +959,8 @@ grant_blocked_locks (xlator_t *this, pl_inode_t *pl_inode)
                 pl_trace_out (this, lock->frame, NULL, NULL, F_SETLKW,
                               &lock->user_flock, 0, 0, NULL);
 
-                STACK_UNWIND (lock->frame, 0, 0, &lock->user_flock);
+                STACK_UNWIND_STRICT (lk, lock->frame, 0, 0,
+                                     &lock->user_flock, NULL);
 
                 GF_FREE (lock);
         }
@@ -933,7 +979,7 @@ pl_send_prelock_unlock (xlator_t *this, pl_inode_t *pl_inode,
         posix_lock_t     *tmp = NULL;
         posix_lock_t     *lock = NULL;
 
-        int ret = 0;
+        int ret = -1;
 
         INIT_LIST_HEAD (&granted_list);
 
@@ -943,15 +989,11 @@ pl_send_prelock_unlock (xlator_t *this, pl_inode_t *pl_inode,
         flock.l_len    = old_lock->user_flock.l_len;
 
 
-        unlock_lock = new_posix_lock (&flock, old_lock->transport,
-                                      old_lock->client_pid, old_lock->owner,
+        unlock_lock = new_posix_lock (&flock, old_lock->client,
+                                      old_lock->client_pid, &old_lock->owner,
                                       old_lock->fd);
-        if (!unlock_lock) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory");
-                ret = -1;
-                goto out;
-        }
+        GF_VALIDATE_OR_GOTO (this->name, unlock_lock, out);
+        ret = 0;
 
         __insert_and_merge (pl_inode, unlock_lock);
 
@@ -963,13 +1005,13 @@ pl_send_prelock_unlock (xlator_t *this, pl_inode_t *pl_inode,
                 pl_trace_out (this, lock->frame, NULL, NULL, F_SETLKW,
                               &lock->user_flock, 0, 0, NULL);
 
-                STACK_UNWIND (lock->frame, 0, 0, &lock->user_flock);
+                STACK_UNWIND_STRICT (lk, lock->frame, 0, 0,
+                                     &lock->user_flock, NULL);
 
                 GF_FREE (lock);
         }
 
 out:
-
         return ret;
 }
 
@@ -985,31 +1027,36 @@ pl_setlk (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
         {
                 /* Send unlock before the actual lock to
                    prevent lock upgrade / downgrade
-                   problems
+                   problems only if:
+                   - it is a blocking call
+                   - it has other conflicting locks
                 */
 
-                ret = pl_send_prelock_unlock (this, pl_inode,
-                                              lock);
-                if (ret)
-                        gf_log (this->name, GF_LOG_DEBUG,
+                if (can_block &&
+                    !(__is_lock_grantable (pl_inode, lock))) {
+                        ret = pl_send_prelock_unlock (this, pl_inode,
+                                                      lock);
+                        if (ret)
+                                gf_log (this->name, GF_LOG_DEBUG,
                                 "Could not send pre-lock "
-                                "unlock");
+                                        "unlock");
+                }
 
                 if (__is_lock_grantable (pl_inode, lock)) {
                         gf_log (this->name, GF_LOG_TRACE,
-                                "%s (pid=%d) lk-owner:%"PRIu64" %"PRId64" - %"PRId64" => OK",
+                                "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => OK",
                                 lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
                                 lock->client_pid,
-                                lock->owner,
+                                lkowner_utoa (&lock->owner),
                                 lock->user_flock.l_start,
                                 lock->user_flock.l_len);
                         __insert_and_merge (pl_inode, lock);
                 } else if (can_block) {
                         gf_log (this->name, GF_LOG_TRACE,
-                                "%s (pid=%d) lk-owner:%"PRIu64" %"PRId64" - %"PRId64" => Blocked",
+                                "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => Blocked",
                                 lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
                                 lock->client_pid,
-                                lock->owner,
+                                lkowner_utoa (&lock->owner),
                                 lock->user_flock.l_start,
                                 lock->user_flock.l_len);
                         lock->blocked = 1;
@@ -1017,10 +1064,10 @@ pl_setlk (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
                         ret = -1;
                 } else {
                         gf_log (this->name, GF_LOG_TRACE,
-                                "%s (pid=%d) lk-owner:%"PRIu64" %"PRId64" - %"PRId64" => NOK",
+                                "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => NOK",
                                 lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
                                 lock->client_pid,
-                                lock->owner,
+                                lkowner_utoa (&lock->owner),
                                 lock->user_flock.l_start,
                                 lock->user_flock.l_len);
                         errno = EAGAIN;
@@ -1042,7 +1089,7 @@ pl_getlk (pl_inode_t *pl_inode, posix_lock_t *lock)
 {
         posix_lock_t *conf = NULL;
 
-        conf = first_overlap (pl_inode, lock);
+        conf = first_conflicting_overlap (pl_inode, lock);
 
         if (conf == NULL) {
                 lock->fl_type = F_UNLCK;
@@ -1051,3 +1098,4 @@ pl_getlk (pl_inode_t *pl_inode, posix_lock_t *lock)
 
         return conf;
 }
+
diff --git a/xlators/features/locks/src/common.h b/xlators/features/locks/src/common.h
index b13f44bc2..5ec630ee8 100644
--- a/xlators/features/locks/src/common.h
+++ b/xlators/features/locks/src/common.h
@@ -1,28 +1,41 @@
 /*
-  Copyright (c) 2006, 2007, 2008 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef __COMMON_H__
 #define __COMMON_H__
 
+#include "lkowner.h"
+/*dump locks format strings */
+#define RANGE_FMT               "type=%s, whence=%hd, start=%llu, len=%llu"
+#define ENTRY_FMT               "type=%s on basename=%s"
+#define DUMP_GEN_FMT            "pid = %llu, owner=%s, client=%p"
+#define GRNTD_AT                "granted at %s"
+#define BLKD_AT                 "blocked at %s"
+#define CONN_ID                 "connection-id=%s"
+#define DUMP_BLKD_FMT           DUMP_GEN_FMT", "CONN_ID", "BLKD_AT
+#define DUMP_GRNTD_FMT          DUMP_GEN_FMT", "CONN_ID", "GRNTD_AT
+#define DUMP_BLKD_GRNTD_FMT     DUMP_GEN_FMT", "CONN_ID", "BLKD_AT", "GRNTD_AT
+
+#define ENTRY_BLKD_FMT          ENTRY_FMT", "DUMP_BLKD_FMT
+#define ENTRY_GRNTD_FMT         ENTRY_FMT", "DUMP_GRNTD_FMT
+#define ENTRY_BLKD_GRNTD_FMT    ENTRY_FMT", "DUMP_BLKD_GRNTD_FMT
+
+#define RANGE_BLKD_FMT          RANGE_FMT", "DUMP_BLKD_FMT
+#define RANGE_GRNTD_FMT         RANGE_FMT", "DUMP_GRNTD_FMT
+#define RANGE_BLKD_GRNTD_FMT    RANGE_FMT", "DUMP_BLKD_GRNTD_FMT
+
+#define SET_FLOCK_PID(flock, lock) ((flock)->l_pid = lock->client_pid)
+
+
 posix_lock_t *
-new_posix_lock (struct gf_flock *flock, void *transport, pid_t client_pid,
-                uint64_t owner, fd_t *fd);
+new_posix_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
+                gf_lkowner_t *owner, fd_t *fd);
 
 pl_inode_t *
 pl_inode_get (xlator_t *this, inode_t *inode);
@@ -54,24 +67,29 @@ pl_dom_list_t *
 get_domain (pl_inode_t *pl_inode, const char *volume);
 
 void
-grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom);
+grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
+                           pl_dom_list_t *dom);
 
 void
 __delete_inode_lock (pl_inode_lock_t *lock);
 
 void
-__destroy_inode_lock (pl_inode_lock_t *lock);
+__pl_inodelk_unref (pl_inode_lock_t *lock);
 
 void
 grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
-                           pl_entry_lock_t *unlocked, pl_dom_list_t *dom);
+                           pl_dom_list_t *dom);
 
 void pl_update_refkeeper (xlator_t *this, inode_t *inode);
 
 int32_t
-get_inodelk_count (xlator_t *this, inode_t *inode);
+__get_inodelk_count (xlator_t *this, pl_inode_t *pl_inode, char *domname);
+int32_t
+get_inodelk_count (xlator_t *this, inode_t *inode, char *domname);
 
 int32_t
+__get_entrylk_count (xlator_t *this, pl_inode_t *pl_inode);
+int32_t
 get_entrylk_count (xlator_t *this, inode_t *inode);
 
 void pl_trace_in (xlator_t *this, call_frame_t *frame, fd_t *fd, loc_t *loc,
@@ -130,4 +148,11 @@ pl_verify_reservelk (xlator_t *this, pl_inode_t *pl_inode,
                      posix_lock_t *lock, int can_block);
 int
 pl_reserve_unlock (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *reqlock);
+
+uint32_t
+check_entrylk_on_basename (xlator_t *this, inode_t *parent, char *basename);
+
+void __pl_inodelk_unref (pl_inode_lock_t *lock);
+void __pl_entrylk_unref (pl_entry_lock_t *lock);
+
 #endif /* __COMMON_H__ */
diff --git a/xlators/features/locks/src/entrylk.c b/xlators/features/locks/src/entrylk.c
index 827b87d20..8496d9d8d 100644
--- a/xlators/features/locks/src/entrylk.c
+++ b/xlators/features/locks/src/entrylk.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006, 2007, 2008 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -33,31 +23,58 @@
 #include "locks.h"
 #include "common.h"
 
+
+void
+__pl_entrylk_unref (pl_entry_lock_t *lock)
+{
+        lock->ref--;
+        if (!lock->ref) {
+		GF_FREE ((char *)lock->basename);
+                GF_FREE (lock->connection_id);
+                GF_FREE (lock);
+        }
+}
+
+
+static void
+__pl_entrylk_ref (pl_entry_lock_t *lock)
+{
+        lock->ref++;
+}
+
+
 static pl_entry_lock_t *
 new_entrylk_lock (pl_inode_t *pinode, const char *basename, entrylk_type type,
-		  void *trans, pid_t client_pid, uint64_t owner, const char *volume)
-
+		  const char *domain, call_frame_t *frame, char *conn_id)
 {
-	pl_entry_lock_t *newlock = NULL;
+        pl_entry_lock_t *newlock = NULL;
 
-	newlock = GF_CALLOC (1, sizeof (pl_entry_lock_t),
+        newlock = GF_CALLOC (1, sizeof (pl_entry_lock_t),
                              gf_locks_mt_pl_entry_lock_t);
-	if (!newlock) {
-		goto out;
-	}
+        if (!newlock) {
+                goto out;
+        }
 
-	newlock->basename       = basename ? gf_strdup (basename) : NULL;
-	newlock->type           = type;
-	newlock->trans          = trans;
-	newlock->volume         = volume;
-        newlock->client_pid     = client_pid;
-        newlock->owner          = owner;
+        newlock->basename   = basename ? gf_strdup (basename) : NULL;
+        newlock->type       = type;
+        newlock->client     = frame->root->client;
+        newlock->client_pid = frame->root->pid;
+        newlock->volume     = domain;
+        newlock->owner      = frame->root->lk_owner;
+	newlock->frame      = frame;
+        newlock->this       = frame->this;
+
+        if (conn_id) {
+                newlock->connection_id = gf_strdup (conn_id);
+        }
 
-	INIT_LIST_HEAD (&newlock->domain_list);
-	INIT_LIST_HEAD (&newlock->blocked_locks);
+        INIT_LIST_HEAD (&newlock->domain_list);
+        INIT_LIST_HEAD (&newlock->blocked_locks);
+	INIT_LIST_HEAD (&newlock->client_list);
 
+	__pl_entrylk_ref (newlock);
 out:
-	return newlock;
+        return newlock;
 }
 
 
@@ -77,55 +94,55 @@ out:
 static int
 names_conflict (const char *n1, const char *n2)
 {
-	return all_names (n1) || all_names (n2) || !strcmp (n1, n2);
+        return all_names (n1) || all_names (n2) || !strcmp (n1, n2);
 }
 
 
-static int
+static inline int
 __same_entrylk_owner (pl_entry_lock_t *l1, pl_entry_lock_t *l2)
 {
 
-        return ((l1->owner == l2->owner) &&
-		(l1->trans  == l2->trans));
+        return (is_same_lkowner (&l1->owner, &l2->owner) &&
+                (l1->client  == l2->client));
 }
 
 
 /**
- * lock_grantable - is this lock grantable?
+ * entrylk_grantable - is this lock grantable?
  * @inode: inode in which to look
  * @basename: name we're trying to lock
  * @type: type of lock
  */
 static pl_entry_lock_t *
-__lock_grantable (pl_dom_list_t *dom, const char *basename, entrylk_type type)
+__entrylk_grantable (pl_dom_list_t *dom, pl_entry_lock_t *lock)
 {
-	pl_entry_lock_t *lock = NULL;
+        pl_entry_lock_t *tmp = NULL;
 
-	if (list_empty (&dom->entrylk_list))
-		return NULL;
+        if (list_empty (&dom->entrylk_list))
+                return NULL;
 
-	list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
-		if (names_conflict (lock->basename, basename))
-			return lock;
-	}
+        list_for_each_entry (tmp, &dom->entrylk_list, domain_list) {
+                if (names_conflict (tmp->basename, lock->basename))
+                        return tmp;
+        }
 
-	return NULL;
+        return NULL;
 }
 
 static pl_entry_lock_t *
-__blocked_lock_conflict (pl_dom_list_t *dom, const char *basename, entrylk_type type)
+__blocked_entrylk_conflict (pl_dom_list_t *dom, pl_entry_lock_t *lock)
 {
-	pl_entry_lock_t *lock = NULL;
+        pl_entry_lock_t *tmp = NULL;
 
-	if (list_empty (&dom->blocked_entrylks))
-		return NULL;
+        if (list_empty (&dom->blocked_entrylks))
+                return NULL;
 
-	list_for_each_entry (lock, &dom->blocked_entrylks, blocked_locks) {
-		if (names_conflict (lock->basename, basename))
-			return lock;
-	}
+        list_for_each_entry (tmp, &dom->blocked_entrylks, blocked_locks) {
+                if (names_conflict (tmp->basename, lock->basename))
+                        return lock;
+        }
 
-	return NULL;
+        return NULL;
 }
 
 static int
@@ -133,23 +150,23 @@ __owner_has_lock (pl_dom_list_t *dom, pl_entry_lock_t *newlock)
 {
         pl_entry_lock_t *lock = NULL;
 
-	list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
-		if (__same_entrylk_owner (lock, newlock))
-			return 1;
-	}
+        list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
+                if (__same_entrylk_owner (lock, newlock))
+                        return 1;
+        }
 
-	list_for_each_entry (lock, &dom->blocked_entrylks, blocked_locks) {
-		if (__same_entrylk_owner (lock, newlock))
-			return 1;
-	}
+        list_for_each_entry (lock, &dom->blocked_entrylks, blocked_locks) {
+                if (__same_entrylk_owner (lock, newlock))
+                        return 1;
+        }
 
-	return 0;
+        return 0;
 }
 
 static int
 names_equal (const char *n1, const char *n2)
 {
-	return (n1 == NULL && n2 == NULL) || (n1 && n2 && !strcmp (n1, n2));
+        return (n1 == NULL && n2 == NULL) || (n1 && n2 && !strcmp (n1, n2));
 }
 
 void
@@ -213,7 +230,7 @@ entrylk_trace_in (xlator_t *this, call_frame_t *frame, const char *domain,
         pl_print_lockee (pl_lockee, 256, fd, loc);
         pl_print_entrylk (pl_entrylk, 256, cmd, type, basename, domain);
 
-        gf_log (this->name, GF_LOG_NORMAL,
+        gf_log (this->name, GF_LOG_INFO,
                 "[REQUEST] Locker = {%s} Lockee = {%s} Lock = {%s}",
                 pl_locker, pl_lockee, pl_entrylk);
 }
@@ -240,7 +257,7 @@ entrylk_trace_out (xlator_t *this, call_frame_t *frame, const char *domain,
         pl_print_entrylk (pl_entrylk, 256, cmd, type, basename, domain);
         pl_print_verdict (verdict, 32, op_ret, op_errno);
 
-        gf_log (this->name, GF_LOG_NORMAL,
+        gf_log (this->name, GF_LOG_INFO,
                 "[%s] Locker = {%s} Lockee = {%s} Lock = {%s}",
                 verdict, pl_locker, pl_lockee, pl_entrylk);
 }
@@ -266,7 +283,7 @@ entrylk_trace_block (xlator_t *this, call_frame_t *frame, const char *volume,
         pl_print_lockee (pl_lockee, 256, fd, loc);
         pl_print_entrylk (pl_entrylk, 256, cmd, type, basename, volume);
 
-        gf_log (this->name, GF_LOG_NORMAL,
+        gf_log (this->name, GF_LOG_INFO,
                 "[BLOCKED] Locker = {%s} Lockee = {%s} Lock = {%s}",
                 pl_locker, pl_lockee, pl_entrylk);
 }
@@ -284,25 +301,25 @@ entrylk_trace_block (xlator_t *this, call_frame_t *frame, const char *volume,
 static pl_entry_lock_t *
 __find_most_matching_lock (pl_dom_list_t *dom, const char *basename)
 {
-	pl_entry_lock_t *lock;
-	pl_entry_lock_t *all = NULL;
-	pl_entry_lock_t *exact = NULL;
-
-	if (list_empty (&dom->entrylk_list))
-		return NULL;
-
-	list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
-		if (all_names (lock->basename))
-			all = lock;
-		else if (names_equal (lock->basename, basename))
-			exact = lock;
-	}
+        pl_entry_lock_t *lock;
+        pl_entry_lock_t *all = NULL;
+        pl_entry_lock_t *exact = NULL;
+
+        if (list_empty (&dom->entrylk_list))
+                return NULL;
+
+        list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
+                if (all_names (lock->basename))
+                        all = lock;
+                else if (names_equal (lock->basename, basename))
+                        exact = lock;
+        }
 
-	return (exact ? exact : all);
+        return (exact ? exact : all);
 }
 
 /**
- * __lock_name - lock a name in a directory
+ * __lock_entrylk - lock a name in a directory
  * @inode: inode for the directory in which to lock
  * @basename: name of the entry to lock
  *            if null, lock the entire directory
@@ -313,416 +330,318 @@ __find_most_matching_lock (pl_dom_list_t *dom, const char *basename)
  */
 
 int
-__lock_name (pl_inode_t *pinode, const char *basename, entrylk_type type,
-	     call_frame_t *frame, pl_dom_list_t *dom, xlator_t *this, int nonblock)
+__lock_entrylk (xlator_t *this, pl_inode_t *pinode, pl_entry_lock_t *lock,
+		int nonblock, pl_dom_list_t *dom)
 {
-	pl_entry_lock_t *lock       = NULL;
-	pl_entry_lock_t *conf       = NULL;
-	void            *trans      = NULL;
-        pid_t            client_pid = 0;
-        uint64_t         owner      = 0;
-
-	int ret = -EINVAL;
-
-	trans = frame->root->trans;
-        client_pid = frame->root->pid;
-        owner      = frame->root->lk_owner;
-
-	lock = new_entrylk_lock (pinode, basename, type, trans, client_pid, owner, dom->domain);
-	if (!lock) {
-		ret = -ENOMEM;
-		goto out;
-	}
+        pl_entry_lock_t *conf = NULL;
+        int              ret  = -EAGAIN;
 
-        lock->frame   = frame;
-        lock->this    = this;
-        lock->trans   = trans;
-
-	conf = __lock_grantable (dom, basename, type);
-	if (conf) {
-		ret = -EAGAIN;
-		if (nonblock){
-			if (lock->basename)
-				GF_FREE ((char *)lock->basename);
-                        GF_FREE (lock);
+        conf = __entrylk_grantable (dom, lock);
+        if (conf) {
+                ret = -EAGAIN;
+                if (nonblock)
                         goto out;
 
-                }
-
-		list_add_tail (&lock->blocked_locks, &dom->blocked_entrylks);
+                gettimeofday (&lock->blkd_time, NULL);
+                list_add_tail (&lock->blocked_locks, &dom->blocked_entrylks);
 
-		gf_log (this->name, GF_LOG_TRACE,
-			"Blocking lock: {pinode=%p, basename=%s}",
-			pinode, basename);
+                gf_log (this->name, GF_LOG_TRACE,
+                        "Blocking lock: {pinode=%p, basename=%s}",
+                        pinode, lock->basename);
 
-		goto out;
-	}
+                goto out;
+        }
 
-        if ( __blocked_lock_conflict (dom, basename, type) && !(__owner_has_lock (dom, lock))) {
+        if (__blocked_entrylk_conflict (dom, lock) && !(__owner_has_lock (dom, lock))) {
                 ret = -EAGAIN;
-                if (nonblock) {
-			if (lock->basename)
-				GF_FREE ((char *) lock->basename);
-                        GF_FREE (lock);
+                if (nonblock)
                         goto out;
 
-                }
-                lock->frame     = frame;
-                lock->this      = this;
-
+                gettimeofday (&lock->blkd_time, NULL);
                 list_add_tail (&lock->blocked_locks, &dom->blocked_entrylks);
 
-                gf_log (this->name, GF_LOG_TRACE,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "Lock is grantable, but blocking to prevent starvation");
-		gf_log (this->name, GF_LOG_TRACE,
-			"Blocking lock: {pinode=%p, basename=%s}",
-			pinode, basename);
+                gf_log (this->name, GF_LOG_TRACE,
+                        "Blocking lock: {pinode=%p, basename=%s}",
+                        pinode, lock->basename);
 
-                ret = -EAGAIN;
-		goto out;
+                goto out;
         }
-        switch (type) {
-
-	case ENTRYLK_WRLCK:
-		list_add_tail (&lock->domain_list, &dom->entrylk_list);
-		break;
-
-        default:
 
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Invalid type for entrylk specified: %d", type);
-                ret = -EINVAL;
-                goto out;
-	}
+        __pl_entrylk_ref (lock);
+        gettimeofday (&lock->granted_time, NULL);
+        list_add (&lock->domain_list, &dom->entrylk_list);
 
-	ret = 0;
+        ret = 0;
 out:
-	return ret;
+        return ret;
 }
 
 /**
- * __unlock_name - unlock a name in a directory
+ * __unlock_entrylk - unlock a name in a directory
  * @inode: inode for the directory to unlock in
  * @basename: name of the entry to unlock
  *            if null, unlock the entire directory
  */
 
 pl_entry_lock_t *
-__unlock_name (pl_dom_list_t *dom, const char *basename, entrylk_type type)
+__unlock_entrylk (pl_dom_list_t *dom, pl_entry_lock_t *lock)
 {
-	pl_entry_lock_t *lock = NULL;
-	pl_entry_lock_t *ret_lock = NULL;
+        pl_entry_lock_t *tmp = NULL;
+        pl_entry_lock_t *ret_lock = NULL;
 
-	lock = __find_most_matching_lock (dom, basename);
+        tmp = __find_most_matching_lock (dom, lock->basename);
 
-	if (!lock) {
-		gf_log ("locks", GF_LOG_DEBUG,
-			"unlock on %s (type=ENTRYLK_WRLCK) attempted but no matching lock found",
-			basename);
-		goto out;
-	}
+        if (!tmp) {
+                gf_log ("locks", GF_LOG_ERROR,
+                        "unlock on %s (type=ENTRYLK_WRLCK) attempted but no matching lock found",
+                        lock->basename);
+                goto out;
+        }
 
-	if (names_equal (lock->basename, basename)
-	    && lock->type == type) {
+        if (names_equal (tmp->basename, lock->basename)
+            && tmp->type == lock->type) {
 
-		if (type == ENTRYLK_WRLCK) {
-			list_del_init (&lock->domain_list);
-			ret_lock = lock;
-		}
-	} else {
-		gf_log ("locks", GF_LOG_DEBUG,
-			"Unlock for a non-existing lock!");
-		goto out;
-	}
+		list_del_init (&tmp->domain_list);
+		ret_lock = tmp;
+
+        } else {
+                gf_log ("locks", GF_LOG_ERROR,
+                        "Unlock on %s for a non-existing lock!", lock->basename);
+                goto out;
+        }
 
 out:
-	return ret_lock;
+        return ret_lock;
 }
 
+uint32_t
+check_entrylk_on_basename (xlator_t *this, inode_t *parent, char *basename)
+{
+        uint32_t        entrylk = 0;
+        pl_inode_t      *pinode = 0;
+        pl_dom_list_t   *dom = NULL;
+        pl_entry_lock_t *conf       = NULL;
+
+        pinode = pl_inode_get (this, parent);
+        if (!pinode)
+                goto out;
+        pthread_mutex_lock (&pinode->mutex);
+        {
+                list_for_each_entry (dom, &pinode->dom_list, inode_list) {
+                        conf = __find_most_matching_lock (dom, basename);
+                        if (conf && conf->basename) {
+                                entrylk = 1;
+                                break;
+                        }
+                }
+        }
+        pthread_mutex_unlock (&pinode->mutex);
+
+out:
+        return entrylk;
+}
 
 void
 __grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
-			     pl_dom_list_t *dom, struct list_head *granted)
+                             pl_dom_list_t *dom, struct list_head *granted)
 {
-	int              bl_ret = 0;
-	pl_entry_lock_t *bl   = NULL;
-	pl_entry_lock_t *tmp  = NULL;
+        int              bl_ret = 0;
+        pl_entry_lock_t *bl   = NULL;
+        pl_entry_lock_t *tmp  = NULL;
 
         struct list_head blocked_list;
 
         INIT_LIST_HEAD (&blocked_list);
         list_splice_init (&dom->blocked_entrylks, &blocked_list);
-        
-	list_for_each_entry_safe (bl, tmp, &blocked_list,
-				  blocked_locks) {
 
-		list_del_init (&bl->blocked_locks);
+        list_for_each_entry_safe (bl, tmp, &blocked_list, blocked_locks) {
 
+                list_del_init (&bl->blocked_locks);
 
-		gf_log ("locks", GF_LOG_TRACE,
-			"Trying to unblock: {pinode=%p, basename=%s}",
-			pl_inode, bl->basename);
+                bl_ret = __lock_entrylk (bl->this, pl_inode, bl, 0, dom);
 
-		bl_ret = __lock_name (pl_inode, bl->basename, bl->type,
-				      bl->frame, dom, bl->this, 0);
-
-		if (bl_ret == 0) {
-			list_add (&bl->blocked_locks, granted);
-		} else {
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                "should never happen");
-			if (bl->basename)
-				GF_FREE ((char *)bl->basename);
-			GF_FREE (bl);
-		}
-	}
-	return;
+                if (bl_ret == 0) {
+                        list_add (&bl->blocked_locks, granted);
+                }
+        }
+        return;
 }
 
 /* Grants locks if possible which are blocked on a lock */
 void
 grant_blocked_entry_locks (xlator_t *this, pl_inode_t *pl_inode,
-			   pl_entry_lock_t *unlocked, pl_dom_list_t *dom)
+			   pl_dom_list_t *dom)
 {
-	struct list_head  granted_list;
-	pl_entry_lock_t  *tmp = NULL;
-	pl_entry_lock_t  *lock = NULL;
-
-	INIT_LIST_HEAD (&granted_list);
+        struct list_head  granted_list;
+        pl_entry_lock_t  *tmp = NULL;
+        pl_entry_lock_t  *lock = NULL;
 
-	pthread_mutex_lock (&pl_inode->mutex);
-	{
-		__grant_blocked_entry_locks (this, pl_inode, dom, &granted_list);
-	}
-	pthread_mutex_unlock (&pl_inode->mutex);
+        INIT_LIST_HEAD (&granted_list);
 
-	list_for_each_entry_safe (lock, tmp, &granted_list, blocked_locks) {
-		list_del_init (&lock->blocked_locks);
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                __grant_blocked_entry_locks (this, pl_inode, dom,
+                                             &granted_list);
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
 
+        list_for_each_entry_safe (lock, tmp, &granted_list, blocked_locks) {
                 entrylk_trace_out (this, lock->frame, NULL, NULL, NULL,
                                    lock->basename, ENTRYLK_LOCK, lock->type,
                                    0, 0);
 
-		STACK_UNWIND_STRICT (entrylk, lock->frame, 0, 0);
-
+                STACK_UNWIND_STRICT (entrylk, lock->frame, 0, 0, NULL);
+		lock->frame = NULL;
 	}
 
-	GF_FREE ((char *)unlocked->basename);
-	GF_FREE (unlocked);
-
-	return;
-}
-
-/**
- * release_entry_locks_for_transport: release all entry locks from this
- * transport for this loc_t
- */
-
-static int
-release_entry_locks_for_transport (xlator_t *this, pl_inode_t *pinode,
-				   pl_dom_list_t *dom, void *trans)
-{
-	pl_entry_lock_t  *lock = NULL;
-	pl_entry_lock_t  *tmp = NULL;
-	struct list_head  granted;
-        struct list_head  released;
-
-	INIT_LIST_HEAD (&granted);
-        INIT_LIST_HEAD (&released);
-
-	pthread_mutex_lock (&pinode->mutex);
-	{
-                list_for_each_entry_safe (lock, tmp, &dom->blocked_entrylks,
-                                          blocked_locks) {
-                        if (lock->trans != trans)
-                                continue;
-
-                        list_del_init (&lock->blocked_locks);
-
-			gf_log (this->name, GF_LOG_TRACE,
-                                "releasing lock on  held by "
-                                "{transport=%p}",trans);
-
-                        list_add (&lock->blocked_locks, &released);
-
-                }
-
-		list_for_each_entry_safe (lock, tmp, &dom->entrylk_list,
-					  domain_list) {
-			if (lock->trans != trans)
-				continue;
-
-			list_del_init (&lock->domain_list);
-
-			gf_log (this->name, GF_LOG_TRACE,
-                                "releasing lock on  held by "
-                                "{transport=%p}",trans);
-
-			GF_FREE ((char *)lock->basename);
-			GF_FREE (lock);
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+		list_for_each_entry_safe (lock, tmp, &granted_list, blocked_locks) {
+			list_del_init (&lock->blocked_locks);
+			__pl_entrylk_unref (lock);
 		}
-
-		__grant_blocked_entry_locks (this, pinode, dom, &granted);
-
-	}
-
-	pthread_mutex_unlock (&pinode->mutex);
-
-        list_for_each_entry_safe (lock, tmp, &released, blocked_locks) {
-                list_del_init (&lock->blocked_locks);
-
-                STACK_UNWIND_STRICT (entrylk, lock->frame, -1, EAGAIN);
-
-                if (lock->basename)
-                        GF_FREE ((char *)lock->basename);
-                GF_FREE (lock);
-
-        }
-
-	list_for_each_entry_safe (lock, tmp, &granted, blocked_locks) {
-		list_del_init (&lock->blocked_locks);
-
-		STACK_UNWIND_STRICT (entrylk, lock->frame, 0, 0);
-
-		if (lock->basename)
-			GF_FREE ((char *)lock->basename);
-		GF_FREE (lock);
 	}
+        pthread_mutex_unlock (&pl_inode->mutex);
 
-	return 0;
+        return;
 }
 
+
 /* Common entrylk code called by pl_entrylk and pl_fentrylk */
 int
 pl_common_entrylk (call_frame_t *frame, xlator_t *this,
                    const char *volume, inode_t *inode, const char *basename,
-                   entrylk_cmd cmd, entrylk_type type, loc_t *loc, fd_t *fd)
+                   entrylk_cmd cmd, entrylk_type type, loc_t *loc, fd_t *fd,
+                   dict_t *xdata)
+
 {
-        uint64_t owner    = 0;
-	int32_t  op_ret   = -1;
-	int32_t  op_errno = 0;
-
-	void *        transport = NULL;
-	pid_t         pid       = -1;
-
-	pl_inode_t *     pinode   = NULL;
-	int              ret      = -1;
-	pl_entry_lock_t *unlocked = NULL;
-	char             unwind   = 1;
-
-	pl_dom_list_t	  *dom = NULL;
-
-	pinode = pl_inode_get (this, inode);
-	if (!pinode) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		op_errno = ENOMEM;
-		goto out;
-	}
+        int32_t          op_ret   = -1;
+        int32_t          op_errno = 0;
+        int              ret      = -1;
+        char             unwind   = 1;
+        GF_UNUSED int    dict_ret = -1;
+        pl_inode_t      *pinode   = NULL;
+        pl_entry_lock_t *reqlock  = NULL;
+        pl_entry_lock_t *unlocked = NULL;
+        pl_dom_list_t   *dom      = NULL;
+        char            *conn_id  = NULL;
+        pl_ctx_t        *ctx      = NULL;
+	int              nonblock = 0;
+
+        if (xdata)
+                dict_ret = dict_get_str (xdata, "connection-id", &conn_id);
+
+        pinode = pl_inode_get (this, inode);
+        if (!pinode) {
+                op_errno = ENOMEM;
+                goto out;
+        }
 
-	dom = get_domain (pinode, volume);
-	if (!dom){
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory");
-		op_errno = ENOMEM;
-		goto out;
+	if (frame->root->client) {
+		ctx = pl_ctx_get (frame->root->client, this);
+		if (!ctx) {
+			op_errno = ENOMEM;
+			gf_log (this->name, GF_LOG_INFO, "pl_ctx_get() failed");
+			goto unwind;
+		}
 	}
 
-        entrylk_trace_in (this, frame, volume, fd, loc, basename, cmd, type);
-
-	pid       = frame->root->pid;
-        owner     = frame->root->lk_owner;
-	transport = frame->root->trans;
-
-	if (owner == 0) {
-		/*
-                  this is a special case that means release
-                  all locks from this transport
-		*/
-
-		gf_log (this->name, GF_LOG_TRACE,
-			"Releasing locks for transport %p", transport);
+        dom = get_domain (pinode, volume);
+        if (!dom){
+                op_errno = ENOMEM;
+                goto out;
+        }
 
-		release_entry_locks_for_transport (this, pinode, dom, transport);
-		op_ret = 0;
+        entrylk_trace_in (this, frame, volume, fd, loc, basename, cmd, type);
 
-		goto out;
-	}
+        reqlock = new_entrylk_lock (pinode, basename, type, dom->domain, frame,
+				    conn_id);
+        if (!reqlock) {
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto unwind;
+        }
 
-	switch (cmd) {
+        switch (cmd) {
+        case ENTRYLK_LOCK_NB:
+		nonblock = 1;
+		/* fall through */
 	case ENTRYLK_LOCK:
-		pthread_mutex_lock (&pinode->mutex);
-		{
-			ret = __lock_name (pinode, basename, type,
-					   frame, dom, this, 0);
-		}
-		pthread_mutex_unlock (&pinode->mutex);
-
-                op_errno = -ret;
-		if (ret < 0) {
-                        if (ret == -EAGAIN)
-                                unwind = 0;
-                        else
-                                unwind = 1;
-			goto out;
-		} else {
-                        op_ret = 0;
-                        op_errno = 0;
-                        unwind = 1;
-                        goto out;
+		if (ctx)
+			pthread_mutex_lock (&ctx->lock);
+                pthread_mutex_lock (&pinode->mutex);
+                {
+			reqlock->pinode = pinode;
+
+                        ret = __lock_entrylk (this, pinode, reqlock, nonblock, dom);
+			if (ret == 0) {
+				reqlock->frame = NULL;
+				op_ret = 0;
+			} else {
+				op_errno = -ret;
+			}
+
+			if (ctx && (!ret || !nonblock))
+				list_add (&reqlock->client_list,
+					  &ctx->entrylk_lockers);
+
+			if (ret == -EAGAIN && !nonblock) {
+				/* blocked */
+				unwind = 0;
+			} else {
+				__pl_entrylk_unref (reqlock);
+			}
                 }
-
+                pthread_mutex_unlock (&pinode->mutex);
+		if (ctx)
+			pthread_mutex_unlock (&ctx->lock);
 		break;
 
-	case ENTRYLK_LOCK_NB:
-		pthread_mutex_lock (&pinode->mutex);
-		{
-			ret = __lock_name (pinode, basename, type,
-					   frame, dom, this, 1);
-		}
-		pthread_mutex_unlock (&pinode->mutex);
-
-		if (ret < 0)
-			op_errno = -ret;
-
-                unwind = 1;
-                goto out;
-		break;
-
-	case ENTRYLK_UNLOCK:
-		pthread_mutex_lock (&pinode->mutex);
-		{
-                        unlocked = __unlock_name (dom, basename, type);
-		}
-		pthread_mutex_unlock (&pinode->mutex);
+        case ENTRYLK_UNLOCK:
+		if (ctx)
+			pthread_mutex_lock (&ctx->lock);
+                pthread_mutex_lock (&pinode->mutex);
+                {
+                        unlocked = __unlock_entrylk (dom, reqlock);
+			if (unlocked) {
+				list_del_init (&unlocked->client_list);
+				__pl_entrylk_unref (unlocked);
+				op_ret = 0;
+			} else {
+				op_errno = EINVAL;
+			}
+			__pl_entrylk_unref (reqlock);
+                }
+                pthread_mutex_unlock (&pinode->mutex);
+		if (ctx)
+			pthread_mutex_unlock (&ctx->lock);
 
-		if (unlocked)
-			grant_blocked_entry_locks (this, pinode, unlocked, dom);
+		grant_blocked_entry_locks (this, pinode, dom);
 
-		break;
+                break;
 
-	default:
-		gf_log (this->name, GF_LOG_ERROR,
-			"Unexpected case in entrylk (cmd=%d). Please file"
+        default:
+                gf_log (this->name, GF_LOG_ERROR,
+                        "Unexpected case in entrylk (cmd=%d). Please file"
                         "a bug report at http://bugs.gluster.com", cmd);
-		goto out;
-	}
-
-	op_ret = 0;
+                goto out;
+        }
 out:
         pl_update_refkeeper (this, inode);
-	if (unwind) {
+
+        if (unwind) {
                 entrylk_trace_out (this, frame, volume, fd, loc, basename,
                                    cmd, type, op_ret, op_errno);
-
-		STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno);
-	} else {
+unwind:
+                STACK_UNWIND_STRICT (entrylk, frame, op_ret, op_errno, NULL);
+        } else {
                 entrylk_trace_block (this, frame, volume, fd, loc, basename,
                                      cmd, type);
         }
 
-
-	return 0;
+        return 0;
 }
 
 /**
@@ -733,11 +652,11 @@ out:
 
 int
 pl_entrylk (call_frame_t *frame, xlator_t *this,
-	    const char *volume, loc_t *loc, const char *basename,
-	    entrylk_cmd cmd, entrylk_type type)
+            const char *volume, loc_t *loc, const char *basename,
+            entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
 {
-
-        pl_common_entrylk (frame, this, volume, loc->inode, basename, cmd, type, loc, NULL);
+        pl_common_entrylk (frame, this, volume, loc->inode, basename, cmd,
+                           type, loc, NULL, xdata);
 
         return 0;
 }
@@ -752,16 +671,142 @@ pl_entrylk (call_frame_t *frame, xlator_t *this,
 int
 pl_fentrylk (call_frame_t *frame, xlator_t *this,
              const char *volume, fd_t *fd, const char *basename,
-             entrylk_cmd cmd, entrylk_type type)
+             entrylk_cmd cmd, entrylk_type type, dict_t *xdata)
+{
+        pl_common_entrylk (frame, this, volume, fd->inode, basename, cmd,
+                           type, NULL, fd, xdata);
+
+        return 0;
+}
+
+
+static void
+pl_entrylk_log_cleanup (pl_entry_lock_t *lock)
+{
+	pl_inode_t *pinode = NULL;
+        char *path = NULL;
+        char *file = NULL;
+
+	pinode = lock->pinode;
+
+	inode_path (pinode->refkeeper, NULL, &path);
+
+	if (path)
+		file = path;
+	else
+		file = uuid_utoa (pinode->refkeeper->gfid);
+
+	gf_log (THIS->name, GF_LOG_WARNING,
+		"releasing lock on %s held by "
+		"{client=%p, pid=%"PRId64" lk-owner=%s}",
+		file, lock->client, (uint64_t) lock->client_pid,
+		lkowner_utoa (&lock->owner));
+	GF_FREE (path);
+}
+
+
+/* Release all entrylks from this client */
+int
+pl_entrylk_client_cleanup (xlator_t *this, pl_ctx_t *ctx)
 {
+        pl_entry_lock_t *tmp = NULL;
+        pl_entry_lock_t *l = NULL;
+	pl_dom_list_t *dom = NULL;
+        pl_inode_t *pinode = NULL;
+
+        struct list_head released;
+        struct list_head unwind;
+
+        INIT_LIST_HEAD (&released);
+        INIT_LIST_HEAD (&unwind);
+
+	pthread_mutex_lock (&ctx->lock);
+        {
+                list_for_each_entry_safe (l, tmp, &ctx->entrylk_lockers,
+					  client_list) {
+                        list_del_init (&l->client_list);
+
+			pl_entrylk_log_cleanup (l);
+
+			pinode = l->pinode;
+
+			pthread_mutex_lock (&pinode->mutex);
+			{
+                        /* If the entrylk object is part of granted list but not
+                         * blocked list, then perform the following actions:
+                         * i.   delete the object from granted list;
+                         * ii.  grant other locks (from other clients) that may
+                         *      have been blocked on this entrylk; and
+                         * iii. unref the object.
+                         *
+                         * If the entrylk object (L1) is part of both granted
+                         * and blocked lists, then this means that a parallel
+                         * unlock on another entrylk (L2 say) may have 'granted'
+                         * L1 and added it to 'granted' list in
+                         * __grant_blocked_entry_locks() (although using the
+                         * 'blocked_locks' member). In that case, the cleanup
+                         * codepath must try and grant other overlapping
+                         * blocked entrylks from other clients, now that L1 is
+                         * out of their way and then unref L1 in the end, and
+                         * leave it to the other thread (the one executing
+                         * unlock codepath) to unwind L1's frame, delete it from
+                         * blocked_locks list, and perform the last unref on L1.
+                         *
+                         * If the entrylk object (L1) is part of blocked list
+                         * only, the cleanup code path must:
+                         * i.   delete it from the blocked_locks list inside
+                         *      this critical section,
+                         * ii.  unwind its frame with EAGAIN,
+                         * iii. try and grant blocked entry locks from other
+                         *      clients that were otherwise grantable, but were
+                         *      blocked to avoid leaving L1 to starve forever.
+                         * iv.  unref the object.
+                         */
+                                if (!list_empty (&l->domain_list)) {
+                                        list_del_init (&l->domain_list);
+                                        list_add_tail (&l->client_list,
+                                                       &released);
+                                } else {
+                                        list_del_init (&l->blocked_locks);
+                                        list_add_tail (&l->client_list,
+                                                       &unwind);
+                                }
+                        }
+			pthread_mutex_unlock (&pinode->mutex);
+                }
+	}
+        pthread_mutex_unlock (&ctx->lock);
+
+        list_for_each_entry_safe (l, tmp, &unwind, client_list) {
+                list_del_init (&l->client_list);
+
+		if (l->frame)
+			STACK_UNWIND_STRICT (entrylk, l->frame, -1, EAGAIN,
+					     NULL);
+                list_add_tail (&l->client_list, &released);
+        }
+
+        list_for_each_entry_safe (l, tmp, &released, client_list) {
+                list_del_init (&l->client_list);
+
+		pinode = l->pinode;
+
+		dom = get_domain (pinode, l->volume);
 
-        pl_common_entrylk (frame, this, volume, fd->inode, basename, cmd, type, NULL, fd);
+		grant_blocked_entry_locks (this, pinode, dom);
+
+		pthread_mutex_lock (&pinode->mutex);
+		{
+			__pl_entrylk_unref (l);
+		}
+		pthread_mutex_unlock (&pinode->mutex);
+        }
 
         return 0;
 }
 
 
-static int32_t
+int32_t
 __get_entrylk_count (xlator_t *this, pl_inode_t *pl_inode)
 {
         int32_t            count = 0;
@@ -770,24 +815,10 @@ __get_entrylk_count (xlator_t *this, pl_inode_t *pl_inode)
 
         list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
                 list_for_each_entry (lock, &dom->entrylk_list, domain_list) {
-
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                " XATTR DEBUG"
-                                " domain: %s  %s on %s state = Active",
-                                dom->domain,
-                                lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" : 
-                                "ENTRYLK_WRLCK", lock->basename);
                         count++;
                 }
 
                 list_for_each_entry (lock, &dom->blocked_entrylks, blocked_locks) {
-
-                        gf_log (this->name, GF_LOG_DEBUG,
-                                " XATTR DEBUG"
-                                " domain: %s  %s on %s state = Blocked",
-                                dom->domain,
-                                lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" : 
-                                "ENTRYLK_WRLCK", lock->basename);
                         count++;
                 }
 
diff --git a/xlators/features/locks/src/inodelk.c b/xlators/features/locks/src/inodelk.c
index bbbd03398..c76cb7f91 100644
--- a/xlators/features/locks/src/inodelk.c
+++ b/xlators/features/locks/src/inodelk.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006, 2007, 2008 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -33,26 +23,36 @@
 #include "locks.h"
 #include "common.h"
 
-void
+inline void
 __delete_inode_lock (pl_inode_lock_t *lock)
 {
-	list_del (&lock->list);
+        list_del_init (&lock->list);
 }
 
-void
-__destroy_inode_lock (pl_inode_lock_t *lock)
+static inline void
+__pl_inodelk_ref (pl_inode_lock_t *lock)
+{
+        lock->ref++;
+}
+
+inline void
+__pl_inodelk_unref (pl_inode_lock_t *lock)
 {
-	GF_FREE (lock);
+        lock->ref--;
+        if (!lock->ref) {
+                GF_FREE (lock->connection_id);
+                GF_FREE (lock);
+        }
 }
 
 /* Check if 2 inodelks are conflicting on type. Only 2 shared locks don't conflict */
-static int
+static inline int
 inodelk_type_conflict (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
 {
-	if (l2->fl_type == F_WRLCK || l1->fl_type == F_WRLCK)
-		return 1;
+        if (l2->fl_type == F_WRLCK || l1->fl_type == F_WRLCK)
+                return 1;
 
-	return 0;
+        return 0;
 }
 
 void
@@ -115,15 +115,16 @@ pl_print_inodelk (char *str, int size, int cmd, struct gf_flock *flock, const ch
 static int
 inodelk_overlap (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
 {
-	return ((l1->fl_end >= l2->fl_start) &&
-		(l2->fl_end >= l1->fl_start));
+        return ((l1->fl_end >= l2->fl_start) &&
+                (l2->fl_end >= l1->fl_start));
 }
 
 /* Returns true if the 2 inodelks have the same owner */
-static int same_inodelk_owner (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
+static inline int
+same_inodelk_owner (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
 {
-	return ((l1->owner == l2->owner) &&
-                (l1->transport  == l2->transport));
+        return (is_same_lkowner (&l1->owner, &l2->owner) &&
+                (l1->client == l2->client));
 }
 
 /* Returns true if the 2 inodelks conflict with each other */
@@ -138,18 +139,19 @@ inodelk_conflict (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
 static pl_inode_lock_t *
 __inodelk_grantable (pl_dom_list_t *dom, pl_inode_lock_t *lock)
 {
-	pl_inode_lock_t *l = NULL;
-	pl_inode_lock_t *ret = NULL;
-	if (list_empty (&dom->inodelk_list))
-		goto out;
-	list_for_each_entry (l, &dom->inodelk_list, list){
-		if (inodelk_conflict (lock, l)) {
-			ret = l;
-			goto out;
-		}
-	}
+        pl_inode_lock_t *l = NULL;
+        pl_inode_lock_t *ret = NULL;
+        if (list_empty (&dom->inodelk_list))
+                goto out;
+        list_for_each_entry (l, &dom->inodelk_list, list){
+                if (inodelk_conflict (lock, l) &&
+                    !same_inodelk_owner (lock, l)) {
+                        ret = l;
+                        goto out;
+                }
+        }
 out:
-	return ret;
+        return ret;
 }
 
 static pl_inode_lock_t *
@@ -158,18 +160,18 @@ __blocked_lock_conflict (pl_dom_list_t *dom, pl_inode_lock_t *lock)
         pl_inode_lock_t *l   = NULL;
         pl_inode_lock_t *ret = NULL;
 
-	if (list_empty (&dom->blocked_entrylks))
-		return NULL;
+        if (list_empty (&dom->blocked_inodelks))
+                return NULL;
 
-	list_for_each_entry (l, &dom->blocked_inodelks, blocked_locks) {
-		if (inodelk_conflict (lock, l)) {
-			ret = l;
-			goto out;
+        list_for_each_entry (l, &dom->blocked_inodelks, blocked_locks) {
+                if (inodelk_conflict (lock, l)) {
+                        ret = l;
+                        goto out;
                 }
-	}
+        }
 
 out:
-	return ret;
+        return ret;
 }
 
 static int
@@ -177,17 +179,17 @@ __owner_has_lock (pl_dom_list_t *dom, pl_inode_lock_t *newlock)
 {
         pl_inode_lock_t *lock = NULL;
 
-	list_for_each_entry (lock, &dom->entrylk_list, list) {
-		if (same_inodelk_owner (lock, newlock))
-			return 1;
-	}
+        list_for_each_entry (lock, &dom->inodelk_list, list) {
+                if (same_inodelk_owner (lock, newlock))
+                        return 1;
+        }
 
-	list_for_each_entry (lock, &dom->blocked_entrylks, blocked_locks) {
-		if (same_inodelk_owner (lock, newlock))
-			return 1;
-	}
+        list_for_each_entry (lock, &dom->blocked_inodelks, blocked_locks) {
+                if (same_inodelk_owner (lock, newlock))
+                        return 1;
+        }
 
-	return 0;
+        return 0;
 }
 
 
@@ -196,81 +198,85 @@ __owner_has_lock (pl_dom_list_t *dom, pl_inode_lock_t *newlock)
  */
 static int
 __lock_inodelk (xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
-		int can_block,  pl_dom_list_t *dom)
+                int can_block,  pl_dom_list_t *dom)
 {
-	pl_inode_lock_t *conf = NULL;
-	int ret = -EINVAL;
+        pl_inode_lock_t *conf = NULL;
+        int ret = -EINVAL;
 
-	conf = __inodelk_grantable (dom, lock);
-	if (conf){
-		ret = -EAGAIN;
-		if (can_block == 0)
-			goto out;
+        conf = __inodelk_grantable (dom, lock);
+        if (conf) {
+                ret = -EAGAIN;
+                if (can_block == 0)
+                        goto out;
 
-		list_add_tail (&lock->blocked_locks, &dom->blocked_inodelks);
+                gettimeofday (&lock->blkd_time, NULL);
+                list_add_tail (&lock->blocked_locks, &dom->blocked_inodelks);
 
                 gf_log (this->name, GF_LOG_TRACE,
-                        "%s (pid=%d) lk-owner:%"PRIu64" %"PRId64" - %"PRId64" => Blocked",
+                        "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => Blocked",
                         lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
                         lock->client_pid,
-                        lock->owner,
+                        lkowner_utoa (&lock->owner),
                         lock->user_flock.l_start,
                         lock->user_flock.l_len);
 
 
-		goto out;
-	}
+                goto out;
+        }
 
         if (__blocked_lock_conflict (dom, lock) && !(__owner_has_lock (dom, lock))) {
                 ret = -EAGAIN;
                 if (can_block == 0)
                         goto out;
 
+                gettimeofday (&lock->blkd_time, NULL);
                 list_add_tail (&lock->blocked_locks, &dom->blocked_inodelks);
 
-                gf_log (this->name, GF_LOG_TRACE,
+                gf_log (this->name, GF_LOG_DEBUG,
                         "Lock is grantable, but blocking to prevent starvation");
-		gf_log (this->name, GF_LOG_TRACE,
-                        "%s (pid=%d) (lk-owner=%"PRIu64") %"PRId64" - %"PRId64" => Blocked",
+                gf_log (this->name, GF_LOG_TRACE,
+                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => Blocked",
                         lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
                         lock->client_pid,
-                        lock->owner,
+                        lkowner_utoa (&lock->owner),
                         lock->user_flock.l_start,
                         lock->user_flock.l_len);
 
 
-		goto out;
+                goto out;
         }
-	list_add (&lock->list, &dom->inodelk_list);
+        __pl_inodelk_ref (lock);
+        gettimeofday (&lock->granted_time, NULL);
+        list_add (&lock->list, &dom->inodelk_list);
 
-	ret = 0;
+        ret = 0;
 
 out:
-	return ret;
+        return ret;
 }
 
 /* Return true if the two inodelks have exactly same lock boundaries */
 static int
 inodelks_equal (pl_inode_lock_t *l1, pl_inode_lock_t *l2)
 {
-	if ((l1->fl_start == l2->fl_start) &&
-	    (l1->fl_end == l2->fl_end))
-		return 1;
+        if ((l1->fl_start == l2->fl_start) &&
+            (l1->fl_end == l2->fl_end))
+                return 1;
 
-	return 0;
+        return 0;
 }
 
 
 static pl_inode_lock_t *
 find_matching_inodelk (pl_inode_lock_t *lock, pl_dom_list_t *dom)
 {
-	pl_inode_lock_t *l = NULL;
-	list_for_each_entry (l, &dom->inodelk_list, list) {
-		if (inodelks_equal (l, lock) &&
+        pl_inode_lock_t *l = NULL;
+        list_for_each_entry (l, &dom->inodelk_list, list) {
+                if (inodelks_equal (l, lock) &&
                     same_inodelk_owner (l, lock))
-			return l;
-	}
-	return NULL;
+                        return l;
+        }
+        return NULL;
 }
 
 /* Set F_UNLCK removes a lock which has the exact same lock boundaries
@@ -280,358 +286,462 @@ static pl_inode_lock_t *
 __inode_unlock_lock (xlator_t *this, pl_inode_lock_t *lock, pl_dom_list_t *dom)
 {
 
-	pl_inode_lock_t *conf = NULL;
+        pl_inode_lock_t *conf = NULL;
 
-	conf = find_matching_inodelk (lock, dom);
-	if (!conf) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        " Matching lock not found for unlock");
-		goto out;
+        conf = find_matching_inodelk (lock, dom);
+        if (!conf) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        " Matching lock not found for unlock %llu-%llu, by %s "
+                        "on %p", (unsigned long long)lock->fl_start,
+                        (unsigned long long)lock->fl_end,
+                        lkowner_utoa (&lock->owner), lock->client);
+                goto out;
         }
-	__delete_inode_lock (conf);
+        __delete_inode_lock (conf);
         gf_log (this->name, GF_LOG_DEBUG,
-                " Matching lock found for unlock");
-        __destroy_inode_lock (lock);
-
+                " Matching lock found for unlock %llu-%llu, by %s on %p",
+                (unsigned long long)lock->fl_start,
+                (unsigned long long)lock->fl_end, lkowner_utoa (&lock->owner),
+                lock->client);
 
 out:
-	return conf;
+        return conf;
+}
 
 
-}
 static void
-__grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode, 
-			     struct list_head *granted, pl_dom_list_t *dom)
+__grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
+                             struct list_head *granted, pl_dom_list_t *dom)
 {
-	int	      bl_ret = 0;
-	pl_inode_lock_t *bl = NULL;
-	pl_inode_lock_t *tmp = NULL;
+        int              bl_ret = 0;
+        pl_inode_lock_t *bl = NULL;
+        pl_inode_lock_t *tmp = NULL;
 
         struct list_head blocked_list;
 
         INIT_LIST_HEAD (&blocked_list);
         list_splice_init (&dom->blocked_inodelks, &blocked_list);
 
-	list_for_each_entry_safe (bl, tmp, &blocked_list, blocked_locks) {
+        list_for_each_entry_safe (bl, tmp, &blocked_list, blocked_locks) {
 
-		list_del_init (&bl->blocked_locks);
+                list_del_init (&bl->blocked_locks);
 
-		bl_ret = __lock_inodelk (this, pl_inode, bl, 1, dom);
+                bl_ret = __lock_inodelk (this, pl_inode, bl, 1, dom);
 
-		if (bl_ret == 0) {
-			list_add (&bl->blocked_locks, granted);
+                if (bl_ret == 0) {
+                        list_add (&bl->blocked_locks, granted);
                 }
         }
-	return;
+        return;
 }
 
 /* Grant all inodelks blocked on a lock */
 void
-grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode, pl_dom_list_t *dom)
+grant_blocked_inode_locks (xlator_t *this, pl_inode_t *pl_inode,
+                           pl_dom_list_t *dom)
 {
-	struct list_head granted;
-	pl_inode_lock_t *lock;
-	pl_inode_lock_t *tmp;
-
-	INIT_LIST_HEAD (&granted);
+        struct list_head granted;
+        pl_inode_lock_t *lock;
+        pl_inode_lock_t *tmp;
 
-        if (list_empty (&dom->blocked_inodelks)) {
-                gf_log (this->name, GF_LOG_TRACE,
-                        "No blocked locks to be granted for domain: %s", dom->domain);
-        }
+        INIT_LIST_HEAD (&granted);
 
         pthread_mutex_lock (&pl_inode->mutex);
-	{
-		__grant_blocked_inode_locks (this, pl_inode, &granted, dom);
-	}
+        {
+                __grant_blocked_inode_locks (this, pl_inode, &granted, dom);
+        }
         pthread_mutex_unlock (&pl_inode->mutex);
 
-	list_for_each_entry_safe (lock, tmp, &granted, blocked_locks) {
+        list_for_each_entry_safe (lock, tmp, &granted, blocked_locks) {
                 gf_log (this->name, GF_LOG_TRACE,
-			"%s (pid=%d) (lk-owner=%"PRIu64") %"PRId64" - %"PRId64" => Granted",
-			lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-			lock->client_pid,
-			lock->owner,
-			lock->user_flock.l_start,
-			lock->user_flock.l_len);
+                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => Granted",
+                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                        lock->client_pid,
+                        lkowner_utoa (&lock->owner),
+                        lock->user_flock.l_start,
+                        lock->user_flock.l_len);
 
-		pl_trace_out (this, lock->frame, NULL, NULL, F_SETLKW,
-			      &lock->user_flock, 0, 0, lock->volume);
+                pl_trace_out (this, lock->frame, NULL, NULL, F_SETLKW,
+                              &lock->user_flock, 0, 0, lock->volume);
 
-		STACK_UNWIND_STRICT (inodelk, lock->frame, 0, 0);
-	}
+                STACK_UNWIND_STRICT (inodelk, lock->frame, 0, 0, NULL);
+		lock->frame = NULL;
+        }
 
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry_safe (lock, tmp, &granted, blocked_locks) {
+                        list_del_init (&lock->blocked_locks);
+                        __pl_inodelk_unref (lock);
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
 }
 
-/* Release all inodelks from this transport */
-static int
-release_inode_locks_of_transport (xlator_t *this, pl_dom_list_t *dom,
-                                  inode_t *inode, void *trans)
-{
-	pl_inode_lock_t *tmp = NULL;
-	pl_inode_lock_t *l = NULL;
-
-        pl_inode_t * pinode = NULL;
-
-        struct list_head granted;
-        struct list_head released;
 
+static void
+pl_inodelk_log_cleanup (pl_inode_lock_t *lock)
+{
+	pl_inode_t *pl_inode = NULL;
         char *path = NULL;
+        char *file = NULL;
 
-        INIT_LIST_HEAD (&granted);
-        INIT_LIST_HEAD (&released);
+	pl_inode = lock->pl_inode;
 
-        pinode = pl_inode_get (this, inode);
+	inode_path (pl_inode->refkeeper, NULL, &path);
 
-        pthread_mutex_lock (&pinode->mutex);
-        {
-
-                list_for_each_entry_safe (l, tmp, &dom->blocked_inodelks, blocked_locks) {
-                        if (l->transport != trans)
-                                continue;
+	if (path)
+		file = path;
+	else
+		file = uuid_utoa (pl_inode->refkeeper->gfid);
+
+	gf_log (THIS->name, GF_LOG_WARNING,
+		"releasing lock on %s held by "
+		"{client=%p, pid=%"PRId64" lk-owner=%s}",
+		file, lock->client, (uint64_t) lock->client_pid,
+		lkowner_utoa (&lock->owner));
+	GF_FREE (path);
+}
 
-                        list_del_init (&l->blocked_locks);
 
-                        if (inode_path (inode, NULL, &path) < 0) {
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "inode_path failed");
-                                goto unlock;
-                        }
+/* Release all inodelks from this client */
+int
+pl_inodelk_client_cleanup (xlator_t *this, pl_ctx_t *ctx)
+{
+        pl_inode_lock_t *tmp = NULL;
+        pl_inode_lock_t *l = NULL;
+	pl_dom_list_t *dom = NULL;
+        pl_inode_t *pl_inode = NULL;
 
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "releasing lock on %s held by "
-                                "{transport=%p, pid=%"PRId64" lk-owner=%"PRIu64"}",
-                                path, trans,
-                                (uint64_t) l->client_pid,
-                                l->owner);
+        struct list_head released;
+        struct list_head unwind;
 
-                        list_add (&l->blocked_locks, &released);
+        INIT_LIST_HEAD (&released);
+        INIT_LIST_HEAD (&unwind);
 
+	pthread_mutex_lock (&ctx->lock);
+        {
+                list_for_each_entry_safe (l, tmp, &ctx->inodelk_lockers,
+					  client_list) {
+                        list_del_init (&l->client_list);
+
+			pl_inodelk_log_cleanup (l);
+
+			pl_inode = l->pl_inode;
+
+			pthread_mutex_lock (&pl_inode->mutex);
+			{
+                        /* If the inodelk object is part of granted list but not
+                         * blocked list, then perform the following actions:
+                         * i.   delete the object from granted list;
+                         * ii.  grant other locks (from other clients) that may
+                         *      have been blocked on this inodelk; and
+                         * iii. unref the object.
+                         *
+                         * If the inodelk object (L1) is part of both granted
+                         * and blocked lists, then this means that a parallel
+                         * unlock on another inodelk (L2 say) may have 'granted'
+                         * L1 and added it to 'granted' list in
+                         * __grant_blocked_node_locks() (although using the
+                         * 'blocked_locks' member). In that case, the cleanup
+                         * codepath must try and grant other overlapping
+                         * blocked inodelks from other clients, now that L1 is
+                         * out of their way and then unref L1 in the end, and
+                         * leave it to the other thread (the one executing
+                         * unlock codepath) to unwind L1's frame, delete it from
+                         * blocked_locks list, and perform the last unref on L1.
+                         *
+                         * If the inodelk object (L1) is part of blocked list
+                         * only, the cleanup code path must:
+                         * i.   delete it from the blocked_locks list inside
+                         *      this critical section,
+                         * ii.  unwind its frame with EAGAIN,
+                         * iii. try and grant blocked inode locks from other
+                         *      clients that were otherwise grantable, but just
+                         *      got blocked to avoid leaving L1 to starve
+                         *      forever.
+                         * iv.  unref the object.
+                         */
+                                if (!list_empty (&l->list)) {
+                                        __delete_inode_lock (l);
+                                        list_add_tail (&l->client_list,
+                                                       &released);
+                                } else {
+                                        list_del_init(&l->blocked_locks);
+                                        list_add_tail (&l->client_list,
+                                                       &unwind);
+                                }
+                        }
+			pthread_mutex_unlock (&pl_inode->mutex);
                 }
+	}
+        pthread_mutex_unlock (&ctx->lock);
 
-                list_for_each_entry_safe (l, tmp, &dom->inodelk_list, list) {
-                        if (l->transport != trans)
-                                continue;
-
-                        __delete_inode_lock (l);
-			__destroy_inode_lock (l);
-
+        list_for_each_entry_safe (l, tmp, &unwind, client_list) {
+                list_del_init (&l->client_list);
 
-                        if (inode_path (inode, NULL, &path) < 0) {
-                                gf_log (this->name, GF_LOG_TRACE,
-                                        "inode_path failed");
-                                goto unlock;
-                        }
+                if (l->frame)
+			STACK_UNWIND_STRICT (inodelk, l->frame, -1, EAGAIN,
+					     NULL);
+                list_add_tail (&l->client_list, &released);
 
-                        gf_log (this->name, GF_LOG_TRACE,
-                                "releasing lock on %s held by "
-                                "{transport=%p, pid=%"PRId64" lk-owner=%"PRIu64"}",
-                                path, trans,
-                                (uint64_t) l->client_pid,
-                                l->owner);
+        }
 
+        list_for_each_entry_safe (l, tmp, &released, client_list) {
+                list_del_init (&l->client_list);
 
-                }
-        }
-unlock:
-        if (path)
-                GF_FREE (path);
+		pl_inode = l->pl_inode;
 
-        pthread_mutex_unlock (&pinode->mutex);
+		dom = get_domain (pl_inode, l->volume);
 
-        list_for_each_entry_safe (l, tmp, &released, blocked_locks) {
-                list_del_init (&l->blocked_locks);
+		grant_blocked_inode_locks (this, pl_inode, dom);
 
-                STACK_UNWIND_STRICT (inodelk, l->frame, -1, EAGAIN);
-                GF_FREE (l);
+		pthread_mutex_lock (&pl_inode->mutex);
+		{
+			__pl_inodelk_unref (l);
+		}
+		pthread_mutex_unlock (&pl_inode->mutex);
         }
 
-	grant_blocked_inode_locks (this, pinode, dom);
-	return 0;
+        return 0;
 }
 
 
 static int
-pl_inode_setlk (xlator_t *this, pl_inode_t *pl_inode, pl_inode_lock_t *lock,
-		int can_block,  pl_dom_list_t *dom)
+pl_inode_setlk (xlator_t *this, pl_ctx_t *ctx, pl_inode_t *pl_inode,
+		pl_inode_lock_t *lock, int can_block, pl_dom_list_t *dom)
 {
-	int ret = -EINVAL;
+        int ret = -EINVAL;
         pl_inode_lock_t *retlock = NULL;
+        gf_boolean_t    unref = _gf_true;
+
+	lock->pl_inode = pl_inode;
 
-	pthread_mutex_lock (&pl_inode->mutex);
-	{
-		if (lock->fl_type != F_UNLCK) {
-			ret = __lock_inodelk (this, pl_inode, lock, can_block, dom);
-			if (ret == 0)
-				gf_log (this->name, GF_LOG_TRACE,
-                                        "%s (pid=%d) (lk-owner=%"PRIu64") %"PRId64" - %"PRId64" => OK",
+	if (ctx)
+		pthread_mutex_lock (&ctx->lock);
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                if (lock->fl_type != F_UNLCK) {
+                        ret = __lock_inodelk (this, pl_inode, lock, can_block, dom);
+                        if (ret == 0) {
+				lock->frame = NULL;
+                                gf_log (this->name, GF_LOG_TRACE,
+                                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => OK",
                                         lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
                                         lock->client_pid,
-                                        lock->owner,
+                                        lkowner_utoa (&lock->owner),
                                         lock->fl_start,
                                         lock->fl_end);
+                        } else if (ret == -EAGAIN) {
+                                gf_log (this->name, GF_LOG_TRACE,
+                                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => NOK",
+                                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                                        lock->client_pid,
+                                        lkowner_utoa (&lock->owner),
+                                        lock->user_flock.l_start,
+                                        lock->user_flock.l_len);
+                                if (can_block)
+                                        unref = _gf_false;
+                        }
 
-			if (ret == -EAGAIN)
-				gf_log (this->name, GF_LOG_TRACE,
-                                        "%s (pid=%d) (lk-owner=%"PRIu64") %"PRId64" - %"PRId64" => NOK",
-					lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-					lock->client_pid,
-                                        lock->owner,
-					lock->user_flock.l_start,
-					lock->user_flock.l_len);
-
-			goto out;
-		}
-
+			if (ctx && (!ret || can_block))
+				list_add_tail (&lock->client_list,
+					       &ctx->inodelk_lockers);
+                } else {
+                        retlock = __inode_unlock_lock (this, lock, dom);
+                        if (!retlock) {
+                                gf_log (this->name, GF_LOG_DEBUG,
+                                        "Bad Unlock issued on Inode lock");
+                                ret = -EINVAL;
+                                goto out;
+                        }
+			list_del_init (&retlock->client_list);
+			__pl_inodelk_unref (retlock);
 
-		retlock = __inode_unlock_lock (this, lock, dom);
-		if (!retlock) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"Bad Unlock issued on Inode lock");
-                        ret = -EINVAL;
-                        goto out;
+                        ret = 0;
                 }
-		__destroy_inode_lock (retlock);
-
-                ret = 0;
+out:
+		if (unref)
+			__pl_inodelk_unref (lock);
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+	if (ctx)
+		pthread_mutex_unlock (&ctx->lock);
 
+        grant_blocked_inode_locks (this, pl_inode, dom);
 
-	}
-out:
-	pthread_mutex_unlock (&pl_inode->mutex);
-	grant_blocked_inode_locks (this, pl_inode, dom);
         return ret;
 }
 
 /* Create a new inode_lock_t */
 pl_inode_lock_t *
-new_inode_lock (struct gf_flock *flock, void *transport, pid_t client_pid,
-                uint64_t owner, const char *volume)
+new_inode_lock (struct gf_flock *flock, client_t *client, pid_t client_pid,
+                call_frame_t *frame, xlator_t *this, const char *volume,
+                char *conn_id)
 
 {
-	pl_inode_lock_t *lock = NULL;
+        pl_inode_lock_t *lock = NULL;
 
-	lock = GF_CALLOC (1, sizeof (*lock),
+        lock = GF_CALLOC (1, sizeof (*lock),
                           gf_locks_mt_pl_inode_lock_t);
-	if (!lock) {
-		return NULL;
-	}
+        if (!lock) {
+                return NULL;
+        }
 
-	lock->fl_start = flock->l_start;
-	lock->fl_type  = flock->l_type;
+        lock->fl_start = flock->l_start;
+        lock->fl_type  = flock->l_type;
 
-	if (flock->l_len == 0)
-		lock->fl_end = LLONG_MAX;
-	else
-		lock->fl_end = flock->l_start + flock->l_len - 1;
+        if (flock->l_len == 0)
+                lock->fl_end = LLONG_MAX;
+        else
+                lock->fl_end = flock->l_start + flock->l_len - 1;
+
+        lock->client     = client;
+        lock->client_pid = client_pid;
+        lock->volume     = volume;
+        lock->owner      = frame->root->lk_owner;
+        lock->frame      = frame;
+        lock->this       = this;
 
-	lock->transport  = transport;
-	lock->client_pid = client_pid;
-        lock->owner      = owner;
-	lock->volume     = volume;
+        if (conn_id) {
+                lock->connection_id = gf_strdup (conn_id);
+        }
 
-	INIT_LIST_HEAD (&lock->list);
-	INIT_LIST_HEAD (&lock->blocked_locks);
+        INIT_LIST_HEAD (&lock->list);
+        INIT_LIST_HEAD (&lock->blocked_locks);
+	INIT_LIST_HEAD (&lock->client_list);
+        __pl_inodelk_ref (lock);
 
-	return lock;
+        return lock;
+}
+
+int32_t
+_pl_convert_volume (const char *volume, char **res)
+{
+        char    *mdata_vol = NULL;
+        int     ret = 0;
+
+        mdata_vol = strrchr (volume, ':');
+        //if the volume already ends with :metadata don't bother
+        if (mdata_vol && (strcmp (mdata_vol, ":metadata") == 0))
+                return 0;
+
+        ret = gf_asprintf (res, "%s:metadata", volume);
+        if (ret <= 0)
+                return ENOMEM;
+        return 0;
+}
+
+int32_t
+_pl_convert_volume_for_special_range (struct gf_flock *flock,
+                                      const char *volume, char **res)
+{
+        int32_t     ret = 0;
+
+        if ((flock->l_start == LLONG_MAX -1) &&
+            (flock->l_len == 0)) {
+                ret = _pl_convert_volume (volume, res);
+        }
+
+        return ret;
 }
 
 /* Common inodelk code called from pl_inodelk and pl_finodelk */
 int
 pl_common_inodelk (call_frame_t *frame, xlator_t *this,
                    const char *volume, inode_t *inode, int32_t cmd,
-                   struct gf_flock *flock, loc_t *loc, fd_t *fd)
+                   struct gf_flock *flock, loc_t *loc, fd_t *fd, dict_t *xdata)
 {
-	int32_t op_ret   = -1;
-	int32_t op_errno = 0;
-	int     ret      = -1;
-	int     can_block = 0;
-	void *                  transport  = NULL;
-	pid_t                   client_pid = -1;
-        uint64_t                owner      = -1;
-	pl_inode_t *            pinode     = NULL;
-	pl_inode_lock_t *       reqlock    = NULL;
-	pl_dom_list_t *		dom	   = NULL;
-
-	VALIDATE_OR_GOTO (frame, out);
-	VALIDATE_OR_GOTO (inode, unwind);
-	VALIDATE_OR_GOTO (flock, unwind);
-
-	if ((flock->l_start < 0) || (flock->l_len < 0)) {
-		op_errno = EINVAL;
-		goto unwind;
-	}
+        int32_t           op_ret     = -1;
+        int32_t           op_errno   = 0;
+        int               ret        = -1;
+        GF_UNUSED int     dict_ret   = -1;
+        int               can_block  = 0;
+        pl_inode_t *      pinode     = NULL;
+        pl_inode_lock_t * reqlock    = NULL;
+        pl_dom_list_t *   dom        = NULL;
+        char             *res        = NULL;
+        char             *res1       = NULL;
+        char             *conn_id    = NULL;
+        pl_ctx_t         *ctx        = NULL;
+
+        if (xdata)
+                dict_ret = dict_get_str (xdata, "connection-id", &conn_id);
+
+        VALIDATE_OR_GOTO (frame, out);
+        VALIDATE_OR_GOTO (inode, unwind);
+        VALIDATE_OR_GOTO (flock, unwind);
+
+        if ((flock->l_start < 0) || (flock->l_len < 0)) {
+                op_errno = EINVAL;
+                goto unwind;
+        }
 
-        pl_trace_in (this, frame, fd, loc, cmd, flock, volume);
+        op_errno = _pl_convert_volume_for_special_range (flock, volume, &res);
+        if (op_errno)
+                goto unwind;
+        if (res)
+                volume = res;
 
-	transport  = frame->root->trans;
-	client_pid = frame->root->pid;
-        owner      = frame->root->lk_owner;
+        pl_trace_in (this, frame, fd, loc, cmd, flock, volume);
 
-	pinode = pl_inode_get (this, inode);
-	if (!pinode) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		op_errno = ENOMEM;
-		goto unwind;
+	if (frame->root->client) {
+		ctx = pl_ctx_get (frame->root->client, this);
+		if (!ctx) {
+			op_errno = ENOMEM;
+			gf_log (this->name, GF_LOG_INFO, "pl_ctx_get() failed");
+			goto unwind;
+		}
 	}
 
-	dom = get_domain (pinode, volume);
-
-	if (owner == 0) {
-		/*
-                  special case: this means release all locks
-                  from this transport
-		*/
-		gf_log (this->name, GF_LOG_TRACE,
-			"Releasing all locks from transport %p", transport);
+        pinode = pl_inode_get (this, inode);
+        if (!pinode) {
+                op_errno = ENOMEM;
+                goto unwind;
+        }
 
-		release_inode_locks_of_transport (this, dom, inode, transport);
-		goto unwind;
-	}
+        dom = get_domain (pinode, volume);
+        if (!dom) {
+                op_errno = ENOMEM;
+                goto unwind;
+        }
 
-	reqlock = new_inode_lock (flock, transport, client_pid, owner, volume);
+        reqlock = new_inode_lock (flock, frame->root->client, frame->root->pid,
+                                  frame, this, volume, conn_id);
 
-	if (!reqlock) {
-		gf_log (this->name, GF_LOG_ERROR,
-			"Out of memory.");
-		op_ret = -1;
-		op_errno = ENOMEM;
-		goto unwind;
-	}
+        if (!reqlock) {
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto unwind;
+        }
 
-        reqlock->frame = frame;
-        reqlock->this  = this;
 
-	switch (cmd) {
-	case F_SETLKW:
-		can_block = 1;
-		reqlock->frame = frame;
-		reqlock->this  = this;
+        switch (cmd) {
+        case F_SETLKW:
+                can_block = 1;
 
-		/* fall through */
+                /* fall through */
 
-	case F_SETLK:
-		memcpy (&reqlock->user_flock, flock, sizeof (struct gf_flock));
-		ret = pl_inode_setlk (this, pinode, reqlock,
-                                      can_block, dom);
+        case F_SETLK:
+                memcpy (&reqlock->user_flock, flock, sizeof (struct gf_flock));
+                ret = pl_inode_setlk (this, ctx, pinode, reqlock, can_block,
+				      dom);
 
-		if (ret < 0) {
-                        if (can_block) {
+                if (ret < 0) {
+                        if ((can_block) && (F_UNLCK != flock->l_type)) {
                                 pl_trace_block (this, frame, fd, loc,
                                                 cmd, flock, volume);
-				goto out;
+                                goto out;
                         }
-			gf_log (this->name, GF_LOG_TRACE, "returning EAGAIN");
-			op_errno = -ret;
-			__destroy_inode_lock (reqlock);
-			goto unwind;
-		}
-		break;
+                        gf_log (this->name, GF_LOG_TRACE, "returning EAGAIN");
+                        op_errno = -ret;
+                        goto unwind;
+                }
+                break;
 
-	default:
-		op_errno = ENOTSUP;
-		gf_log (this->name, GF_LOG_DEBUG,
+        default:
+                op_errno = ENOTSUP;
+                gf_log (this->name, GF_LOG_DEBUG,
                         "Lock command F_GETLK not supported for [f]inodelk "
                         "(cmd=%d)",
                         cmd);
@@ -641,85 +751,85 @@ pl_common_inodelk (call_frame_t *frame, xlator_t *this,
         op_ret = 0;
 
 unwind:
-	if ((inode != NULL) && (flock !=NULL)) {
-		pl_update_refkeeper (this, inode);
-		pl_trace_out (this, frame, fd, loc, cmd, flock, op_ret, op_errno, volume);
-	}
+        if ((inode != NULL) && (flock !=NULL)) {
+                pl_update_refkeeper (this, inode);
+                pl_trace_out (this, frame, fd, loc, cmd, flock, op_ret, op_errno, volume);
+        }
 
-        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (inodelk, frame, op_ret, op_errno, NULL);
 out:
+        GF_FREE (res);
+        GF_FREE (res1);
         return 0;
 }
 
 int
 pl_inodelk (call_frame_t *frame, xlator_t *this,
-            const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock)
+            const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock,
+            dict_t *xdata)
 {
-
-        pl_common_inodelk (frame, this, volume, loc->inode, cmd, flock, loc, NULL);
+        pl_common_inodelk (frame, this, volume, loc->inode, cmd, flock,
+                           loc, NULL, xdata);
 
         return 0;
 }
 
 int
 pl_finodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock)
+             const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock,
+             dict_t *xdata)
 {
-
-        pl_common_inodelk (frame, this, volume, fd->inode, cmd, flock, NULL, fd);
+        pl_common_inodelk (frame, this, volume, fd->inode, cmd, flock,
+                           NULL, fd, xdata);
 
         return 0;
 
 }
 
+static inline int32_t
+__get_inodelk_dom_count (pl_dom_list_t *dom)
+{
+        pl_inode_lock_t     *lock   = NULL;
+        int32_t             count   = 0;
+
+        list_for_each_entry (lock, &dom->inodelk_list, list) {
+                count++;
+        }
+        list_for_each_entry (lock, &dom->blocked_inodelks, blocked_locks) {
+                count++;
+        }
+        return count;
+}
 
-static int32_t
-__get_inodelk_count (xlator_t *this, pl_inode_t *pl_inode)
+/* Returns the no. of locks (blocked/granted) held on a given domain name
+ * If @domname is NULL, returns the no. of locks in all the domains present.
+ * If @domname is non-NULL and non-existent, returns 0 */
+int32_t
+__get_inodelk_count (xlator_t *this, pl_inode_t *pl_inode, char *domname)
 {
         int32_t            count  = 0;
-        pl_inode_lock_t   *lock   = NULL;
         pl_dom_list_t     *dom    = NULL;
 
         list_for_each_entry (dom, &pl_inode->dom_list, inode_list) {
-                list_for_each_entry (lock, &dom->inodelk_list, list) {
-
-			gf_log (this->name, GF_LOG_DEBUG,
-                                " XATTR DEBUG"
-				" domain: %s %s (pid=%d) (lk-owner=%"PRIu64") %"PRId64" - %"PRId64" "
-                                "state = Active",
-                                dom->domain,
-				lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-				lock->client_pid,
-                                lock->owner,
-				lock->user_flock.l_start,
-				lock->user_flock.l_len);
-
-                        count++;
-                }
-
-                list_for_each_entry (lock, &dom->blocked_inodelks, blocked_locks) {
+                if (domname) {
+                        if (strcmp (domname, dom->domain) == 0) {
+                                count = __get_inodelk_dom_count (dom);
+                                goto out;
+                        }
 
-			gf_log (this->name, GF_LOG_DEBUG,
-                                " XATTR DEBUG"
-				" domain: %s %s (pid=%d) (lk-owner=%"PRIu64") %"PRId64" - %"PRId64" "
-                                "state = Blocked",
-                                dom->domain,
-				lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-				lock->client_pid,
-                                lock->owner,
-				lock->user_flock.l_start,
-				lock->user_flock.l_len);
+                } else {
+                    /* Counting locks from all domains */
+                        count += __get_inodelk_dom_count (dom);
 
-                        count++;
                 }
-
         }
 
+out:
         return count;
 }
 
 int32_t
-get_inodelk_count (xlator_t *this, inode_t *inode)
+get_inodelk_count (xlator_t *this, inode_t *inode, char *domname)
 {
         pl_inode_t   *pl_inode = NULL;
         uint64_t      tmp_pl_inode = 0;
@@ -735,7 +845,7 @@ get_inodelk_count (xlator_t *this, inode_t *inode)
 
         pthread_mutex_lock (&pl_inode->mutex);
         {
-                count = __get_inodelk_count (this, pl_inode);
+                count = __get_inodelk_count (this, pl_inode, domname);
         }
         pthread_mutex_unlock (&pl_inode->mutex);
 
diff --git a/xlators/features/locks/src/locks-mem-types.h b/xlators/features/locks/src/locks-mem-types.h
index b4f7e28f1..08aeb0a79 100644
--- a/xlators/features/locks/src/locks-mem-types.h
+++ b/xlators/features/locks/src/locks-mem-types.h
@@ -1,23 +1,13 @@
 /*
-   Copyright (c) 2008-2009 Gluster, Inc. <http://www.gluster.com>
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
    This file is part of GlusterFS.
 
-   GlusterFS is free software; you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published
-   by the Free Software Foundation; either version 3 of the License,
-   or (at your option) any later version.
-
-   GlusterFS is distributed in the hope that it will be useful, but
-   WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see
-   <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
 
-
 #ifndef __LOCKS_MEM_TYPES_H__
 #define __LOCKS_MEM_TYPES_H__
 
@@ -32,7 +22,6 @@ enum gf_locks_mem_types_ {
         gf_locks_mt_truncate_ops,
         gf_locks_mt_pl_rw_req_t,
         gf_locks_mt_posix_locks_private_t,
-        gf_locks_mt_pl_local_t,
         gf_locks_mt_pl_fdctx_t,
         gf_locks_mt_end
 };
diff --git a/xlators/features/locks/src/locks.h b/xlators/features/locks/src/locks.h
index 614bddb64..8c2a6f867 100644
--- a/xlators/features/locks/src/locks.h
+++ b/xlators/features/locks/src/locks.h
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006, 2007, 2008 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef __POSIX_LOCKS_H__
 #define __POSIX_LOCKS_H__
 
@@ -29,6 +19,9 @@
 #include "stack.h"
 #include "call-stub.h"
 #include "locks-mem-types.h"
+#include "client_t.h"
+
+#include "lkowner.h"
 
 struct __pl_fd;
 
@@ -40,25 +33,29 @@ struct __posix_lock {
         off_t              fl_end;
 
         short              blocked;    /* waiting to acquire */
-        struct gf_flock       user_flock; /* the flock supplied by the user */
+        struct gf_flock    user_flock; /* the flock supplied by the user */
         xlator_t          *this;       /* required for blocked locks */
         unsigned long      fd_num;
 
         fd_t              *fd;
         call_frame_t      *frame;
 
+        struct timeval     blkd_time;   /*time at which lock was queued into blkd list*/
+        struct timeval     granted_time; /*time at which lock was queued into active list*/
+
         /* These two together serve to uniquely identify each process
            across nodes */
 
-        void              *transport;     /* to identify client node */
+        void              *client;     /* to identify client node */
+        gf_lkowner_t       owner;
         pid_t              client_pid;    /* pid of client process */
-        uint64_t           owner;         /* lock owner from fuse */
 };
 typedef struct __posix_lock posix_lock_t;
 
 struct __pl_inode_lock {
         struct list_head   list;
         struct list_head   blocked_locks; /* list_head pointing to blocked_inodelks */
+        int                ref;
 
         short              fl_type;
         off_t              fl_start;
@@ -66,18 +63,25 @@ struct __pl_inode_lock {
 
         const char        *volume;
 
-        struct gf_flock       user_flock; /* the flock supplied by the user */
+        struct gf_flock    user_flock; /* the flock supplied by the user */
         xlator_t          *this;       /* required for blocked locks */
-        fd_t              *fd;
+	struct __pl_inode *pl_inode;
 
         call_frame_t      *frame;
 
+        struct timeval     blkd_time;   /*time at which lock was queued into blkd list*/
+        struct timeval     granted_time; /*time at which lock was queued into active list*/
+
         /* These two together serve to uniquely identify each process
            across nodes */
 
-        void              *transport;     /* to identify client node */
+        void              *client;     /* to identify client node */
+        gf_lkowner_t       owner;
         pid_t              client_pid;    /* pid of client process */
-        uint64_t           owner;
+
+        char              *connection_id; /* stores the client connection id */
+
+	struct list_head   client_list; /* list of all locks from a client */
 };
 typedef struct __pl_inode_lock pl_inode_lock_t;
 
@@ -101,18 +105,27 @@ typedef struct __pl_dom_list_t pl_dom_list_t;
 struct __entry_lock {
         struct list_head  domain_list;    /* list_head back to pl_dom_list_t */
         struct list_head  blocked_locks; /* list_head back to blocked_entrylks */
+	int ref;
 
         call_frame_t     *frame;
         xlator_t         *this;
+	struct __pl_inode *pinode;
 
         const char       *volume;
 
         const char       *basename;
         entrylk_type      type;
 
-        void      *trans;
-        pid_t             client_pid;    /* pid of client process */
-        uint64_t          owner;
+        struct timeval     blkd_time;   /*time at which lock was queued into blkd list*/
+        struct timeval     granted_time; /*time at which lock was queued into active list*/
+
+        void             *client;
+        gf_lkowner_t      owner;
+	pid_t             client_pid;    /* pid of client process */
+
+        char             *connection_id; /* stores the client connection id */
+
+	struct list_head   client_list; /* list of all locks from a client */
 };
 typedef struct __entry_lock pl_entry_lock_t;
 
@@ -137,25 +150,55 @@ struct __pl_inode {
 typedef struct __pl_inode pl_inode_t;
 
 
-struct __pl_fd {
-        gf_boolean_t nonblocking;       /* whether O_NONBLOCK has been set */
-};
-typedef struct __pl_fd pl_fd_t;
-
-
 typedef struct {
         gf_boolean_t    mandatory;      /* if mandatory locking is enabled */
         gf_boolean_t    trace;          /* trace lock requests in and out */
+        char           *brickname;
 } posix_locks_private_t;
 
+
 typedef struct {
         gf_boolean_t   entrylk_count_req;
         gf_boolean_t   inodelk_count_req;
+        gf_boolean_t   inodelk_dom_count_req;
         gf_boolean_t   posixlk_count_req;
+        gf_boolean_t   parent_entrylk_req;
+
+        /* used by {f,}truncate */
+        loc_t  loc;
+        fd_t  *fd;
+        off_t  offset;
+        dict_t *xdata;
+        enum {TRUNCATE, FTRUNCATE} op;
 } pl_local_t;
 
+
 typedef struct {
         struct list_head locks_list;
 } pl_fdctx_t;
 
+
+struct _locker {
+        struct list_head  lockers;
+        char             *volume;
+        inode_t          *inode;
+        gf_lkowner_t      owner;
+};
+
+typedef struct _locks_ctx {
+        pthread_mutex_t      lock;
+        struct list_head     inodelk_lockers;
+        struct list_head     entrylk_lockers;
+} pl_ctx_t;
+
+
+pl_ctx_t *
+pl_ctx_get (client_t *client, xlator_t *xlator);
+
+int
+pl_inodelk_client_cleanup (xlator_t *this, pl_ctx_t *ctx);
+
+int
+pl_entrylk_client_cleanup (xlator_t *this, pl_ctx_t *ctx);
+
 #endif /* __POSIX_LOCKS_H__ */
diff --git a/xlators/features/locks/src/posix.c b/xlators/features/locks/src/posix.c
index 29b3bb079..337623d65 100644
--- a/xlators/features/locks/src/posix.c
+++ b/xlators/features/locks/src/posix.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006, 2007, 2008 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #include <unistd.h>
 #include <fcntl.h>
 #include <limits.h>
@@ -37,6 +27,9 @@
 #include "locks.h"
 #include "common.h"
 #include "statedump.h"
+#include "clear.h"
+#include "defaults.h"
+#include "syncop.h"
 
 #ifndef LLONG_MAX
 #define LLONG_MAX LONG_LONG_MAX /* compat with old gcc */
@@ -47,13 +40,9 @@
 
 void do_blocked_rw (pl_inode_t *);
 static int __rw_allowable (pl_inode_t *, posix_lock_t *, glusterfs_fop_t);
-
-struct _truncate_ops {
-        loc_t  loc;
-        fd_t  *fd;
-        off_t  offset;
-        enum {TRUNCATE, FTRUNCATE} op;
-};
+static int format_brickname(char *);
+int pl_lockinfo_get_brickname (xlator_t *, inode_t *, int32_t *);
+static int fetch_pathinfo(xlator_t *, inode_t *, int32_t *, char **);
 
 static pl_fdctx_t *
 pl_new_fdctx ()
@@ -62,76 +51,76 @@ pl_new_fdctx ()
 
         fdctx = GF_CALLOC (1, sizeof (*fdctx),
                            gf_locks_mt_pl_fdctx_t);
-        if (!fdctx)
-                return NULL;
+        GF_VALIDATE_OR_GOTO ("posix-locks", fdctx, out);
 
         INIT_LIST_HEAD (&fdctx->locks_list);
 
+out:
         return fdctx;
 }
 
 static pl_fdctx_t *
-pl_get_fdctx (xlator_t *this, fd_t *fd)
+pl_check_n_create_fdctx (xlator_t *this, fd_t *fd)
 {
-        int ret = 0;
-        uint64_t tmp = 0;
-
-        ret = fd_ctx_get (fd, this, &tmp);
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Could not get fdctx");
-                goto out;
-        }
-
-        return ((pl_fdctx_t *) (long) tmp);
-
-out:
-        return NULL;
+        int         ret   = 0;
+        uint64_t    tmp   = 0;
+        pl_fdctx_t *fdctx = NULL;
 
-}
+        GF_VALIDATE_OR_GOTO ("posix-locks", this, out);
+        GF_VALIDATE_OR_GOTO (this->name, fd, out);
 
-static int
-pl_set_fdctx (xlator_t *this, fd_t *fd,
-              pl_fdctx_t *fdctx)
-{
-        int ret = 0;
-
-        ret = fd_ctx_set (fd, this,
-                          (uint64_t) (unsigned long) (fdctx));
+        LOCK (&fd->lock);
+        {
+                ret = __fd_ctx_get (fd, this, &tmp);
+                if ((ret != 0) || (tmp == 0)) {
+                        fdctx = pl_new_fdctx ();
+                        if (fdctx == NULL) {
+                                goto unlock;
+                        }
+                }
 
-        if (ret) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "Could not set fdctx");
-                goto out;
+                ret = __fd_ctx_set (fd, this, (uint64_t)(long)fdctx);
+                if (ret != 0) {
+                        GF_FREE (fdctx);
+                        fdctx = NULL;
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "failed to set fd ctx");
+                }
         }
+unlock:
+        UNLOCK (&fd->lock);
 
 out:
-        return ret;
-
+        return fdctx;
 }
 
 int
 pl_truncate_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                  int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-                 struct iatt *postbuf)
+                 struct iatt *postbuf, dict_t *xdata)
 {
-        struct _truncate_ops *local = NULL;
+        pl_local_t *local = NULL;
 
         local = frame->local;
 
         if (local->op == TRUNCATE)
                 loc_wipe (&local->loc);
 
+        if (local->xdata)
+                dict_unref (local->xdata);
+        if (local->fd)
+                fd_unref (local->fd);
+
         STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno,
-                             prebuf, postbuf);
+                             prebuf, postbuf, xdata);
         return 0;
 }
 
 
 static int
 truncate_allowed (pl_inode_t *pl_inode,
-                  void *transport, pid_t client_pid,
-                  uint64_t owner, off_t offset)
+                  client_t *client, pid_t client_pid,
+                  gf_lkowner_t *owner, off_t offset)
 {
         posix_lock_t *l = NULL;
         posix_lock_t  region = {.list = {0, }, };
@@ -139,9 +128,9 @@ truncate_allowed (pl_inode_t *pl_inode,
 
         region.fl_start   = offset;
         region.fl_end     = LLONG_MAX;
-        region.transport  = transport;
+        region.client     = client;
         region.client_pid = client_pid;
-        region.owner      = owner;
+        region.owner      = *owner;
 
         pthread_mutex_lock (&pl_inode->mutex);
         {
@@ -150,6 +139,8 @@ truncate_allowed (pl_inode_t *pl_inode,
                             && locks_overlap (&region, l)
                             && !same_owner (&region, l)) {
                                 ret = 0;
+                                gf_log ("posix-locks", GF_LOG_TRACE, "Truncate "
+                                        "allowed");
                                 break;
                         }
                 }
@@ -162,10 +153,11 @@ truncate_allowed (pl_inode_t *pl_inode,
 
 static int
 truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                   int32_t op_ret, int32_t op_errno, struct iatt *buf)
+                   int32_t op_ret, int32_t op_errno, struct iatt *buf,
+                   dict_t *xdata)
 {
         posix_locks_private_t *priv = NULL;
-        struct _truncate_ops  *local = NULL;
+        pl_local_t            *local = NULL;
         inode_t               *inode = NULL;
         pl_inode_t            *pl_inode = NULL;
 
@@ -187,8 +179,6 @@ truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         pl_inode = pl_inode_get (this, inode);
         if (!pl_inode) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
                 op_ret   = -1;
                 op_errno = ENOMEM;
                 goto unwind;
@@ -196,8 +186,8 @@ truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
         if (priv->mandatory
             && pl_inode->mandatory
-            && !truncate_allowed (pl_inode, frame->root->trans,
-                                  frame->root->pid, frame->root->lk_owner,
+            && !truncate_allowed (pl_inode, frame->root->client,
+                                  frame->root->pid, &frame->root->lk_owner,
                                   local->offset)) {
                 op_ret   = -1;
                 op_errno = EAGAIN;
@@ -208,53 +198,58 @@ truncate_stat_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
         case TRUNCATE:
                 STACK_WIND (frame, pl_truncate_cbk, FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->truncate,
-                            &local->loc, local->offset);
+                            &local->loc, local->offset, local->xdata);
                 break;
         case FTRUNCATE:
                 STACK_WIND (frame, pl_truncate_cbk, FIRST_CHILD (this),
                             FIRST_CHILD (this)->fops->ftruncate,
-                            local->fd, local->offset);
+                            local->fd, local->offset, local->xdata);
                 break;
         }
 
         return 0;
 
 unwind:
+        gf_log (this->name, GF_LOG_ERROR, "truncate failed with ret: %d, "
+                "error: %s", op_ret, strerror (op_errno));
         if (local->op == TRUNCATE)
                 loc_wipe (&local->loc);
+        if (local->xdata)
+                dict_unref (local->xdata);
+        if (local->fd)
+                fd_unref (local->fd);
 
-        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, buf, NULL);
+        STACK_UNWIND_STRICT (truncate, frame, op_ret, op_errno, buf, NULL, xdata);
         return 0;
 }
 
 
 int
 pl_truncate (call_frame_t *frame, xlator_t *this,
-             loc_t *loc, off_t offset)
+             loc_t *loc, off_t offset, dict_t *xdata)
 {
-        struct _truncate_ops *local = NULL;
+        pl_local_t *local = NULL;
 
-	local = GF_CALLOC (1, sizeof (struct _truncate_ops),
-                           gf_locks_mt_truncate_ops);
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
-                goto unwind;
-        }
+        local = mem_get0 (this->local_pool);
+        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
 
         local->op         = TRUNCATE;
         local->offset     = offset;
         loc_copy (&local->loc, loc);
+        if (xdata)
+                local->xdata = dict_ref (xdata);
 
         frame->local = local;
 
         STACK_WIND (frame, truncate_stat_cbk, FIRST_CHILD (this),
-                    FIRST_CHILD (this)->fops->stat, loc);
+                    FIRST_CHILD (this)->fops->stat, loc, NULL);
 
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL);
+        gf_log (this->name, GF_LOG_ERROR, "truncate for %s failed with ret: %d, "
+                "error: %s", loc->path, -1, strerror (ENOMEM));
+        STACK_UNWIND_STRICT (truncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
 }
@@ -262,34 +257,54 @@ unwind:
 
 int
 pl_ftruncate (call_frame_t *frame, xlator_t *this,
-              fd_t *fd, off_t offset)
+              fd_t *fd, off_t offset, dict_t *xdata)
 {
-        struct _truncate_ops *local = NULL;
+        pl_local_t *local = NULL;
 
-	local = GF_CALLOC (1, sizeof (struct _truncate_ops),
-                           gf_locks_mt_truncate_ops);
-        if (!local) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
-                goto unwind;
-        }
+        local = mem_get0 (this->local_pool);
+        GF_VALIDATE_OR_GOTO (this->name, local, unwind);
 
         local->op         = FTRUNCATE;
         local->offset     = offset;
-        local->fd         = fd;
+        local->fd         = fd_ref (fd);
+        if (xdata)
+                local->xdata = dict_ref (xdata);
 
         frame->local = local;
 
         STACK_WIND (frame, truncate_stat_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->fstat, fd);
+                    FIRST_CHILD(this)->fops->fstat, fd, xdata);
         return 0;
 
 unwind:
-        STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL);
+        gf_log (this->name, GF_LOG_ERROR, "ftruncate failed with ret: %d, "
+                "error: %s", -1, strerror (ENOMEM));
+        STACK_UNWIND_STRICT (ftruncate, frame, -1, ENOMEM, NULL, NULL, NULL);
 
         return 0;
 }
 
+int
+pl_locks_by_fd (pl_inode_t *pl_inode, fd_t *fd)
+{
+       posix_lock_t *l = NULL;
+       int found = 0;
+
+       pthread_mutex_lock (&pl_inode->mutex);
+       {
+
+               list_for_each_entry (l, &pl_inode->ext_list, list) {
+                       if (l->fd_num == fd_to_fdnum(fd)) {
+                               found = 1;
+                               break;
+                       }
+               }
+
+       }
+       pthread_mutex_unlock (&pl_inode->mutex);
+       return found;
+}
+
 static void
 delete_locks_of_fd (xlator_t *this, pl_inode_t *pl_inode, fd_t *fd)
 {
@@ -304,7 +319,7 @@ delete_locks_of_fd (xlator_t *this, pl_inode_t *pl_inode, fd_t *fd)
        {
 
                list_for_each_entry_safe (l, tmp, &pl_inode->ext_list, list) {
-                       if ((l->fd_num == fd_to_fdnum(fd))) {
+                       if (l->fd_num == fd_to_fdnum(fd)) {
                                if (l->blocked) {
                                        list_move_tail (&l->list, &blocked_list);
                                        continue;
@@ -319,7 +334,8 @@ delete_locks_of_fd (xlator_t *this, pl_inode_t *pl_inode, fd_t *fd)
 
        list_for_each_entry_safe (l, tmp, &blocked_list, list) {
                list_del_init(&l->list);
-               STACK_UNWIND_STRICT (lk, l->frame, -1, EAGAIN, &l->user_flock);
+               STACK_UNWIND_STRICT (lk, l->frame, -1, EAGAIN, &l->user_flock,
+                                    NULL);
                __destroy_lock (l);
        }
 
@@ -331,7 +347,7 @@ delete_locks_of_fd (xlator_t *this, pl_inode_t *pl_inode, fd_t *fd)
 
 static void
 __delete_locks_of_owner (pl_inode_t *pl_inode,
-                         void *transport, uint64_t owner)
+                         client_t *client, gf_lkowner_t *owner)
 {
         posix_lock_t *tmp = NULL;
         posix_lock_t *l = NULL;
@@ -339,16 +355,18 @@ __delete_locks_of_owner (pl_inode_t *pl_inode,
         /* TODO: what if it is a blocked lock with pending l->frame */
 
         list_for_each_entry_safe (l, tmp, &pl_inode->ext_list, list) {
-                if ((l->transport == transport)
-                    && (l->owner == owner)) {
-			gf_log ("posix-locks", GF_LOG_TRACE,
+                if (l->blocked)
+                        continue;
+                if ((l->client == client) &&
+                    is_same_lkowner (&l->owner, owner)) {
+                        gf_log ("posix-locks", GF_LOG_TRACE,
                                 " Flushing lock"
-				"%s (pid=%d) (lk-owner=%"PRIu64") %"PRId64" - %"PRId64" state: %s",
-				l->fl_type == F_UNLCK ? "Unlock" : "Lock",
-				l->client_pid,
-                                l->owner,
-				l->user_flock.l_start,
-				l->user_flock.l_len,
+                                "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" state: %s",
+                                l->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                                l->client_pid,
+                                lkowner_utoa (&l->owner),
+                                l->user_flock.l_start,
+                                l->user_flock.l_len,
                                 l->blocked == 1 ? "Blocked" : "Active");
 
                         __delete_lock (pl_inode, l);
@@ -359,70 +377,580 @@ __delete_locks_of_owner (pl_inode_t *pl_inode,
         return;
 }
 
+
 int32_t
-pl_opendir_cbk (call_frame_t *frame,
-		     void *cookie,
-		     xlator_t *this,
-		     int32_t op_ret,
-		     int32_t op_errno,
-		     fd_t *fd)
+pl_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int32_t op_ret, int32_t op_errno, dict_t *dict, dict_t *xdata)
 {
-        pl_fdctx_t *fdctx = NULL;
-        int ret = -1;
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
 
-        if (op_ret < 0)
-                goto unwind;
+}
 
-        fdctx = pl_get_fdctx (this, fd);
-        if (!fdctx) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "fdctx not present");
-                fdctx = pl_new_fdctx ();
-                if (!fdctx) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
+int32_t
+pl_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+             const char *name, dict_t *xdata)
+{
+        int32_t                 op_errno        = EINVAL;
+        int                     op_ret          = -1;
+        int32_t                 bcount          = 0;
+        int32_t                 gcount          = 0;
+        char                    key[PATH_MAX]   = {0, };
+        char                    *lk_summary     = NULL;
+        pl_inode_t              *pl_inode       = NULL;
+        dict_t                  *dict           = NULL;
+        clrlk_args              args            = {0,};
+        char                    *brickname      = NULL;
+
+        if (!name)
+                goto usual;
+
+        if (strncmp (name, GF_XATTR_CLRLK_CMD, strlen (GF_XATTR_CLRLK_CMD)))
+                goto usual;
+
+        if (clrlk_parse_args (name, &args)) {
+                op_errno = EINVAL;
+                goto out;
+        }
+
+        dict = dict_new ();
+        if (!dict) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        pl_inode = pl_inode_get (this, loc->inode);
+        if (!pl_inode) {
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        switch (args.type) {
+                case CLRLK_INODE:
+                case CLRLK_ENTRY:
+                        op_ret = clrlk_clear_lks_in_all_domains (this, pl_inode,
+                                                                 &args, &bcount,
+                                                                 &gcount,
+                                                                 &op_errno);
+                        if (op_ret)
+                                goto out;
+                        break;
+                case CLRLK_POSIX:
+                        op_ret = clrlk_clear_posixlk (this, pl_inode, &args,
+                                                      &bcount, &gcount,
+                                                      &op_errno);
+                        if (op_ret)
+                                goto out;
+                        break;
+                case CLRLK_TYPE_MAX:
+                        op_errno = EINVAL;
+                        goto out;
+        }
+
+        op_ret = fetch_pathinfo (this, loc->inode, &op_errno, &brickname);
+        if (op_ret) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Couldn't get brickname");
+        } else {
+                op_ret = format_brickname(brickname);
+                if (op_ret) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Couldn't format brickname");
+                        GF_FREE(brickname);
+                        brickname = NULL;
+                }
+        }
+
+        if (!gcount && !bcount) {
+                if (gf_asprintf (&lk_summary, "No locks cleared.") == -1) {
+                        op_ret = -1;
                         op_errno = ENOMEM;
-                        op_ret   = -1;
-                        goto unwind;
+                        goto out;
                 }
+        } else if (gf_asprintf (&lk_summary, "%s: %s blocked locks=%d "
+                                "granted locks=%d",
+                                (brickname == NULL)? this->name : brickname,
+                                (args.type == CLRLK_INODE)? "inode":
+                                (args.type == CLRLK_ENTRY)? "entry":
+                                (args.type == CLRLK_POSIX)? "posix": " ",
+                                bcount, gcount) == -1) {
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto out;
+        }
 
-                ret = pl_set_fdctx (this, fd, fdctx);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "could not set fdctx");
+        strncpy (key, name, strlen (name));
+        if (dict_set_dynstr (dict, key, lk_summary)) {
+                op_ret = -1;
+                op_errno = ENOMEM;
+                goto out;
+        }
+
+        op_ret = 0;
+out:
+        GF_FREE(brickname);
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
+
+        GF_FREE (args.opts);
+        if (op_ret && lk_summary)
+                GF_FREE (lk_summary);
+        if (dict)
+                dict_unref (dict);
+        return 0;
+
+usual:
+        STACK_WIND (frame, pl_getxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->getxattr, loc, name, xdata);
+        return 0;
+}
+
+static int
+format_brickname(char *brickname)
+{
+       int   ret             = -1;
+       char *hostname        = NULL;
+       char *volume          = NULL;
+       char *saveptr         = NULL;
+
+       if (!brickname)
+               goto out;
+
+       strtok_r(brickname, ":", &saveptr);
+       hostname = gf_strdup(strtok_r(NULL, ":", &saveptr));
+       if (hostname == NULL)
+               goto out;
+       volume = gf_strdup(strtok_r(NULL, ".", &saveptr));
+       if (volume == NULL)
+               goto out;
+
+       sprintf(brickname, "%s:%s", hostname, volume);
+
+       ret = 0;
+out:
+       GF_FREE(hostname);
+       GF_FREE(volume);
+       return ret;
+}
+
+static int
+fetch_pathinfo (xlator_t *this, inode_t *inode, int32_t *op_errno,
+                char **brickname)
+{
+        int                    ret       = -1;
+        loc_t                  loc       = {0, };
+        dict_t                *dict      = NULL;
+
+        if (!brickname)
+                goto out;
+
+        if (!op_errno)
+                goto out;
+
+        uuid_copy (loc.gfid, inode->gfid);
+        loc.inode = inode_ref (inode);
+
+        ret = syncop_getxattr (FIRST_CHILD(this), &loc, &dict,
+                               GF_XATTR_PATHINFO_KEY);
+        if (ret < 0) {
+                *op_errno = -ret;
+                ret = -1;
+                goto out;
+        }
+
+        ret = dict_get_str (dict, GF_XATTR_PATHINFO_KEY, brickname);
+        if (ret)
+                goto out;
+
+        *brickname = gf_strdup(*brickname);
+        if (*brickname == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        ret = 0;
+out:
+        if (dict != NULL) {
+                dict_unref (dict);
+        }
+        loc_wipe(&loc);
+
+        return ret;
+}
+
+
+int
+pl_lockinfo_get_brickname (xlator_t *this, inode_t *inode, int32_t *op_errno)
+{
+        int                    ret       = -1;
+        posix_locks_private_t *priv      = NULL;
+        char                  *brickname = NULL;
+        char                  *end       = NULL;
+        char                  *tmp       = NULL;
+
+        priv = this->private;
+
+        ret = fetch_pathinfo (this, inode, op_errno, &brickname);
+        if (ret)
+                goto out;
+
+        end = strrchr (brickname, ':');
+        if (!end) {
+                GF_FREE(brickname);
+                ret = -1;
+                goto out;
+        }
+
+        tmp = brickname;
+        brickname = gf_strndup (brickname, (end - brickname));
+        if (brickname == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        priv->brickname = brickname;
+        ret = 0;
+out:
+        GF_FREE(tmp);
+        return ret;
+}
+
+char *
+pl_lockinfo_key (xlator_t *this, inode_t *inode, int32_t *op_errno)
+{
+        posix_locks_private_t *priv = NULL;
+        char                  *key  = NULL;
+        int                    ret = 0;
+
+        priv = this->private;
+
+        if (priv->brickname == NULL) {
+                ret = pl_lockinfo_get_brickname (this, inode, op_errno);
+                if (ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "cannot get brickname");
+                        goto out;
+                }
+        }
+
+        key = priv->brickname;
+out:
+        return key;
+}
+
+int32_t
+pl_fgetxattr_handle_lockinfo (xlator_t *this, fd_t *fd,
+                              dict_t *dict, int32_t *op_errno)
+{
+        pl_inode_t    *pl_inode = NULL;
+        char          *key      = NULL, *buf = NULL;
+        int32_t        op_ret   = 0;
+        unsigned long  fdnum    = 0;
+	int32_t        len      = 0;
+        dict_t        *tmp      = NULL;
+
+        pl_inode = pl_inode_get (this, fd->inode);
+
+        if (!pl_inode) {
+                gf_log (this->name, GF_LOG_DEBUG, "Could not get inode.");
+                *op_errno = EBADFD;
+                op_ret = -1;
+                goto out;
+        }
+
+        if (!pl_locks_by_fd (pl_inode, fd)) {
+                op_ret = 0;
+                goto out;
+        }
+
+        fdnum = fd_to_fdnum (fd);
+
+        key = pl_lockinfo_key (this, fd->inode, op_errno);
+        if (key == NULL) {
+                op_ret = -1;
+                goto out;
+        }
+
+        tmp = dict_new ();
+        if (tmp == NULL) {
+                op_ret = -1;
+                *op_errno = ENOMEM;
+                goto out;
+        }
+
+        op_ret = dict_set_uint64 (tmp, key, fdnum);
+        if (op_ret < 0) {
+                *op_errno = -op_ret;
+                op_ret = -1;
+                gf_log (this->name, GF_LOG_WARNING, "setting lockinfo value "
+                        "(%lu) for fd (ptr:%p inode-gfid:%s) failed (%s)",
+                        fdnum, fd, uuid_utoa (fd->inode->gfid),
+                        strerror (*op_errno));
+                goto out;
+        }
+
+        len = dict_serialized_length (tmp);
+        if (len < 0) {
+                *op_errno = -op_ret;
+                op_ret = -1;
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dict_serialized_length failed (%s) while handling "
+                        "lockinfo for fd (ptr:%p inode-gfid:%s)",
+                        strerror (*op_errno), fd, uuid_utoa (fd->inode->gfid));
+                goto out;
+        }
+
+        buf = GF_CALLOC (1, len, gf_common_mt_char);
+        if (buf == NULL) {
+                op_ret = -1;
+                *op_errno = ENOMEM;
+                goto out;
+        }
+
+        op_ret = dict_serialize (tmp, buf);
+        if (op_ret < 0) {
+                *op_errno = -op_ret;
+                op_ret = -1;
+                gf_log (this->name, GF_LOG_WARNING,
+                        "dict_serialize failed (%s) while handling lockinfo "
+                        "for fd (ptr: %p inode-gfid:%s)", strerror (*op_errno),
+                        fd, uuid_utoa (fd->inode->gfid));
+                goto out;
+        }
+
+        op_ret = dict_set_dynptr (dict, GF_XATTR_LOCKINFO_KEY, buf, len);
+        if (op_ret < 0) {
+                *op_errno = -op_ret;
+                op_ret = -1;
+                gf_log (this->name, GF_LOG_WARNING, "setting lockinfo value "
+                        "(%lu) for fd (ptr:%p inode-gfid:%s) failed (%s)",
+                        fdnum, fd, uuid_utoa (fd->inode->gfid),
+                        strerror (*op_errno));
+                goto out;
+        }
+
+        buf = NULL;
+out:
+        if (tmp != NULL) {
+                dict_unref (tmp);
+        }
+
+        if (buf != NULL) {
+                GF_FREE (buf);
+        }
+
+        return op_ret;
+}
+
+
+int32_t
+pl_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+              const char *name, dict_t *xdata)
+{
+        int32_t  op_ret = 0, op_errno = 0;
+        dict_t  *dict   = NULL;
+
+        if (!name) {
+                goto usual;
+        }
+
+        if (strcmp (name, GF_XATTR_LOCKINFO_KEY) == 0) {
+                dict = dict_new ();
+                if (dict == NULL) {
                         op_ret = -1;
                         op_errno = ENOMEM;
                         goto unwind;
                 }
+
+                op_ret = pl_fgetxattr_handle_lockinfo (this, fd, dict,
+                                                       &op_errno);
+                if (op_ret < 0) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "getting lockinfo on fd (ptr:%p inode-gfid:%s) "
+                                "failed (%s)", fd, uuid_utoa (fd->inode->gfid),
+                                strerror (op_errno));
+                }
+
+                goto unwind;
+        } else {
+                goto usual;
+        }
+
+unwind:
+        STACK_UNWIND_STRICT (fgetxattr, frame, op_ret, op_errno, dict, NULL);
+        if (dict != NULL) {
+                dict_unref (dict);
+        }
+
+        return 0;
+
+usual:
+        STACK_WIND (frame, default_fgetxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fgetxattr, fd, name, xdata);
+        return 0;
+}
+
+int32_t
+pl_migrate_locks (call_frame_t *frame, fd_t *newfd, uint64_t oldfd_num,
+                  int32_t *op_errno)
+{
+        pl_inode_t   *pl_inode  = NULL;
+        uint64_t      newfd_num = 0;
+        posix_lock_t *l         = NULL;
+        int32_t       op_ret    = 0;
+
+        newfd_num = fd_to_fdnum (newfd);
+
+        pl_inode = pl_inode_get (frame->this, newfd->inode);
+        if (pl_inode == NULL) {
+                op_ret = -1;
+                *op_errno = EBADFD;
+                goto out;
+        }
+
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
+                list_for_each_entry (l, &pl_inode->ext_list, list) {
+                        if (l->fd_num == oldfd_num) {
+                                l->fd_num = newfd_num;
+                                l->client = frame->root->client;
+                        }
+                }
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
+
+        op_ret = 0;
+out:
+        return op_ret;
+}
+
+int32_t
+pl_fsetxattr_handle_lockinfo (call_frame_t *frame, fd_t *fd, char *lockinfo_buf,
+                              int len, int32_t *op_errno)
+{
+        int32_t   op_ret    = -1;
+        dict_t   *lockinfo  = NULL;
+        uint64_t  oldfd_num = 0;
+        char     *key       = NULL;
+
+        lockinfo = dict_new ();
+        if (lockinfo == NULL) {
+                op_ret = -1;
+                *op_errno = ENOMEM;
+                goto out;
+        }
+
+        op_ret = dict_unserialize (lockinfo_buf, len, &lockinfo);
+        if (op_ret < 0) {
+                *op_errno = -op_ret;
+                op_ret = -1;
+                goto out;
+        }
+
+        key = pl_lockinfo_key (frame->this, fd->inode, op_errno);
+        if (key == NULL) {
+                op_ret = -1;
+                goto out;
+        }
+
+        op_ret = dict_get_uint64 (lockinfo, key, &oldfd_num);
+
+        if (oldfd_num == 0) {
+                op_ret = 0;
+                goto out;
+        }
+
+        op_ret = pl_migrate_locks (frame, fd, oldfd_num, op_errno);
+        if (op_ret < 0) {
+                gf_log (frame->this->name, GF_LOG_WARNING,
+                        "migration of locks from oldfd (ptr:%p) to newfd "
+                        "(ptr:%p) (inode-gfid:%s)", (void *)oldfd_num, fd,
+                        uuid_utoa (fd->inode->gfid));
+                goto out;
+        }
+
+out:
+        dict_unref (lockinfo);
+
+        return op_ret;
+}
+
+int32_t
+pl_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
+              int32_t flags, dict_t *xdata)
+{
+        int32_t  op_ret       = 0, op_errno = 0;
+        void    *lockinfo_buf = NULL;
+        int      len          = 0;
+
+        op_ret = dict_get_ptr_and_len (dict, GF_XATTR_LOCKINFO_KEY,
+                                       &lockinfo_buf, &len);
+        if (lockinfo_buf == NULL) {
+                goto usual;
+        }
+
+        op_ret = pl_fsetxattr_handle_lockinfo (frame, fd, lockinfo_buf, len,
+                                               &op_errno);
+        if (op_ret < 0) {
+                goto unwind;
         }
 
+usual:
+        STACK_WIND (frame, default_fsetxattr_cbk, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fsetxattr, fd, dict, flags, xdata);
+        return 0;
+
 unwind:
-	STACK_UNWIND_STRICT (opendir,
+        STACK_UNWIND_STRICT (fsetxattr, frame, op_ret, op_errno, NULL);
+        return 0;
+}
+
+int32_t
+pl_opendir_cbk (call_frame_t *frame,
+                void *cookie,
+                xlator_t *this,
+                int32_t op_ret,
+                int32_t op_errno,
+                fd_t *fd, dict_t *xdata)
+{
+        pl_fdctx_t *fdctx = NULL;
+
+        if (op_ret < 0)
+                goto unwind;
+
+        fdctx = pl_check_n_create_fdctx (this, fd);
+        if (!fdctx) {
+                op_errno = ENOMEM;
+                op_ret   = -1;
+                goto unwind;
+        }
+
+unwind:
+        STACK_UNWIND_STRICT (opendir,
                              frame,
                              op_ret,
                              op_errno,
-                             fd);
-	return 0;
+                             fd, xdata);
+        return 0;
 }
 
-int32_t 
+int32_t
 pl_opendir (call_frame_t *frame, xlator_t *this,
-	     loc_t *loc, fd_t *fd)
+             loc_t *loc, fd_t *fd, dict_t *xdata)
 {
-	STACK_WIND (frame,
-		    pl_opendir_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->opendir,
-		    loc, fd);
-	return 0;
+        STACK_WIND (frame,
+                    pl_opendir_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->opendir,
+                    loc, fd, xdata);
+        return 0;
 
 }
 
 int
 pl_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-              int32_t op_ret, int32_t op_errno)
+              int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno);
+        STACK_UNWIND_STRICT (flush, frame, op_ret, op_errno, xdata);
 
         return 0;
 }
@@ -430,38 +958,35 @@ pl_flush_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 
 int
 pl_flush (call_frame_t *frame, xlator_t *this,
-          fd_t *fd)
+          fd_t *fd, dict_t *xdata)
 {
-        pl_inode_t            *pl_inode = NULL;
-        uint64_t              owner     = -1;
-
-        owner = frame->root->lk_owner;
+        pl_inode_t *pl_inode = NULL;
 
         pl_inode = pl_inode_get (this, fd->inode);
 
         if (!pl_inode) {
                 gf_log (this->name, GF_LOG_DEBUG, "Could not get inode.");
-                STACK_UNWIND_STRICT (flush, frame, -1, EBADFD);
+                STACK_UNWIND_STRICT (flush, frame, -1, EBADFD, NULL);
                 return 0;
         }
 
         pl_trace_flush (this, frame, fd);
 
-        if (owner == 0) {
+        if (frame->root->lk_owner.len == 0) {
                 /* Handle special case when protocol/server sets lk-owner to zero.
                  * This usually happens due to a client disconnection. Hence, free
                  * all locks opened with this fd.
                  */
                 gf_log (this->name, GF_LOG_TRACE,
-			"Releasing all locks with fd %p", fd);
+                        "Releasing all locks with fd %p", fd);
                 delete_locks_of_fd (this, pl_inode, fd);
                 goto wind;
 
         }
         pthread_mutex_lock (&pl_inode->mutex);
         {
-                __delete_locks_of_owner (pl_inode, frame->root->trans,
-                                         owner);
+                __delete_locks_of_owner (pl_inode, frame->root->client,
+                                         &frame->root->lk_owner);
         }
         pthread_mutex_unlock (&pl_inode->mutex);
 
@@ -471,46 +996,29 @@ pl_flush (call_frame_t *frame, xlator_t *this,
 
 wind:
         STACK_WIND (frame, pl_flush_cbk, FIRST_CHILD(this),
-                    FIRST_CHILD(this)->fops->flush, fd);
+                    FIRST_CHILD(this)->fops->flush, fd, xdata);
         return 0;
 }
 
 
 int
 pl_open_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-             int32_t op_ret, int32_t op_errno, fd_t *fd)
+             int32_t op_ret, int32_t op_errno, fd_t *fd, dict_t *xdata)
 {
         pl_fdctx_t *fdctx = NULL;
-        int ret = -1;
 
         if (op_ret < 0)
                 goto unwind;
 
-        fdctx = pl_get_fdctx (this, fd);
+        fdctx = pl_check_n_create_fdctx (this, fd);
         if (!fdctx) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "fdctx not present");
-                fdctx = pl_new_fdctx ();
-                if (!fdctx) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
-                        op_errno = ENOMEM;
-                        op_ret   = -1;
-                        goto unwind;
-                }
-
-                ret = pl_set_fdctx (this, fd, fdctx);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "could not set fdctx");
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+                op_errno = ENOMEM;
+                op_ret   = -1;
+                goto unwind;
         }
 
 unwind:
-        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd);
+        STACK_UNWIND_STRICT (open, frame, op_ret, op_errno, fd, xdata);
 
         return 0;
 }
@@ -518,12 +1026,11 @@ unwind:
 
 int
 pl_open (call_frame_t *frame, xlator_t *this, loc_t *loc, int32_t flags,
-         fd_t *fd, int32_t wbflags)
+         fd_t *fd, dict_t *xdata)
 {
-        /* why isn't O_TRUNC being handled ? */
         STACK_WIND (frame, pl_open_cbk,
                     FIRST_CHILD(this), FIRST_CHILD(this)->fops->open,
-                    loc, flags & ~O_TRUNC, fd, wbflags);
+                    loc, flags, fd, xdata);
 
         return 0;
 }
@@ -533,40 +1040,23 @@ int
 pl_create_cbk (call_frame_t *frame, void *cookie,
                xlator_t *this, int32_t op_ret, int32_t op_errno,
                fd_t *fd, inode_t *inode, struct iatt *buf,
-               struct iatt *preparent, struct iatt *postparent)
+               struct iatt *preparent, struct iatt *postparent, dict_t *xdata)
 {
         pl_fdctx_t *fdctx = NULL;
-        int ret = -1;
 
         if (op_ret < 0)
                 goto unwind;
 
-        fdctx = pl_get_fdctx (this, fd);
+        fdctx = pl_check_n_create_fdctx (this, fd);
         if (!fdctx) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        "fdctx not present");
-                fdctx = pl_new_fdctx ();
-                if (!fdctx) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "Out of memory");
-                        op_errno = ENOMEM;
-                        op_ret   = -1;
-                        goto unwind;
-                }
-
-                ret = pl_set_fdctx (this, fd, fdctx);
-                if (ret) {
-                        gf_log (this->name, GF_LOG_ERROR,
-                                "could not set fdctx");
-                        op_ret = -1;
-                        op_errno = ENOMEM;
-                        goto unwind;
-                }
+                op_errno = ENOMEM;
+                op_ret   = -1;
+                goto unwind;
         }
 
 unwind:
         STACK_UNWIND_STRICT (create, frame, op_ret, op_errno, fd, inode, buf,
-                             preparent, postparent);
+                             preparent, postparent, xdata);
 
         return 0;
 }
@@ -574,11 +1064,12 @@ unwind:
 
 int
 pl_create (call_frame_t *frame, xlator_t *this,
-           loc_t *loc, int32_t flags, mode_t mode, fd_t *fd, dict_t *params)
+           loc_t *loc, int32_t flags, mode_t mode, mode_t umask, fd_t *fd,
+           dict_t *xdata)
 {
         STACK_WIND (frame, pl_create_cbk,
                     FIRST_CHILD (this), FIRST_CHILD (this)->fops->create,
-                    loc, flags, mode, fd, params);
+                    loc, flags, mode, umask, fd, xdata);
         return 0;
 }
 
@@ -587,10 +1078,10 @@ int
 pl_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
               int32_t op_ret, int32_t op_errno,
               struct iovec *vector, int32_t count, struct iatt *stbuf,
-              struct iobref *iobref)
+              struct iobref *iobref, dict_t *xdata)
 {
         STACK_UNWIND_STRICT (readv, frame, op_ret, op_errno,
-                             vector, count, stbuf, iobref);
+                             vector, count, stbuf, iobref, xdata);
 
         return 0;
 }
@@ -598,9 +1089,10 @@ pl_readv_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
 int
 pl_writev_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
                int32_t op_ret, int32_t op_errno, struct iatt *prebuf,
-               struct iatt *postbuf)
+               struct iatt *postbuf, dict_t *xdata)
 {
-        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf);
+        STACK_UNWIND_STRICT (writev, frame, op_ret, op_errno, prebuf, postbuf,
+                             xdata);
 
         return 0;
 }
@@ -658,12 +1150,12 @@ __rw_allowable (pl_inode_t *pl_inode, posix_lock_t *region,
 
 
 int
-pl_readv_cont (call_frame_t *frame, xlator_t *this,
-               fd_t *fd, size_t size, off_t offset)
+pl_readv_cont (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+               off_t offset, uint32_t flags, dict_t *xdata)
 {
         STACK_WIND (frame, pl_readv_cbk,
                     FIRST_CHILD (this), FIRST_CHILD (this)->fops->readv,
-                    fd, size, offset);
+                    fd, size, offset, flags, xdata);
 
         return 0;
 }
@@ -671,7 +1163,7 @@ pl_readv_cont (call_frame_t *frame, xlator_t *this,
 
 int
 pl_readv (call_frame_t *frame, xlator_t *this,
-          fd_t *fd, size_t size, off_t offset)
+          fd_t *fd, size_t size, off_t offset, uint32_t flags, dict_t *xdata)
 {
         posix_locks_private_t *priv = NULL;
         pl_inode_t            *pl_inode = NULL;
@@ -688,7 +1180,7 @@ pl_readv (call_frame_t *frame, xlator_t *this,
         if (priv->mandatory && pl_inode->mandatory) {
                 region.fl_start   = offset;
                 region.fl_end     = offset + size - 1;
-                region.transport  = frame->root->trans;
+                region.client     = frame->root->client;
                 region.fd_num     = fd_to_fdnum(fd);
                 region.client_pid = frame->root->pid;
                 region.owner      = frame->root->lk_owner;
@@ -709,21 +1201,18 @@ pl_readv (call_frame_t *frame, xlator_t *this,
                                 goto unlock;
                         }
 
-			rw = GF_CALLOC (1, sizeof (*rw),
+                        rw = GF_CALLOC (1, sizeof (*rw),
                                         gf_locks_mt_pl_rw_req_t);
                         if (!rw) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Out of memory.");
                                 op_errno = ENOMEM;
                                 op_ret = -1;
                                 goto unlock;
                         }
 
                         rw->stub = fop_readv_stub (frame, pl_readv_cont,
-                                                   fd, size, offset);
+                                                   fd, size, offset, flags,
+                                                   xdata);
                         if (!rw->stub) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Out of memory.");
                                 op_errno = ENOMEM;
                                 op_ret = -1;
                                 GF_FREE (rw);
@@ -742,12 +1231,12 @@ pl_readv (call_frame_t *frame, xlator_t *this,
         if (wind_needed) {
                 STACK_WIND (frame, pl_readv_cbk,
                             FIRST_CHILD (this), FIRST_CHILD (this)->fops->readv,
-                            fd, size, offset);
+                            fd, size, offset, flags, xdata);
         }
 
         if (op_ret == -1)
                 STACK_UNWIND_STRICT (readv, frame, -1, op_errno,
-                                     NULL, 0, NULL, NULL);
+                                     NULL, 0, NULL, NULL, NULL);
 
         return 0;
 }
@@ -756,11 +1245,11 @@ pl_readv (call_frame_t *frame, xlator_t *this,
 int
 pl_writev_cont (call_frame_t *frame, xlator_t *this, fd_t *fd,
                 struct iovec *vector, int count, off_t offset,
-                struct iobref *iobref)
+                uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         STACK_WIND (frame, pl_writev_cbk,
                     FIRST_CHILD (this), FIRST_CHILD (this)->fops->writev,
-                    fd, vector, count, offset, iobref);
+                    fd, vector, count, offset, flags, iobref, xdata);
 
         return 0;
 }
@@ -769,7 +1258,7 @@ pl_writev_cont (call_frame_t *frame, xlator_t *this, fd_t *fd,
 int
 pl_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
            struct iovec *vector, int32_t count, off_t offset,
-           struct iobref *iobref)
+           uint32_t flags, struct iobref *iobref, dict_t *xdata)
 {
         posix_locks_private_t *priv = NULL;
         pl_inode_t            *pl_inode = NULL;
@@ -779,14 +1268,13 @@ pl_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         int                    op_errno = 0;
         char                   wind_needed = 1;
 
-
         priv = this->private;
         pl_inode = pl_inode_get (this, fd->inode);
 
         if (priv->mandatory && pl_inode->mandatory) {
                 region.fl_start   = offset;
                 region.fl_end     = offset + iov_length (vector, count) - 1;
-                region.transport  = frame->root->trans;
+                region.client     = frame->root->client;
                 region.fd_num     = fd_to_fdnum(fd);
                 region.client_pid = frame->root->pid;
                 region.owner      = frame->root->lk_owner;
@@ -807,11 +1295,9 @@ pl_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
                                 goto unlock;
                         }
 
-			rw = GF_CALLOC (1, sizeof (*rw),
+                        rw = GF_CALLOC (1, sizeof (*rw),
                                         gf_locks_mt_pl_rw_req_t);
                         if (!rw) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Out of memory.");
                                 op_errno = ENOMEM;
                                 op_ret = -1;
                                 goto unlock;
@@ -819,10 +1305,8 @@ pl_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
 
                         rw->stub = fop_writev_stub (frame, pl_writev_cont,
                                                     fd, vector, count, offset,
-                                                    iobref);
+                                                    flags, iobref, xdata);
                         if (!rw->stub) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Out of memory.");
                                 op_errno = ENOMEM;
                                 op_ret = -1;
                                 GF_FREE (rw);
@@ -841,10 +1325,11 @@ pl_writev (call_frame_t *frame, xlator_t *this, fd_t *fd,
         if (wind_needed)
                 STACK_WIND (frame, pl_writev_cbk,
                             FIRST_CHILD (this), FIRST_CHILD (this)->fops->writev,
-                            fd, vector, count, offset, iobref);
+                            fd, vector, count, offset, flags, iobref, xdata);
 
         if (op_ret == -1)
-                STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL);
+                STACK_UNWIND_STRICT (writev, frame, -1, op_errno, NULL, NULL,
+                                     NULL);
 
         return 0;
 }
@@ -856,7 +1341,7 @@ __fd_has_locks (pl_inode_t *pl_inode, fd_t *fd)
         posix_lock_t *l     = NULL;
 
         list_for_each_entry (l, &pl_inode->ext_list, list) {
-                if ((l->fd_num == fd_to_fdnum(fd))) {
+                if (l->fd_num == fd_to_fdnum(fd)) {
                         found = 1;
                         break;
                 }
@@ -870,8 +1355,8 @@ lock_dup (posix_lock_t *lock)
 {
         posix_lock_t *new_lock = NULL;
 
-        new_lock = new_posix_lock (&lock->user_flock, lock->transport,
-                                   lock->client_pid, lock->owner,
+        new_lock = new_posix_lock (&lock->user_flock, lock->client,
+                                   lock->client_pid, &lock->owner,
                                    (fd_t *)lock->fd_num);
         return new_lock;
 }
@@ -885,11 +1370,9 @@ __dup_locks_to_fdctx (pl_inode_t *pl_inode, fd_t *fd,
         int ret = 0;
 
         list_for_each_entry (l, &pl_inode->ext_list, list) {
-                if ((l->fd_num == fd_to_fdnum(fd))) {
+                if (l->fd_num == fd_to_fdnum(fd)) {
                         duplock = lock_dup (l);
                         if (!duplock) {
-                                gf_log (THIS->name, GF_LOG_DEBUG,
-                                        "Out of memory");
                                 ret = -1;
                                 break;
                         }
@@ -975,6 +1458,7 @@ out:
 
         return ret;
 }
+
 static int
 pl_getlk_fd (xlator_t *this, pl_inode_t *pl_inode,
              fd_t *fd, posix_lock_t *reqlock)
@@ -996,7 +1480,7 @@ pl_getlk_fd (xlator_t *this, pl_inode_t *pl_inode,
                         "There are active locks on fd");
 
                 ret = fd_ctx_get (fd, this, &tmp);
-                fdctx = (pl_fdctx_t *) tmp;
+                fdctx = (pl_fdctx_t *)(long) tmp;
 
                 if (list_empty (&fdctx->locks_list)) {
                         gf_log (this->name, GF_LOG_TRACE,
@@ -1004,8 +1488,6 @@ pl_getlk_fd (xlator_t *this, pl_inode_t *pl_inode,
 
                         ret = __copy_locks_to_fdctx (pl_inode, fd, fdctx);
                         if (ret) {
-                                gf_log (this->name, GF_LOG_ERROR,
-                                        "Out of memory");
                                 goto unlock;
                         }
 
@@ -1031,22 +1513,15 @@ unlock:
 
 int
 pl_lk (call_frame_t *frame, xlator_t *this,
-       fd_t *fd, int32_t cmd, struct gf_flock *flock)
+       fd_t *fd, int32_t cmd, struct gf_flock *flock, dict_t *xdata)
 {
-        void                  *transport = NULL;
-        pid_t                  client_pid = 0;
-        uint64_t               owner      = 0;
-        pl_inode_t            *pl_inode = NULL;
-        int                    op_ret = 0;
-        int                    op_errno = 0;
-        int                    can_block = 0;
-        posix_lock_t          *reqlock = NULL;
-        posix_lock_t          *conf = NULL;
-        int                    ret = 0;
-
-        transport  = frame->root->trans;
-        client_pid = frame->root->pid;
-        owner      = frame->root->lk_owner;
+        pl_inode_t   *pl_inode   = NULL;
+        int           op_ret     = 0;
+        int           op_errno   = 0;
+        int           can_block  = 0;
+        posix_lock_t *reqlock    = NULL;
+        posix_lock_t *conf       = NULL;
+        int           ret        = 0;
 
         if ((flock->l_start < 0) || (flock->l_len < 0)) {
                 op_ret = -1;
@@ -1056,19 +1531,15 @@ pl_lk (call_frame_t *frame, xlator_t *this,
 
         pl_inode = pl_inode_get (this, fd->inode);
         if (!pl_inode) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
                 op_ret = -1;
                 op_errno = ENOMEM;
                 goto unwind;
         }
 
-        reqlock = new_posix_lock (flock, transport, client_pid,
-                                  owner, fd);
+        reqlock = new_posix_lock (flock, frame->root->client, frame->root->pid,
+                                  &frame->root->lk_owner, fd);
 
         if (!reqlock) {
-                gf_log (this->name, GF_LOG_ERROR,
-                        "Out of memory.");
                 op_ret = -1;
                 op_errno = ENOMEM;
                 goto unwind;
@@ -1175,7 +1646,7 @@ pl_lk (call_frame_t *frame, xlator_t *this,
                                 can_block);
 
                 if (ret == -1) {
-                        if (can_block) {
+                        if ((can_block) && (F_UNLCK != flock->l_type)) {
                                 pl_trace_block (this, frame, fd, NULL, cmd, flock, NULL);
                                 goto out;
                         }
@@ -1183,13 +1654,22 @@ pl_lk (call_frame_t *frame, xlator_t *this,
                         op_ret = -1;
                         op_errno = EAGAIN;
                         __destroy_lock (reqlock);
+
+                } else if ((0 == ret) && (F_UNLCK == flock->l_type)) {
+                        /* For NLM's last "unlock on fd" detection */
+                        if (pl_locks_by_fd (pl_inode, fd))
+                                flock->l_type = F_RDLCK;
+                        else
+                                flock->l_type = F_UNLCK;
                 }
         }
 
 unwind:
         pl_trace_out (this, frame, fd, NULL, cmd, flock, op_ret, op_errno, NULL);
         pl_update_refkeeper (this, fd->inode);
-        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, flock);
+
+
+        STACK_UNWIND_STRICT (lk, frame, op_ret, op_errno, flock, xdata);
 out:
         return 0;
 }
@@ -1204,120 +1684,121 @@ pl_forget (xlator_t *this,
 
         posix_lock_t *ext_tmp = NULL;
         posix_lock_t *ext_l   = NULL;
-	struct list_head posixlks_released;
+        struct list_head posixlks_released;
 
         pl_inode_lock_t *ino_tmp = NULL;
         pl_inode_lock_t *ino_l   = NULL;
-	struct list_head inodelks_released;
+        struct list_head inodelks_released;
 
         pl_rw_req_t *rw_tmp = NULL;
         pl_rw_req_t *rw_req = NULL;
 
         pl_entry_lock_t *entry_tmp = NULL;
         pl_entry_lock_t *entry_l   = NULL;
-	struct list_head entrylks_released;
+        struct list_head entrylks_released;
 
         pl_dom_list_t *dom = NULL;
         pl_dom_list_t *dom_tmp = NULL;
 
-	INIT_LIST_HEAD (&posixlks_released);
-	INIT_LIST_HEAD (&inodelks_released);
-	INIT_LIST_HEAD (&entrylks_released);
+        INIT_LIST_HEAD (&posixlks_released);
+        INIT_LIST_HEAD (&inodelks_released);
+        INIT_LIST_HEAD (&entrylks_released);
 
         pl_inode = pl_inode_get (this, inode);
 
-	pthread_mutex_lock (&pl_inode->mutex);
-	{
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
 
-		if (!list_empty (&pl_inode->rw_list)) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Pending R/W requests found, releasing.");
+                if (!list_empty (&pl_inode->rw_list)) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Pending R/W requests found, releasing.");
 
-			list_for_each_entry_safe (rw_req, rw_tmp, &pl_inode->rw_list,
-						  list) {
+                        list_for_each_entry_safe (rw_req, rw_tmp, &pl_inode->rw_list,
+                                                  list) {
 
-				list_del (&rw_req->list);
-				GF_FREE (rw_req);
-			}
-		}
+                                list_del (&rw_req->list);
+                                GF_FREE (rw_req);
+                        }
+                }
 
-		if (!list_empty (&pl_inode->ext_list)) {
-			gf_log (this->name, GF_LOG_WARNING,
-				"Pending fcntl locks found, releasing.");
+                if (!list_empty (&pl_inode->ext_list)) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "Pending fcntl locks found, releasing.");
 
-			list_for_each_entry_safe (ext_l, ext_tmp, &pl_inode->ext_list,
-						  list) {
+                        list_for_each_entry_safe (ext_l, ext_tmp, &pl_inode->ext_list,
+                                                  list) {
 
-				__delete_lock (pl_inode, ext_l);
-				if (ext_l->blocked) {
-					list_add_tail (&ext_l->list, &posixlks_released);
-					continue;
-				}
-				__destroy_lock (ext_l);
-			}
-		}
+                                __delete_lock (pl_inode, ext_l);
+                                if (ext_l->blocked) {
+                                        list_add_tail (&ext_l->list, &posixlks_released);
+                                        continue;
+                                }
+                                __destroy_lock (ext_l);
+                        }
+                }
 
 
-		list_for_each_entry_safe (dom, dom_tmp, &pl_inode->dom_list, inode_list) {
+                list_for_each_entry_safe (dom, dom_tmp, &pl_inode->dom_list, inode_list) {
 
-			if (!list_empty (&dom->inodelk_list)) {
-				gf_log (this->name, GF_LOG_WARNING,
-					"Pending inode locks found, releasing.");
+                        if (!list_empty (&dom->inodelk_list)) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Pending inode locks found, releasing.");
 
-				list_for_each_entry_safe (ino_l, ino_tmp, &dom->inodelk_list, list) {
-					__delete_inode_lock (ino_l);
-					__destroy_inode_lock (ino_l);
-				}
+                                list_for_each_entry_safe (ino_l, ino_tmp, &dom->inodelk_list, list) {
+                                        __delete_inode_lock (ino_l);
+                                        __pl_inodelk_unref (ino_l);
+                                }
 
-				list_splice_init (&dom->blocked_inodelks, &inodelks_released);
-				
+                                list_splice_init (&dom->blocked_inodelks, &inodelks_released);
 
-			}
-			if (!list_empty (&dom->entrylk_list)) {
-				gf_log (this->name, GF_LOG_WARNING,
-					"Pending entry locks found, releasing.");
 
-				list_for_each_entry_safe (entry_l, entry_tmp, &dom->entrylk_list, domain_list) {
-					list_del_init (&entry_l->domain_list);
+                        }
+                        if (!list_empty (&dom->entrylk_list)) {
+                                gf_log (this->name, GF_LOG_WARNING,
+                                        "Pending entry locks found, releasing.");
 
-					if (entry_l->basename)
-						GF_FREE ((char *)entry_l->basename);
-					GF_FREE (entry_l);
-				}
+                                list_for_each_entry_safe (entry_l, entry_tmp, &dom->entrylk_list, domain_list) {
+                                        list_del_init (&entry_l->domain_list);
 
-				list_splice_init (&dom->blocked_entrylks, &entrylks_released);
-			}
+                                        GF_FREE ((char *)entry_l->basename);
+                                        GF_FREE (entry_l->connection_id);
+                                        GF_FREE (entry_l);
+                                }
 
-			list_del (&dom->inode_list);
-			gf_log ("posix-locks", GF_LOG_TRACE,
-				" Cleaning up domain: %s", dom->domain);
-			GF_FREE ((char *)(dom->domain));
-			GF_FREE (dom);
-		}
+                                list_splice_init (&dom->blocked_entrylks, &entrylks_released);
+                        }
+
+                        list_del (&dom->inode_list);
+                        gf_log ("posix-locks", GF_LOG_TRACE,
+                                " Cleaning up domain: %s", dom->domain);
+                        GF_FREE ((char *)(dom->domain));
+                        GF_FREE (dom);
+                }
 
-	}
-	pthread_mutex_unlock (&pl_inode->mutex);
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
 
-	list_for_each_entry_safe (ext_l, ext_tmp, &posixlks_released, list) {
+        list_for_each_entry_safe (ext_l, ext_tmp, &posixlks_released, list) {
 
-		STACK_UNWIND_STRICT (lk, ext_l->frame, -1, 0, &ext_l->user_flock);
-		__destroy_lock (ext_l);
-	}
+                STACK_UNWIND_STRICT (lk, ext_l->frame, -1, 0,
+                                     &ext_l->user_flock, NULL);
+                __destroy_lock (ext_l);
+        }
 
-	list_for_each_entry_safe (ino_l, ino_tmp, &inodelks_released, blocked_locks) {
+        list_for_each_entry_safe (ino_l, ino_tmp, &inodelks_released, blocked_locks) {
 
-		STACK_UNWIND_STRICT (inodelk, ino_l->frame, -1, 0);
-		__destroy_inode_lock (ino_l);
-	}
+                STACK_UNWIND_STRICT (inodelk, ino_l->frame, -1, 0, NULL);
+                __pl_inodelk_unref (ino_l);
+        }
 
-	list_for_each_entry_safe (entry_l, entry_tmp, &entrylks_released, blocked_locks) {
+        list_for_each_entry_safe (entry_l, entry_tmp, &entrylks_released, blocked_locks) {
 
-		STACK_UNWIND_STRICT (entrylk, entry_l->frame, -1, 0);
-		if (entry_l->basename)
-			GF_FREE ((char *)entry_l->basename);
-		GF_FREE (entry_l);
+                STACK_UNWIND_STRICT (entrylk, entry_l->frame, -1, 0, NULL);
+                GF_FREE ((char *)entry_l->basename);
+                GF_FREE (entry_l->connection_id);
+                GF_FREE (entry_l);
 
-	}
+        }
 
         GF_FREE (pl_inode);
 
@@ -1330,8 +1811,14 @@ pl_release (xlator_t *this, fd_t *fd)
         pl_inode_t *pl_inode     = NULL;
         uint64_t    tmp_pl_inode = 0;
         int         ret          = -1;
+        uint64_t    tmp          = 0;
+        pl_fdctx_t *fdctx        = NULL;
+
+        if (fd == NULL) {
+                goto out;
+        }
 
-	ret = inode_ctx_get (fd->inode, this, &tmp_pl_inode);
+        ret = inode_ctx_get (fd->inode, this, &tmp_pl_inode);
         if (ret != 0)
                 goto out;
 
@@ -1343,11 +1830,48 @@ pl_release (xlator_t *this, fd_t *fd)
                 "Releasing all locks with fd %p", fd);
 
         delete_locks_of_fd (this, pl_inode, fd);
+        pl_update_refkeeper (this, fd->inode);
+
+        ret = fd_ctx_del (fd, this, &tmp);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Could not get fdctx");
+                goto out;
+        }
 
+        fdctx = (pl_fdctx_t *)(long)tmp;
+
+        GF_FREE (fdctx);
 out:
         return ret;
 }
-static int32_t
+
+int
+pl_releasedir (xlator_t *this, fd_t *fd)
+{
+        int         ret          = -1;
+        uint64_t    tmp          = 0;
+        pl_fdctx_t *fdctx        = NULL;
+
+        if (fd == NULL) {
+                goto out;
+        }
+
+        ret = fd_ctx_del (fd, this, &tmp);
+        if (ret) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "Could not get fdctx");
+                goto out;
+        }
+
+        fdctx = (pl_fdctx_t *)(long)tmp;
+
+        GF_FREE (fdctx);
+out:
+        return ret;
+}
+
+int32_t
 __get_posixlk_count (xlator_t *this, pl_inode_t *pl_inode)
 {
         posix_lock_t *lock   = NULL;
@@ -1355,16 +1879,6 @@ __get_posixlk_count (xlator_t *this, pl_inode_t *pl_inode)
 
         list_for_each_entry (lock, &pl_inode->ext_list, list) {
 
-			gf_log (this->name, GF_LOG_DEBUG,
-                                " XATTR DEBUG"
-				"%s (pid=%d) (lk-owner=%"PRIu64") %"PRId64" - %"PRId64" state: %s",
-				lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-				lock->client_pid,
-                                lock->owner,
-				lock->user_flock.l_start,
-				lock->user_flock.l_len,
-                                lock->blocked == 1 ? "Blocked" : "Active");
-
                 count++;
         }
 
@@ -1397,6 +1911,24 @@ out:
 }
 
 void
+pl_parent_entrylk_xattr_fill (xlator_t *this, inode_t *parent,
+                              char *basename, dict_t *dict)
+{
+        uint32_t         entrylk = 0;
+        int             ret     = -1;
+
+        if (!parent || !basename || !strlen (basename))
+                goto out;
+        entrylk = check_entrylk_on_basename (this, parent, basename);
+out:
+        ret = dict_set_uint32 (dict, GLUSTERFS_PARENT_ENTRYLK, entrylk);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        " dict_set failed on key %s", GLUSTERFS_PARENT_ENTRYLK);
+        }
+}
+
+void
 pl_entrylk_xattr_fill (xlator_t *this, inode_t *inode,
                        dict_t *dict)
 {
@@ -1413,19 +1945,34 @@ pl_entrylk_xattr_fill (xlator_t *this, inode_t *inode,
 }
 
 void
-pl_inodelk_xattr_fill (xlator_t *this, inode_t *inode,
-                       dict_t *dict)
+pl_inodelk_xattr_fill (xlator_t *this, inode_t *inode, dict_t *dict,
+                       gf_boolean_t per_dom)
 {
         int32_t     count = 0;
         int         ret   = -1;
+        char        *domname = NULL;
+
+
+        if (per_dom){
+                ret = dict_get_str (dict, GLUSTERFS_INODELK_DOM_COUNT,
+                                    &domname);
+                if (ret) {
+                        gf_log (this->name, GF_LOG_ERROR, "Failed to get "
+                                "value for key %s",GLUSTERFS_INODELK_DOM_COUNT);
+                        goto out;
+                }
+        }
+
+        count = get_inodelk_count (this, inode, domname);
 
-        count = get_inodelk_count (this, inode);
         ret = dict_set_int32 (dict, GLUSTERFS_INODELK_COUNT, count);
         if (ret < 0) {
-                gf_log (this->name, GF_LOG_DEBUG,
-                        " dict_set failed on key %s", GLUSTERFS_INODELK_COUNT);
+                gf_log (this->name, GF_LOG_DEBUG, "Failed to set count for "
+                        "key %s", GLUSTERFS_INODELK_COUNT);
         }
 
+out:
+        return;
 }
 
 void
@@ -1446,58 +1993,63 @@ pl_posixlk_xattr_fill (xlator_t *this, inode_t *inode,
 
 int32_t
 pl_lookup_cbk (call_frame_t *frame,
-		    void *cookie,
-		    xlator_t *this,
-		    int32_t op_ret,
-		    int32_t op_errno,
-		    inode_t *inode,
-		    struct iatt *buf,
-                    dict_t *dict,
+                    void *cookie,
+                    xlator_t *this,
+                    int32_t op_ret,
+                    int32_t op_errno,
+                    inode_t *inode,
+                    struct iatt *buf,
+                    dict_t *xdata,
                     struct iatt *postparent)
 {
         pl_local_t *local = NULL;
 
-        if (!frame->local) {
-                goto out;
-        }
+        GF_VALIDATE_OR_GOTO (this->name, frame->local, out);
 
-        if (op_ret) {
+        if (op_ret)
                 goto out;
-        }
 
         local = frame->local;
 
+        if (local->parent_entrylk_req)
+                pl_parent_entrylk_xattr_fill (this, local->loc.parent,
+                                              (char*)local->loc.name, xdata);
         if (local->entrylk_count_req)
-                pl_entrylk_xattr_fill (this, inode, dict);
+                pl_entrylk_xattr_fill (this, inode, xdata);
         if (local->inodelk_count_req)
-                pl_inodelk_xattr_fill (this, inode, dict);
+                pl_inodelk_xattr_fill (this, inode, xdata, _gf_false);
+        if (local->inodelk_dom_count_req)
+                pl_inodelk_xattr_fill (this, inode, xdata, _gf_true);
         if (local->posixlk_count_req)
-                pl_posixlk_xattr_fill (this, inode, dict);
+                pl_posixlk_xattr_fill (this, inode, xdata);
 
 
+out:
+        local = frame->local;
         frame->local = NULL;
 
-        if (local != NULL)
-                GF_FREE (local);
+        if (local != NULL) {
+                loc_wipe (&local->loc);
+                mem_put (local);
+        }
 
-out:
-	STACK_UNWIND_STRICT (
+        STACK_UNWIND_STRICT (
                      lookup,
                      frame,
-		      op_ret,
-		      op_errno,
-		      inode,
-		      buf,
-                      dict,
-                      postparent);
-	return 0;
+                     op_ret,
+                     op_errno,
+                     inode,
+                     buf,
+                     xdata,
+                     postparent);
+        return 0;
 }
 
 int32_t
 pl_lookup (call_frame_t *frame,
-		xlator_t *this,
-		loc_t *loc,
-		dict_t *xattr_req)
+           xlator_t *this,
+           loc_t *loc,
+           dict_t *xdata)
 {
         pl_local_t *local  = NULL;
         int         ret    = -1;
@@ -1506,42 +2058,114 @@ pl_lookup (call_frame_t *frame,
         VALIDATE_OR_GOTO (this, out);
         VALIDATE_OR_GOTO (loc, out);
 
-        local = GF_CALLOC (1, sizeof (*local), gf_locks_mt_pl_local_t);
-        if (!local) {
-                ret = -1;
-                gf_log (this->name, GF_LOG_ERROR,
-                        " Out of memory");
-                goto out;
+        local = mem_get0 (this->local_pool);
+        GF_VALIDATE_OR_GOTO (this->name, local, out);
+
+        if (xdata) {
+                if (dict_get (xdata, GLUSTERFS_ENTRYLK_COUNT))
+                        local->entrylk_count_req = 1;
+                if (dict_get (xdata, GLUSTERFS_INODELK_COUNT))
+                        local->inodelk_count_req = 1;
+                if (dict_get (xdata, GLUSTERFS_INODELK_DOM_COUNT))
+                        local->inodelk_dom_count_req = 1;
+                if (dict_get (xdata, GLUSTERFS_POSIXLK_COUNT))
+                        local->posixlk_count_req = 1;
+                if (dict_get (xdata, GLUSTERFS_PARENT_ENTRYLK))
+                        local->parent_entrylk_req = 1;
         }
 
-        if (dict_get (xattr_req, GLUSTERFS_ENTRYLK_COUNT))
-                local->entrylk_count_req = 1;
-        if (dict_get (xattr_req, GLUSTERFS_INODELK_COUNT))
-                local->inodelk_count_req = 1;
-        if (dict_get (xattr_req, GLUSTERFS_POSIXLK_COUNT))
-                local->posixlk_count_req = 1;
-
         frame->local = local;
+        loc_copy (&local->loc, loc);
 
-	STACK_WIND (frame,
-		    pl_lookup_cbk,
-		    FIRST_CHILD(this),
-		    FIRST_CHILD(this)->fops->lookup,
-		    loc,
-		    xattr_req);
+        STACK_WIND (frame,
+                    pl_lookup_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup,
+                    loc, xdata);
         ret = 0;
 out:
         if (ret == -1)
-                STACK_UNWIND_STRICT (lookup, frame, -1, 0, NULL, NULL, NULL, NULL);
+                STACK_UNWIND_STRICT (lookup, frame, -1, 0, NULL,
+                                     NULL, NULL, NULL);
 
-	return 0;
+        return 0;
+}
+int
+pl_readdirp_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                 int op_ret, int op_errno, gf_dirent_t *entries, dict_t *xdata)
+{
+        pl_local_t *local  = NULL;
+        gf_dirent_t *entry = NULL;
+
+        local = frame->local;
+
+        if (op_ret <= 0)
+                goto unwind;
+
+        list_for_each_entry (entry, &entries->list, list) {
+                if (local->entrylk_count_req)
+                        pl_entrylk_xattr_fill (this, entry->inode, entry->dict);
+                if (local->inodelk_count_req)
+                        pl_inodelk_xattr_fill (this, entry->inode, entry->dict,
+                                               _gf_false);
+                if (local->inodelk_dom_count_req)
+                        pl_inodelk_xattr_fill (this, entry->inode, entry->dict,
+                                               _gf_true);
+                if (local->posixlk_count_req)
+                        pl_posixlk_xattr_fill (this, entry->inode, entry->dict);
+        }
+
+unwind:
+        frame->local = NULL;
+        STACK_UNWIND_STRICT (readdirp, frame, op_ret, op_errno, entries, xdata);
+
+        if (local)
+                mem_put (local);
+
+        return 0;
 }
 
+int
+pl_readdirp (call_frame_t *frame, xlator_t *this, fd_t *fd, size_t size,
+             off_t offset, dict_t *dict)
+{
+        pl_local_t *local  = NULL;
+
+        local = mem_get0 (this->local_pool);
+        GF_VALIDATE_OR_GOTO (this->name, local, out);
+
+        if (dict) {
+                if (dict_get (dict, GLUSTERFS_ENTRYLK_COUNT))
+                        local->entrylk_count_req = 1;
+                if (dict_get (dict, GLUSTERFS_INODELK_COUNT))
+                        local->inodelk_count_req = 1;
+                if (dict_get (dict, GLUSTERFS_INODELK_DOM_COUNT))
+                        local->inodelk_dom_count_req = 1;
+                if (dict_get (dict, GLUSTERFS_POSIXLK_COUNT))
+                        local->posixlk_count_req = 1;
+        }
+
+        frame->local = local;
+
+        STACK_WIND (frame, pl_readdirp_cbk,
+                    FIRST_CHILD(this), FIRST_CHILD(this)->fops->readdirp,
+                    fd, size, offset, dict);
+
+        return 0;
+out:
+        STACK_UNWIND_STRICT (readdirp, frame, -1, ENOMEM, NULL, NULL);
+        return 0;
+}
+
+
 void
 pl_dump_lock (char *str, int size, struct gf_flock *flock,
-              uint64_t owner, void *trans)
+              gf_lkowner_t *owner, void *trans, char *conn_id,
+              time_t *granted_time, time_t *blkd_time, gf_boolean_t active)
 {
-        char *type_str = NULL;
+        char  *type_str    = NULL;
+        char   granted[32] = {0,};
+        char   blocked[32] = {0,};
 
         switch (flock->l_type) {
         case F_RDLCK:
@@ -1558,13 +2182,35 @@ pl_dump_lock (char *str, int size, struct gf_flock *flock,
                 break;
         }
 
-        snprintf (str, size, "type=%s, start=%llu, len=%llu, pid=%llu, lk-owner=%llu, transport=%p",
-                  type_str, (unsigned long long) flock->l_start,
-                  (unsigned long long) flock->l_len,
-                  (unsigned long long) flock->l_pid,
-                  (unsigned long long) owner,
-                  trans);
-
+        if (active) {
+                if (blkd_time && *blkd_time == 0) {
+                        snprintf (str, size, RANGE_GRNTD_FMT,
+                                  type_str, flock->l_whence,
+                                  (unsigned long long) flock->l_start,
+                                  (unsigned long long) flock->l_len,
+                                  (unsigned long long) flock->l_pid,
+                                  lkowner_utoa (owner), trans, conn_id,
+                                  ctime_r (granted_time, granted));
+                } else {
+                        snprintf (str, size, RANGE_BLKD_GRNTD_FMT,
+                                  type_str, flock->l_whence,
+                                  (unsigned long long) flock->l_start,
+                                  (unsigned long long) flock->l_len,
+                                  (unsigned long long) flock->l_pid,
+                                  lkowner_utoa (owner), trans, conn_id,
+                                  ctime_r (blkd_time, blocked),
+                                  ctime_r (granted_time, granted));
+                }
+        }
+        else {
+                snprintf (str, size, RANGE_BLKD_FMT,
+                          type_str, flock->l_whence,
+                          (unsigned long long) flock->l_start,
+                          (unsigned long long) flock->l_len,
+                          (unsigned long long) flock->l_pid,
+                          lkowner_utoa (owner), trans, conn_id,
+                          ctime_r (blkd_time, blocked));
+        }
 
 }
 
@@ -1573,8 +2219,10 @@ __dump_entrylks (pl_inode_t *pl_inode)
 {
         pl_dom_list_t   *dom  = NULL;
         pl_entry_lock_t *lock = NULL;
-        int             count = 0;
-        char            key[GF_DUMP_MAX_BUF_LEN];
+        char             blocked[32] = {0,};
+        char             granted[32] = {0,};
+        int              count = 0;
+        char             key[GF_DUMP_MAX_BUF_LEN] = {0,};
 
         char tmp[256];
 
@@ -1583,7 +2231,7 @@ __dump_entrylks (pl_inode_t *pl_inode)
                 count = 0;
 
                 gf_proc_dump_build_key(key,
-                                       "xlator.feature.locks.lock-dump.domain",
+                                       "lock-dump.domain",
                                        "domain");
                 gf_proc_dump_write(key, "%s", dom->domain);
 
@@ -1591,12 +2239,25 @@ __dump_entrylks (pl_inode_t *pl_inode)
 
                         gf_proc_dump_build_key(key,
                                                "xlator.feature.locks.lock-dump.domain.entrylk",
-                                               "entrylk[%d](ACTIVE)",count );
-                        snprintf (tmp, 256," %s on %s owner=%llu, transport=%p",
-                                  lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" : 
-                                  "ENTRYLK_WRLCK", lock->basename,
-                                  (unsigned long long) lock->owner,
-                                  lock->trans);
+                                               "entrylk[%d](ACTIVE)", count );
+                        if (lock->blkd_time.tv_sec == 0 && lock->blkd_time.tv_usec == 0) {
+                                snprintf (tmp, 256, ENTRY_GRNTD_FMT,
+                                          lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" :
+                                          "ENTRYLK_WRLCK", lock->basename,
+                                          (unsigned long long) lock->client_pid,
+                                          lkowner_utoa (&lock->owner), lock->client,
+                                          lock->connection_id,
+                                          ctime_r (&lock->granted_time.tv_sec, granted));
+                        } else {
+                                snprintf (tmp, 256, ENTRY_BLKD_GRNTD_FMT,
+                                          lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" :
+                                          "ENTRYLK_WRLCK", lock->basename,
+                                          (unsigned long long) lock->client_pid,
+                                          lkowner_utoa (&lock->owner), lock->client,
+                                          lock->connection_id,
+                                          ctime_r (&lock->blkd_time.tv_sec, blocked),
+                                          ctime_r (&lock->granted_time.tv_sec, granted));
+                        }
 
                         gf_proc_dump_write(key, tmp);
 
@@ -1607,10 +2268,14 @@ __dump_entrylks (pl_inode_t *pl_inode)
 
                         gf_proc_dump_build_key(key,
                                                "xlator.feature.locks.lock-dump.domain.entrylk",
-                                               "entrylk[%d](BLOCKED)",count );
-                        snprintf (tmp, 256," %s on %s state = Blocked",
-                                  lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" : 
-                                  "ENTRYLK_WRLCK", lock->basename);
+                                               "entrylk[%d](BLOCKED)", count );
+                        snprintf (tmp, 256, ENTRY_BLKD_FMT,
+                                  lock->type == ENTRYLK_RDLCK ? "ENTRYLK_RDLCK" :
+                                  "ENTRYLK_WRLCK", lock->basename,
+                                  (unsigned long long) lock->client_pid,
+                                  lkowner_utoa (&lock->owner), lock->client,
+                                  lock->connection_id,
+                                  ctime_r (&lock->blkd_time.tv_sec, blocked));
 
                         gf_proc_dump_write(key, tmp);
 
@@ -1647,18 +2312,23 @@ __dump_inodelks (pl_inode_t *pl_inode)
                 count = 0;
 
                 gf_proc_dump_build_key(key,
-                                       "xlator.feature.locks.lock-dump.domain",
+                                       "lock-dump.domain",
                                        "domain");
                 gf_proc_dump_write(key, "%s", dom->domain);
 
                 list_for_each_entry (lock, &dom->inodelk_list, list) {
 
                         gf_proc_dump_build_key(key,
-                                               "xlator.feature.locks.lock-dump.domain.inodelk",
+                                               "inodelk",
                                                "inodelk[%d](ACTIVE)",count );
 
+                        SET_FLOCK_PID (&lock->user_flock, lock);
                         pl_dump_lock (tmp, 256, &lock->user_flock,
-                                      lock->owner, lock->transport);
+                                      &lock->owner,
+                                      lock->client, lock->connection_id,
+                                      &lock->granted_time.tv_sec,
+                                      &lock->blkd_time.tv_sec,
+                                      _gf_true);
                         gf_proc_dump_write(key, tmp);
 
                         count++;
@@ -1667,10 +2337,14 @@ __dump_inodelks (pl_inode_t *pl_inode)
                 list_for_each_entry (lock, &dom->blocked_inodelks, blocked_locks) {
 
                         gf_proc_dump_build_key(key,
-                                               "xlator.feature.locks.lock-dump.domain.inodelk",
+                                               "inodelk",
                                                "inodelk[%d](BLOCKED)",count );
+                        SET_FLOCK_PID (&lock->user_flock, lock);
                         pl_dump_lock (tmp, 256, &lock->user_flock,
-                                      lock->owner, lock->transport);
+                                      &lock->owner,
+                                      lock->client, lock->connection_id,
+                                      0, &lock->blkd_time.tv_sec,
+                                      _gf_false);
                         gf_proc_dump_write(key, tmp);
 
                         count++;
@@ -1702,20 +2376,20 @@ __dump_posixlks (pl_inode_t *pl_inode)
 
       list_for_each_entry (lock, &pl_inode->ext_list, list) {
 
+              SET_FLOCK_PID (&lock->user_flock, lock);
               gf_proc_dump_build_key(key,
-                                     "xlator.feature.locks.lock-dump.domain.posixlk",
+                                     "posixlk",
                                      "posixlk[%d](%s)",
                                      count,
                                      lock->blocked ? "BLOCKED" : "ACTIVE");
               pl_dump_lock (tmp, 256, &lock->user_flock,
-                            lock->owner, lock->transport);
+                            &lock->owner, lock->client, NULL,
+                            &lock->granted_time.tv_sec, &lock->blkd_time.tv_sec,
+                            (lock->blocked)? _gf_false: _gf_true);
               gf_proc_dump_write(key, tmp);
 
               count++;
         }
-
-
-
 }
 
 void
@@ -1736,75 +2410,82 @@ pl_dump_inode_priv (xlator_t *this, inode_t *inode)
         int             ret = -1;
         uint64_t        tmp_pl_inode = 0;
         pl_inode_t      *pl_inode = NULL;
-        char            key[GF_DUMP_MAX_BUF_LEN];
+        char            *pathname = NULL;
+        gf_boolean_t    section_added = _gf_false;
 
         int count      = 0;
 
-        if (!inode)
-                return -1;
-
-        ret = inode_ctx_get (inode, this, &tmp_pl_inode);
+        if (!inode) {
+                errno = EINVAL;
+                goto out;
+        }
 
-        if (ret != 0)
-                return ret;
+        ret = TRY_LOCK (&inode->lock);
+        if (ret)
+                goto out;
+        {
+                ret = __inode_ctx_get (inode, this, &tmp_pl_inode);
+                if (ret)
+                        goto unlock;
+        }
+unlock:
+        UNLOCK (&inode->lock);
+        if (ret)
+                goto out;
 
         pl_inode = (pl_inode_t *)(long)tmp_pl_inode;
+        if (!pl_inode) {
+                ret = -1;
+                goto out;
+        }
 
-        if (!pl_inode)
-                return -1;
-
-        gf_proc_dump_build_key(key,
-                               "xlator.feature.locks.inode",
-                               "%ld.mandatory",inode->ino);
-        gf_proc_dump_write(key, "%d", pl_inode->mandatory);
-
-
-                count = get_entrylk_count (this, inode);
-                gf_proc_dump_build_key(key,
-                                       "xlator.feature.locks.entrylk-count",
-                                       "%ld.entrylk-count", inode->ino);
-                gf_proc_dump_write(key, "%d", count);
-
-                dump_entrylks(pl_inode);
-
-                count = get_inodelk_count (this, inode);
-                gf_proc_dump_build_key(key,
-                                       "xlator.feature.locks.inodelk-count",
-                                       "%ld.inodelk-count", inode->ino);
-                gf_proc_dump_write(key, "%d", count);
-
-                dump_inodelks(pl_inode);
-
-                count = get_posixlk_count (this, inode);
-                gf_proc_dump_build_key(key,
-                                       "xlator.feature.locks.posixlk-count",
-                                       "%ld.posixlk-count", inode->ino);
-                gf_proc_dump_write(key, "%d", count);
-
-                dump_posixlks(pl_inode);
+        gf_proc_dump_add_section("xlator.features.locks.%s.inode", this->name);
+        section_added = _gf_true;
 
+        /*We are safe to call __inode_path since we have the
+         * inode->table->lock */
+        __inode_path (inode, NULL, &pathname);
+        if (pathname)
+                gf_proc_dump_write ("path", "%s", pathname);
 
-        return 0;
-}
+        gf_proc_dump_write("mandatory", "%d", pl_inode->mandatory);
 
+        ret = pthread_mutex_trylock (&pl_inode->mutex);
+        if (ret)
+                goto out;
+        {
+                count = __get_entrylk_count (this, pl_inode);
+                if (count) {
+                        gf_proc_dump_write("entrylk-count", "%d", count);
+                        __dump_entrylks (pl_inode);
+                }
 
+                count = __get_inodelk_count (this, pl_inode, NULL);
+                if (count) {
+                        gf_proc_dump_write("inodelk-count", "%d", count);
+                        __dump_inodelks (pl_inode);
+                }
 
-/*
- * pl_dump_inode - inode dump function for posix locks
- *
- */
-int
-pl_dump_inode (xlator_t *this)
-{
-
-        assert(this);
-
-        if (this->itable) {
-                inode_table_dump(this->itable,
-                                 "xlator.features.locks.inode_table");
+                count = __get_posixlk_count (this, pl_inode);
+                if (count) {
+                        gf_proc_dump_write("posixlk-count", "%d", count);
+                        __dump_posixlks (pl_inode);
+                }
         }
+        pthread_mutex_unlock (&pl_inode->mutex);
 
-        return 0;
+out:
+        GF_FREE (pathname);
+
+        if (ret && inode) {
+                if (!section_added)
+                        gf_proc_dump_add_section ("xlator.features.locks.%s."
+                                                  "inode", this->name);
+                gf_proc_dump_write ("Unable to print lock state", "(Lock "
+                                    "acquisition failure) %s",
+                                    uuid_utoa (inode->gfid));
+        }
+        return ret;
 }
 
 int32_t
@@ -1816,7 +2497,7 @@ mem_acct_init (xlator_t *this)
                 return ret;
 
         ret = xlator_mem_acct_init (this, gf_locks_mt_end + 1);
-        
+
         if (ret != 0) {
                 gf_log (this->name, GF_LOG_ERROR, "Memory accounting init"
                                 "failed");
@@ -1826,18 +2507,92 @@ mem_acct_init (xlator_t *this)
         return ret;
 }
 
+
+pl_ctx_t*
+pl_ctx_get (client_t *client, xlator_t *xlator)
+{
+        void *tmp = NULL;
+        pl_ctx_t *ctx = NULL;
+
+        client_ctx_get (client, xlator, &tmp);
+
+        ctx = tmp;
+
+        if (ctx != NULL)
+                goto out;
+
+        ctx = GF_CALLOC (1, sizeof (pl_ctx_t), gf_locks_mt_posix_lock_t);
+
+        if (ctx == NULL)
+                goto out;
+
+        pthread_mutex_init (&ctx->lock, NULL);
+	INIT_LIST_HEAD (&ctx->inodelk_lockers);
+	INIT_LIST_HEAD (&ctx->entrylk_lockers);
+
+        if (client_ctx_set (client, xlator, ctx) != 0) {
+                pthread_mutex_destroy (&ctx->lock);
+                GF_FREE (ctx);
+                ctx = NULL;
+        }
+out:
+        return ctx;
+}
+
+
+static int
+pl_client_disconnect_cbk (xlator_t *this, client_t *client)
+{
+        pl_ctx_t *pl_ctx = NULL;
+
+	pl_ctx = pl_ctx_get (client, this);
+
+	pl_inodelk_client_cleanup (this, pl_ctx);
+
+	pl_entrylk_client_cleanup (this, pl_ctx);
+
+	return 0;
+}
+
+
+static int
+pl_client_destroy_cbk (xlator_t *this, client_t *client)
+{
+        void     *tmp    = NULL;
+        pl_ctx_t *pl_ctx = NULL;
+
+	pl_client_disconnect_cbk (this, client);
+
+        client_ctx_del (client, this, &tmp);
+
+        if (tmp == NULL)
+                return 0;
+
+        pl_ctx = tmp;
+
+	GF_ASSERT (list_empty(&pl_ctx->inodelk_lockers));
+	GF_ASSERT (list_empty(&pl_ctx->entrylk_lockers));
+
+	pthread_mutex_destroy (&pl_ctx->lock);
+        GF_FREE (pl_ctx);
+
+        return 0;
+}
+
+
 int
 init (xlator_t *this)
 {
         posix_locks_private_t *priv = NULL;
         xlator_list_t         *trav = NULL;
         data_t                *mandatory = NULL;
-	data_t                *trace = NULL;
+        data_t                *trace = NULL;
+        int                   ret = -1;
 
         if (!this->children || this->children->next) {
                 gf_log (this->name, GF_LOG_CRITICAL,
                         "FATAL: posix-locks should have exactly one child");
-                return -1;
+                goto out;
         }
 
         if (!this->parents) {
@@ -1853,10 +2608,10 @@ init (xlator_t *this)
                 gf_log (this->name, GF_LOG_CRITICAL,
                         "'locks' translator is not loaded over a storage "
                         "translator");
-                return -1;
+                goto out;
         }
 
-	priv = GF_CALLOC (1, sizeof (*priv),
+        priv = GF_CALLOC (1, sizeof (*priv),
                           gf_locks_mt_posix_locks_private_t);
 
         mandatory = dict_get (this->options, "mandatory-locks");
@@ -1864,18 +2619,32 @@ init (xlator_t *this)
                 gf_log (this->name, GF_LOG_WARNING,
                         "mandatory locks not supported in this minor release.");
 
-	trace = dict_get (this->options, "trace");
-	if (trace) {
-		if (gf_string2boolean (trace->data,
-				       &priv->trace) == -1) {
-			gf_log (this->name, GF_LOG_ERROR,
-				"'trace' takes on only boolean values.");
-			return -1;
-		}
-	}
+        trace = dict_get (this->options, "trace");
+        if (trace) {
+                if (gf_string2boolean (trace->data,
+                                       &priv->trace) == -1) {
+                        gf_log (this->name, GF_LOG_ERROR,
+                                "'trace' takes on only boolean values.");
+                        goto out;
+                }
+        }
+
+        this->local_pool = mem_pool_new (pl_local_t, 32);
+        if (!this->local_pool) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_ERROR,
+                        "failed to create local_t's memory pool");
+                goto out;
+        }
 
         this->private = priv;
-        return 0;
+        ret = 0;
+
+out:
+        if (ret) {
+                GF_FREE (priv);
+        }
+        return ret;
 }
 
 
@@ -1888,6 +2657,7 @@ fini (xlator_t *this)
         if (!priv)
                 return 0;
         this->private = NULL;
+        GF_FREE (priv->brickname);
         GF_FREE (priv);
 
         return 0;
@@ -1896,21 +2666,23 @@ fini (xlator_t *this)
 
 int
 pl_inodelk (call_frame_t *frame, xlator_t *this,
-            const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock);
+            const char *volume, loc_t *loc, int32_t cmd, struct gf_flock *flock,
+            dict_t *xdata);
 
 int
 pl_finodelk (call_frame_t *frame, xlator_t *this,
-             const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock);
+             const char *volume, fd_t *fd, int32_t cmd, struct gf_flock *flock,
+             dict_t *xdata);
 
 int
 pl_entrylk (call_frame_t *frame, xlator_t *this,
             const char *volume, loc_t *loc, const char *basename,
-            entrylk_cmd cmd, entrylk_type type);
+            entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
 
 int
 pl_fentrylk (call_frame_t *frame, xlator_t *this,
              const char *volume, fd_t *fd, const char *basename,
-             entrylk_cmd cmd, entrylk_type type);
+             entrylk_cmd cmd, entrylk_type type, dict_t *xdata);
 
 struct xlator_fops fops = {
         .lookup      = pl_lookup,
@@ -1927,6 +2699,10 @@ struct xlator_fops fops = {
         .fentrylk    = pl_fentrylk,
         .flush       = pl_flush,
         .opendir     = pl_opendir,
+        .readdirp    = pl_readdirp,
+        .getxattr    = pl_getxattr,
+        .fgetxattr   = pl_fgetxattr,
+        .fsetxattr   = pl_fsetxattr,
 };
 
 struct xlator_dumpops dumpops = {
@@ -1934,8 +2710,11 @@ struct xlator_dumpops dumpops = {
 };
 
 struct xlator_cbks cbks = {
-        .forget      = pl_forget,
-        .release     = pl_release,
+        .forget            = pl_forget,
+        .release           = pl_release,
+        .releasedir        = pl_releasedir,
+        .client_destroy    = pl_client_destroy_cbk,
+        .client_disconnect = pl_client_disconnect_cbk,
 };
 
 
@@ -1943,8 +2722,8 @@ struct volume_options options[] = {
         { .key  = { "mandatory-locks", "mandatory" },
           .type = GF_OPTION_TYPE_BOOL
         },
-	{ .key  = { "trace" },
-	  .type = GF_OPTION_TYPE_BOOL
-	},
+        { .key  = { "trace" },
+          .type = GF_OPTION_TYPE_BOOL
+        },
         { .key = {NULL} },
 };
diff --git a/xlators/features/locks/src/reservelk.c b/xlators/features/locks/src/reservelk.c
index e2b88140a..11abd26d8 100644
--- a/xlators/features/locks/src/reservelk.c
+++ b/xlators/features/locks/src/reservelk.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006, 2007, 2008 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -36,24 +26,24 @@
 void
 __delete_reserve_lock (posix_lock_t *lock)
 {
-	list_del (&lock->list);
+        list_del (&lock->list);
 }
 
 void
 __destroy_reserve_lock (posix_lock_t *lock)
 {
-	GF_FREE (lock);
+        GF_FREE (lock);
 }
 
 /* Return true if the two reservelks have exactly same lock boundaries */
 int
 reservelks_equal (posix_lock_t *l1, posix_lock_t *l2)
 {
-	if ((l1->fl_start == l2->fl_start) &&
-	    (l1->fl_end == l2->fl_end))
-		return 1;
+        if ((l1->fl_start == l2->fl_start) &&
+            (l1->fl_end == l2->fl_end))
+                return 1;
 
-	return 0;
+        return 0;
 }
 
 /* Determine if lock is grantable or not */
@@ -61,30 +51,30 @@ static posix_lock_t *
 __reservelk_grantable (pl_inode_t *pl_inode, posix_lock_t *lock)
 {
         xlator_t     *this     = NULL;
-	posix_lock_t *l        = NULL;
+        posix_lock_t *l        = NULL;
         posix_lock_t *ret_lock = NULL;
 
         this = THIS;
 
-	if (list_empty (&pl_inode->reservelk_list)) {
+        if (list_empty (&pl_inode->reservelk_list)) {
                 gf_log (this->name, GF_LOG_TRACE,
                         "No reservelks in list");
-		goto out;
+                goto out;
         }
-	list_for_each_entry (l, &pl_inode->reservelk_list, list){
-		if (reservelks_equal (lock, l)) {
+        list_for_each_entry (l, &pl_inode->reservelk_list, list){
+                if (reservelks_equal (lock, l)) {
                         ret_lock = l;
                         break;
-		}
-	}
+                }
+        }
 out:
-	return ret_lock;
+        return ret_lock;
 }
 
-static int
+static inline int
 __same_owner_reservelk (posix_lock_t *l1, posix_lock_t *l2)
 {
-        return ((l1->owner == l2->owner));
+        return (is_same_lkowner (&l1->owner, &l2->owner));
 
 }
 
@@ -175,46 +165,46 @@ static int
 __lock_reservelk (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
                   int can_block)
 {
-	posix_lock_t *conf = NULL;
-	int ret = -EINVAL;
+        posix_lock_t *conf = NULL;
+        int ret = -EINVAL;
 
-	conf = __reservelk_grantable (pl_inode, lock);
-	if (conf){
-		ret = -EAGAIN;
-		if (can_block == 0)
-			goto out;
+        conf = __reservelk_grantable (pl_inode, lock);
+        if (conf){
+                ret = -EAGAIN;
+                if (can_block == 0)
+                        goto out;
 
-		list_add_tail (&lock->list, &pl_inode->blocked_reservelks);
+                list_add_tail (&lock->list, &pl_inode->blocked_reservelks);
 
                 gf_log (this->name, GF_LOG_TRACE,
-                        "%s (pid=%d) lk-owner:%"PRIu64" %"PRId64" - %"PRId64" => Blocked",
+                        "%s (pid=%d) lk-owner:%s %"PRId64" - %"PRId64" => Blocked",
                         lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
                         lock->client_pid,
-                        lock->owner,
+                        lkowner_utoa (&lock->owner),
                         lock->user_flock.l_start,
                         lock->user_flock.l_len);
 
 
-		goto out;
-	}
+                goto out;
+        }
 
-	list_add (&lock->list, &pl_inode->reservelk_list);
+        list_add (&lock->list, &pl_inode->reservelk_list);
 
-	ret = 0;
+        ret = 0;
 
 out:
-	return ret;
+        return ret;
 }
 
 static posix_lock_t *
 find_matching_reservelk (posix_lock_t *lock, pl_inode_t *pl_inode)
 {
-	posix_lock_t *l = NULL;
-	list_for_each_entry (l, &pl_inode->reservelk_list, list) {
-		if (reservelks_equal (l, lock))
-			return l;
-	}
-	return NULL;
+        posix_lock_t *l = NULL;
+        list_for_each_entry (l, &pl_inode->reservelk_list, list) {
+                if (reservelks_equal (l, lock))
+                        return l;
+        }
+        return NULL;
 }
 
 /* Set F_UNLCK removes a lock which has the exact same lock boundaries
@@ -224,20 +214,20 @@ static posix_lock_t *
 __reserve_unlock_lock (xlator_t *this, posix_lock_t *lock, pl_inode_t *pl_inode)
 {
 
-	posix_lock_t *conf = NULL;
+        posix_lock_t *conf = NULL;
 
-	conf = find_matching_reservelk (lock, pl_inode);
-	if (!conf) {
+        conf = find_matching_reservelk (lock, pl_inode);
+        if (!conf) {
                 gf_log (this->name, GF_LOG_DEBUG,
                         " Matching lock not found for unlock");
-		goto out;
+                goto out;
         }
-	__delete_reserve_lock (conf);
+        __delete_reserve_lock (conf);
         gf_log (this->name, GF_LOG_DEBUG,
                 " Matching lock found for unlock");
 
 out:
-	return conf;
+        return conf;
 
 
 }
@@ -246,37 +236,37 @@ static void
 __grant_blocked_reserve_locks (xlator_t *this, pl_inode_t *pl_inode,
                                struct list_head *granted)
 {
-	int	      bl_ret = 0;
-	posix_lock_t *bl = NULL;
-	posix_lock_t *tmp = NULL;
+        int              bl_ret = 0;
+        posix_lock_t *bl = NULL;
+        posix_lock_t *tmp = NULL;
 
         struct list_head blocked_list;
 
         INIT_LIST_HEAD (&blocked_list);
         list_splice_init (&pl_inode->blocked_reservelks, &blocked_list);
 
-	list_for_each_entry_safe (bl, tmp, &blocked_list, list) {
+        list_for_each_entry_safe (bl, tmp, &blocked_list, list) {
 
-		list_del_init (&bl->list);
+                list_del_init (&bl->list);
 
-		bl_ret = __lock_reservelk (this, pl_inode, bl, 1);
+                bl_ret = __lock_reservelk (this, pl_inode, bl, 1);
 
-		if (bl_ret == 0) {
-			list_add (&bl->list, granted);
+                if (bl_ret == 0) {
+                        list_add (&bl->list, granted);
                 }
         }
-	return;
+        return;
 }
 
 /* Grant all reservelks blocked on lock(s) */
 void
 grant_blocked_reserve_locks (xlator_t *this, pl_inode_t *pl_inode)
 {
-	struct list_head granted;
-	posix_lock_t *lock = NULL;
-	posix_lock_t *tmp = NULL;
+        struct list_head granted;
+        posix_lock_t *lock = NULL;
+        posix_lock_t *tmp = NULL;
 
-	INIT_LIST_HEAD (&granted);
+        INIT_LIST_HEAD (&granted);
 
         if (list_empty (&pl_inode->blocked_reservelks)) {
                 gf_log (this->name, GF_LOG_TRACE,
@@ -285,22 +275,23 @@ grant_blocked_reserve_locks (xlator_t *this, pl_inode_t *pl_inode)
         }
 
         pthread_mutex_lock (&pl_inode->mutex);
-	{
-		__grant_blocked_reserve_locks (this, pl_inode, &granted);
-	}
+        {
+                __grant_blocked_reserve_locks (this, pl_inode, &granted);
+        }
         pthread_mutex_unlock (&pl_inode->mutex);
 
-	list_for_each_entry_safe (lock, tmp, &granted, list) {
+        list_for_each_entry_safe (lock, tmp, &granted, list) {
                 gf_log (this->name, GF_LOG_TRACE,
-			"%s (pid=%d) (lk-owner=%"PRIu64") %"PRId64" - %"PRId64" => Granted",
-			lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-			lock->client_pid,
-			lock->owner,
-			lock->user_flock.l_start,
-			lock->user_flock.l_len);
+                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => Granted",
+                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                        lock->client_pid,
+                        lkowner_utoa (&lock->owner),
+                        lock->user_flock.l_start,
+                        lock->user_flock.l_len);
 
-		STACK_UNWIND_STRICT (lk, lock->frame, 0, 0, &lock->user_flock);
-	}
+                STACK_UNWIND_STRICT (lk, lock->frame, 0, 0, &lock->user_flock,
+                                     NULL);
+        }
 
 }
 
@@ -308,34 +299,34 @@ static void
 __grant_blocked_lock_calls (xlator_t *this, pl_inode_t *pl_inode,
                             struct list_head *granted)
 {
-	int	      bl_ret = 0;
-	posix_lock_t *bl = NULL;
-	posix_lock_t *tmp = NULL;
+        int              bl_ret = 0;
+        posix_lock_t *bl = NULL;
+        posix_lock_t *tmp = NULL;
 
         struct list_head blocked_list;
 
         INIT_LIST_HEAD (&blocked_list);
         list_splice_init (&pl_inode->blocked_reservelks, &blocked_list);
 
-	list_for_each_entry_safe (bl, tmp, &blocked_list, list) {
+        list_for_each_entry_safe (bl, tmp, &blocked_list, list) {
 
-		list_del_init (&bl->list);
+                list_del_init (&bl->list);
 
-		bl_ret = pl_verify_reservelk (this, pl_inode, bl, bl->blocked);
+                bl_ret = pl_verify_reservelk (this, pl_inode, bl, bl->blocked);
 
-		if (bl_ret == 0) {
+                if (bl_ret == 0) {
                         list_add_tail (&bl->list, granted);
                 }
         }
-	return;
+        return;
 }
 
 void
 grant_blocked_lock_calls (xlator_t *this, pl_inode_t *pl_inode)
 {
-	struct list_head granted;
-	posix_lock_t *lock = NULL;
-	posix_lock_t *tmp = NULL;
+        struct list_head granted;
+        posix_lock_t *lock = NULL;
+        posix_lock_t *tmp = NULL;
         fd_t         *fd  = NULL;
 
         int can_block = 0;
@@ -349,9 +340,9 @@ grant_blocked_lock_calls (xlator_t *this, pl_inode_t *pl_inode)
         }
 
         pthread_mutex_lock (&pl_inode->mutex);
-	{
-		__grant_blocked_lock_calls (this, pl_inode, &granted);
-	}
+        {
+                __grant_blocked_lock_calls (this, pl_inode, &granted);
+        }
         pthread_mutex_unlock (&pl_inode->mutex);
 
         list_for_each_entry_safe (lock, tmp, &granted, list) {
@@ -376,7 +367,9 @@ grant_blocked_lock_calls (xlator_t *this, pl_inode_t *pl_inode)
                                 pl_trace_out (this, lock->frame, fd, NULL, cmd,
                                               &lock->user_flock, -1, EAGAIN, NULL);
                                 pl_update_refkeeper (this, fd->inode);
-                                STACK_UNWIND_STRICT (lk, lock->frame, -1, EAGAIN, &lock->user_flock);
+                                STACK_UNWIND_STRICT (lk, lock->frame, -1,
+                                                     EAGAIN, &lock->user_flock,
+                                                     NULL);
                                 __destroy_lock (lock);
                         }
                 }
@@ -393,18 +386,18 @@ pl_reserve_unlock (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock)
         int ret = -1;
 
         pthread_mutex_lock (&pl_inode->mutex);
-	{
-		retlock = __reserve_unlock_lock (this, lock, pl_inode);
-		if (!retlock) {
-			gf_log (this->name, GF_LOG_DEBUG,
-				"Bad Unlock issued on Inode lock");
+        {
+                retlock = __reserve_unlock_lock (this, lock, pl_inode);
+                if (!retlock) {
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "Bad Unlock issued on Inode lock");
                         ret = -EINVAL;
                         goto out;
                 }
 
                 gf_log (this->name, GF_LOG_TRACE,
                         "Reservelk Unlock successful");
-		__destroy_reserve_lock (retlock);
+                __destroy_reserve_lock (retlock);
                 ret = 0;
         }
 out:
@@ -421,30 +414,30 @@ int
 pl_reserve_setlk (xlator_t *this, pl_inode_t *pl_inode, posix_lock_t *lock,
                   int can_block)
 {
-	int ret = -EINVAL;
+        int ret = -EINVAL;
 
-	pthread_mutex_lock (&pl_inode->mutex);
-	{
+        pthread_mutex_lock (&pl_inode->mutex);
+        {
 
-			ret = __lock_reservelk (this, pl_inode, lock, can_block);
+                        ret = __lock_reservelk (this, pl_inode, lock, can_block);
                         if (ret < 0)
-				gf_log (this->name, GF_LOG_TRACE,
-                                        "%s (pid=%d) (lk-owner=%"PRIu64") %"PRId64" - %"PRId64" => NOK",
-					lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
-					lock->client_pid,
-                                        lock->owner,
-					lock->user_flock.l_start,
-					lock->user_flock.l_len);
+                                gf_log (this->name, GF_LOG_TRACE,
+                                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => NOK",
+                                        lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
+                                        lock->client_pid,
+                                        lkowner_utoa (&lock->owner),
+                                        lock->user_flock.l_start,
+                                        lock->user_flock.l_len);
                         else
-				gf_log (this->name, GF_LOG_TRACE,
-                                        "%s (pid=%d) (lk-owner=%"PRIu64") %"PRId64" - %"PRId64" => OK",
+                                gf_log (this->name, GF_LOG_TRACE,
+                                        "%s (pid=%d) (lk-owner=%s) %"PRId64" - %"PRId64" => OK",
                                         lock->fl_type == F_UNLCK ? "Unlock" : "Lock",
                                         lock->client_pid,
-                                        lock->owner,
+                                        lkowner_utoa (&lock->owner),
                                         lock->fl_start,
                                         lock->fl_end);
 
-	}
-	pthread_mutex_unlock (&pl_inode->mutex);
+        }
+        pthread_mutex_unlock (&pl_inode->mutex);
         return ret;
 }
diff --git a/xlators/features/locks/tests/unit-test.c b/xlators/features/locks/tests/unit-test.c
index 889c304da..d2cca32de 100644
--- a/xlators/features/locks/tests/unit-test.c
+++ b/xlators/features/locks/tests/unit-test.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2006-2009 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
+   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
 */
-
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
diff --git a/xlators/features/mac-compat/src/Makefile.am b/xlators/features/mac-compat/src/Makefile.am
index 915c13e30..42ed350e9 100644
--- a/xlators/features/mac-compat/src/Makefile.am
+++ b/xlators/features/mac-compat/src/Makefile.am
@@ -1,13 +1,15 @@
 xlator_LTLIBRARIES = mac-compat.la
 xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
 
-mac_compat_la_LDFLAGS = -module -avoidversion
+mac_compat_la_LDFLAGS = -module -avoid-version
 
 mac_compat_la_SOURCES = mac-compat.c
 mac_compat_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
 
-AM_CFLAGS = -fPIC -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -Wall -D$(GF_HOST_OS) \
-	-I$(top_srcdir)/libglusterfs/src -shared -nostartfiles $(GF_CFLAGS)
+noinst_HEADERS = mac-compat.h
 
-CLEANFILES = 
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src
 
+AM_CFLAGS = -Wall $(GF_CFLAGS)
+
+CLEANFILES =
diff --git a/xlators/features/mac-compat/src/mac-compat.c b/xlators/features/mac-compat/src/mac-compat.c
index b1277ec16..0eaf563e8 100644
--- a/xlators/features/mac-compat/src/mac-compat.c
+++ b/xlators/features/mac-compat/src/mac-compat.c
@@ -1,22 +1,12 @@
 /*
-  Copyright (c) 2008-2010 Gluster, Inc. <http://www.gluster.com>
-  This file is part of GlusterFS.
-
-  GlusterFS is free software; you can redistribute it and/or modify
-  it under the terms of the GNU Affero General Public License as published
-  by the Free Software Foundation; either version 3 of the License,
-  or (at your option) any later version.
-
-  GlusterFS is distributed in the hope that it will be useful, but
-  WITHOUT ANY WARRANTY; without even the implied warranty of
-  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-  Affero General Public License for more details.
-
-  You should have received a copy of the GNU Affero General Public License
-  along with this program.  If not, see
-  <http://www.gnu.org/licenses/>.
-*/
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
 
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
 #ifndef _CONFIG_H
 #define _CONFIG_H
 #include "config.h"
@@ -25,141 +15,203 @@
 #include "xlator.h"
 #include "defaults.h"
 #include "compat-errno.h"
+#include "syscall.h"
+#include "mem-pool.h"
+#include "mac-compat.h"
 
-
-enum apple_xattr {
-        GF_FINDER_INFO_XATTR,
-        GF_RESOURCE_FORK_XATTR,
-        GF_XATTR_ALL,
-        GF_XATTR_NONE
-};
-
-static char *apple_xattr_name[] = {
-        [GF_FINDER_INFO_XATTR]   = "com.apple.FinderInfo",
-        [GF_RESOURCE_FORK_XATTR] = "com.apple.ResourceFork"
-};
-
-static const char *apple_xattr_value[] = {
-        [GF_FINDER_INFO_XATTR]   =
-        /* 1 2 3 4 5 6 7 8 */
-         "\0\0\0\0\0\0\0\0"
-         "\0\0\0\0\0\0\0\0"
-         "\0\0\0\0\0\0\0\0"
-         "\0\0\0\0\0\0\0\0",
-        [GF_RESOURCE_FORK_XATTR] = ""
-};
-
-static int32_t apple_xattr_len[] = {
-        [GF_FINDER_INFO_XATTR]   = 32,
-        [GF_RESOURCE_FORK_XATTR] = 1
-};
-
+static int
+dict_key_remove_namespace(dict_t *dict, char *key, data_t *value, void *data)
+{
+  /*
+    char buffer[3*value->len+1];
+    int index = 0;
+    for (index = 0; index < value->len; index++)
+    sprintf(buffer+3*index, " %02x", value->data[index]);
+  */
+        xlator_t *this = (xlator_t *) data;
+        if (strncmp(key, "user.", 5) == 0) {
+                dict_set (dict, key + 5, value);
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "remove_namespace_dict: %s -> %s ", key, key + 5);
+                dict_del (dict, key);
+        }
+        return 0;
+}
 
 int32_t
 maccomp_getxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno, dict_t *dict)
+                      int32_t op_ret, int32_t op_errno, dict_t *dict,
+                      dict_t *xdata)
 {
         intptr_t ax = (intptr_t)this->private;
         int i = 0;
 
+        gf_log (this->name, GF_LOG_DEBUG,
+                "getxattr_cbk: dict %p private: %p xdata %p ", dict,
+                this->private, xdata);
+
+        if (dict) {
+                dict_foreach(dict, dict_key_remove_namespace, this);
+        }
+        else {
+                // TODO: we expect dict to exist here, don't know why this
+                // this is needed
+                dict = dict_new();
+        }
+        gf_log (this->name, GF_LOG_DEBUG,
+                "getxattr_cbk: dict %p ax: %ld op_ret %d op_err %d  ", dict, ax,
+                op_ret, op_errno);
         if ((ax == GF_XATTR_ALL && op_ret >= 0) || ax != GF_XATTR_NONE) {
                 op_ret = op_errno = 0;
-
                 for (i = 0; i < GF_XATTR_ALL; i++) {
                         if (dict_get (dict, apple_xattr_name[i]))
                                 continue;
-
+                        /* set dummy data */
+                        gf_log (this->name, GF_LOG_DEBUG,
+                                "getxattr_cbk: setting dummy data %p, %s", dict,
+                                apple_xattr_name[i]);
                         if (dict_set (dict, apple_xattr_name[i],
                                       bin_to_data ((void *)apple_xattr_value[i],
                                                    apple_xattr_len[i])) == -1) {
                                 op_ret = -1;
-                                op_errno = ENOMEM;
+                                op_errno = ENOATTR;
 
                                 break;
                         }
                 }
          }
+        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict, xdata);
+        return 0;
+}
 
-        STACK_UNWIND_STRICT (getxattr, frame, op_ret, op_errno, dict);
 
+static
+int prepend_xattr_user_namespace(dict_t *dict, char *key, data_t *value, void *obj)
+{
+        xlator_t *this = (xlator_t *) obj;
+        dict_t *newdict = (dict_t *) this->private;
+        char *newkey = NULL;
+        gf_add_prefix(XATTR_USER_PREFIX, key, &newkey);
+        key = newkey;
+        dict_set(newdict, (char *)key, value);
+        if (newkey)
+                GF_FREE(newkey);
         return 0;
 }
 
-
-int32_t
-maccomp_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
-                  const char *name)
+intptr_t
+check_name(const char *name, char **newkey)
 {
         intptr_t ax = GF_XATTR_NONE;
-        int i = 0;
-
         if (name) {
+                int i = 0;
                 for (i = 0; i < GF_XATTR_ALL; i++) {
                         if (strcmp (apple_xattr_name[i], name) == 0) {
                                 ax = i;
-
                                 break;
                         }
                 }
+                gf_add_prefix("user.", name, newkey);
         } else
                 ax = GF_XATTR_ALL;
+        return ax;
+}
 
-        this->private = (void *)ax;
+int32_t
+maccomp_getxattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                  const char *name, dict_t *xdata)
+{
+        char *newkey = NULL;
+        this->private = (void *) check_name(name, &newkey);
 
+        gf_log (this->name, GF_LOG_DEBUG,
+                "getxattr: name %s private: %p xdata %p ", name,
+                this->private, xdata);
         STACK_WIND (frame, maccomp_getxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->getxattr,
-                    loc, name);
+                    loc, newkey, xdata);
         return 0;
 }
 
 
 int32_t
 maccomp_fgetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
-                   const char *name)
+                   const char *name, dict_t *xdata)
 {
-        intptr_t ax = GF_XATTR_NONE;
-        int i = 0;
-
-        if (name) {
-                for (i = 0; i < GF_XATTR_ALL; i++) {
-                        if (strcmp (apple_xattr_name[i], name) == 0) {
-                                ax = i;
-
-                                break;
-                        }
-                }
-        } else
-                ax = GF_XATTR_ALL;
-
-        this->private = (void *)ax;
+        char *newkey = NULL;
+        this->private = (void *) check_name(name, &newkey);
 
         STACK_WIND (frame, maccomp_getxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fgetxattr,
-                    fd, name);
+                    fd, newkey, xdata);
+        GF_FREE(newkey);
         return 0;
 }
 
-
 int32_t
 maccomp_setxattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
-                      int32_t op_ret, int32_t op_errno)
+                      int32_t op_ret, int32_t op_errno, dict_t *xdata)
 {
         intptr_t ax = (intptr_t)this->private;
 
         if (op_ret == -1 && ax != GF_XATTR_NONE)
                 op_ret = op_errno = 0;
+        gf_log (this->name, GF_LOG_DEBUG,
+                "setxattr_cbk op_ret %d op_errno %d private: %p xdata %p ",
+                op_ret, op_errno, this->private, xdata);
+        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno, xdata);
+        return 0;
+}
 
-        STACK_UNWIND_STRICT (setxattr, frame, op_ret, op_errno);
-
+int32_t
+maccomp_setattr_cbk (call_frame_t *frame, void *cookie, xlator_t *this,
+                     int32_t op_ret, int32_t op_errno, struct iatt *iatt1,
+                     struct iatt *iattr2, dict_t *xdata)
+{
+        gf_log (this->name, GF_LOG_DEBUG,
+                "setattr_cbk op_ret %d op_errno %d private: %p xdata %p ",
+                op_ret, op_errno, this->private, xdata);
+        STACK_UNWIND_STRICT (setattr, frame, op_ret, op_errno,
+                             iatt1, iattr2, xdata);
         return 0;
 }
 
+int map_flags(int flags)
+{
+        /* DARWIN has different defines on XATTR_ flags.
+           There do not seem to be a POSIX standard
+           Parse any other flags over.
+           NOFOLLOW is always true on Linux and Darwin
+        */
+        int linux_flags = flags & ~(GF_XATTR_CREATE | GF_XATTR_REPLACE | XATTR_REPLACE);
+        if (XATTR_CREATE & flags)
+                linux_flags |= GF_XATTR_CREATE;
+        if (XATTR_REPLACE & flags)
+                linux_flags |= GF_XATTR_REPLACE;
+        return linux_flags;
+}
+
+int32_t
+maccomp_fremovexattr (call_frame_t *frame, xlator_t *this, fd_t *fd,
+                   const char *name, dict_t *xdata)
+{
+        char *newkey = NULL;
+
+        this->private = (void *) check_name(name, &newkey);
+
+        STACK_WIND (frame, default_fremovexattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->fremovexattr,
+                    fd, newkey, xdata);
+        GF_FREE(newkey);
+        return 0;
+}
 
 int32_t
 maccomp_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
-                  int32_t flags)
+                  int32_t flags, dict_t *xdata)
 {
         intptr_t ax = GF_XATTR_NONE;
         int i = 0;
@@ -171,20 +223,60 @@ maccomp_setxattr (call_frame_t *frame, xlator_t *this, loc_t *loc, dict_t *dict,
                         break;
                 }
         }
+        dict_t *newdict = dict_new();
+        this->private = (void *) newdict;
+        dict_foreach(dict, prepend_xattr_user_namespace, this);
 
         this->private = (void *)ax;
-
+        int linux_flags = map_flags(flags);
+        gf_log (this->name, GF_LOG_DEBUG,
+                "setxattr flags: %d -> %d dict %p private: %p xdata %p ",
+                flags, linux_flags, dict, this->private, xdata);
         STACK_WIND (frame, maccomp_setxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->setxattr,
-                    loc, dict, flags);
+                    loc, newdict, linux_flags, xdata);
+        dict_unref(newdict);
+        return 0;
+}
+
+int32_t
+maccomp_setattr (call_frame_t *frame, xlator_t *this, loc_t *loc, struct iatt *iattr,
+                  int32_t flags, dict_t *xdata)
+{
+        gf_log (this->name, GF_LOG_DEBUG,
+                "setattr iattr %p private: %p xdata %p ",
+                iattr, this->private, xdata);
+        STACK_WIND (frame, maccomp_setattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setattr,
+                    loc, iattr, flags, xdata);
         return 0;
 }
 
+int32_t
+maccomp_removexattr (call_frame_t *frame, xlator_t *this, loc_t *loc,
+                     const char *name, dict_t *xdata)
+{
+        char *newkey = NULL;
+        this->private = (void *) check_name(name, &newkey);
+
+        STACK_WIND (frame, default_removexattr_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->removexattr,
+                    loc, newkey, xdata);
+
+        gf_log (this->name, GF_LOG_TRACE,
+                "removeattr name %p private: %p xdata %p ",
+                name, this->private, xdata);
+        GF_FREE(newkey);
+        return 0;
+
+}
 
 int32_t
 maccomp_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
-                   int32_t flags)
+                   int32_t flags, dict_t *xdata)
 {
         intptr_t ax = GF_XATTR_NONE;
         int i = 0;
@@ -197,12 +289,20 @@ maccomp_fsetxattr (call_frame_t *frame, xlator_t *this, fd_t *fd, dict_t *dict,
                 }
         }
 
-        this->private = (void *)ax;
+        dict_t *newdict = dict_new();
+        this->private = (void *) newdict;
+        dict_foreach(dict, prepend_xattr_user_namespace, this);
 
+        this->private = (void *)ax;
+        int linux_flags = map_flags(flags);
+        gf_log (this->name, GF_LOG_DEBUG,
+                "fsetxattr flags: %d -> %d dict %p private: %p xdata %p ",
+                flags, linux_flags, dict, this->private, xdata);
         STACK_WIND (frame, maccomp_setxattr_cbk,
                     FIRST_CHILD(this),
                     FIRST_CHILD(this)->fops->fsetxattr,
-                    fd, dict, flags);
+                    fd, newdict, linux_flags, xdata);
+        dict_unref(newdict);
         return 0;
 }
 
@@ -233,14 +333,16 @@ fini (xlator_t *this)
 
 
 struct xlator_fops fops = {
-        .getxattr    = maccomp_getxattr,
-        .fgetxattr   = maccomp_fgetxattr,
-        .setxattr    = maccomp_setxattr,
-        .fsetxattr   = maccomp_fsetxattr,
+        .getxattr       = maccomp_getxattr,
+        .fgetxattr      = maccomp_fgetxattr,
+        .setxattr       = maccomp_setxattr,
+        .setattr        = maccomp_setattr,
+        .fsetxattr      = maccomp_fsetxattr,
+        .removexattr    = maccomp_removexattr,
+        .fremovexattr   = maccomp_fremovexattr,
 };
 
-struct xlator_cbks cbks = {
-};
+struct xlator_cbks cbks;
 
 struct volume_options options[] = {
         { .key = {NULL} },
diff --git a/xlators/features/mac-compat/src/mac-compat.h b/xlators/features/mac-compat/src/mac-compat.h
new file mode 100644
index 000000000..b033ca0e4
--- /dev/null
+++ b/xlators/features/mac-compat/src/mac-compat.h
@@ -0,0 +1,41 @@
+/*
+   Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef __MAC_COMPAT_H__
+#define __MAC_COMPAT_H__
+
+enum apple_xattr {
+        GF_FINDER_INFO_XATTR,
+        GF_RESOURCE_FORK_XATTR,
+        GF_XATTR_ALL,
+        GF_XATTR_NONE
+};
+
+static char *apple_xattr_name[] = {
+        [GF_FINDER_INFO_XATTR]   = "com.apple.FinderInfo",
+        [GF_RESOURCE_FORK_XATTR] = "com.apple.ResourceFork"
+};
+
+static const char *apple_xattr_value[] = {
+        [GF_FINDER_INFO_XATTR]   =
+        /* 1 2 3 4 5 6 7 8 */
+         "\0\0\0\0\0\0\0\0"
+         "\0\0\0\0\0\0\0\0"
+         "\0\0\0\0\0\0\0\0"
+         "\0\0\0\0\0\0\0\0",
+        [GF_RESOURCE_FORK_XATTR] = ""
+};
+
+static int32_t apple_xattr_len[] = {
+        [GF_FINDER_INFO_XATTR]   = 32,
+        [GF_RESOURCE_FORK_XATTR] = 1
+};
+
+#endif /* __MAC_COMPAT_H__ */
diff --git a/xlators/features/marker/Makefile.am b/xlators/features/marker/Makefile.am
new file mode 100644
index 000000000..a985f42a8
--- /dev/null
+++ b/xlators/features/marker/Makefile.am
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES =
diff --git a/xlators/features/marker/src/Makefile.am b/xlators/features/marker/src/Makefile.am
new file mode 100644
index 000000000..a7c676472
--- /dev/null
+++ b/xlators/features/marker/src/Makefile.am
@@ -0,0 +1,17 @@
+xlator_LTLIBRARIES = marker.la
+xlatordir = $(libdir)/glusterfs/$(PACKAGE_VERSION)/xlator/features
+
+marker_la_LDFLAGS = -module -avoid-version
+
+marker_la_SOURCES = marker.c marker-quota.c marker-quota-helper.c marker-common.c
+marker_la_LIBADD = $(top_builddir)/libglusterfs/src/libglusterfs.la
+
+noinst_HEADERS = marker-mem-types.h marker.h marker-quota.h marker-quota-helper.h marker-common.h $(top_builddir)/xlators/lib/src/libxlator.h
+
+AM_CPPFLAGS = $(GF_CPPFLAGS) -I$(top_srcdir)/libglusterfs/src \
+	-I$(top_srcdir)/xlators/lib/src
+
+AM_CFLAGS = -Wall -fno-strict-aliasing $(GF_CFLAGS)
+
+CLEANFILES =
+
diff --git a/xlators/features/marker/src/marker-common.c b/xlators/features/marker/src/marker-common.c
new file mode 100644
index 000000000..84a718add
--- /dev/null
+++ b/xlators/features/marker/src/marker-common.c
@@ -0,0 +1,69 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+#include <fnmatch.h>
+#include "marker-common.h"
+
+marker_inode_ctx_t *
+marker_inode_ctx_new ()
+{
+        marker_inode_ctx_t *ctx = NULL;
+
+        ctx = GF_CALLOC (1, sizeof (marker_inode_ctx_t),
+                         gf_marker_mt_marker_inode_ctx_t);
+        if (ctx == NULL)
+                goto out;
+
+        ctx->quota_ctx = NULL;
+out:
+        return ctx;
+}
+
+int32_t
+marker_force_inode_ctx_get (inode_t *inode, xlator_t *this,
+                            marker_inode_ctx_t **ctx)
+{
+        int32_t  ret     = -1;
+        uint64_t ctx_int = 0;
+
+        LOCK (&inode->lock);
+        {
+                ret = __inode_ctx_get (inode, this, &ctx_int);
+                if (ret == 0)
+                        *ctx = (marker_inode_ctx_t *) (unsigned long)ctx_int;
+                else {
+                        *ctx = marker_inode_ctx_new ();
+                        if (*ctx == NULL)
+                                goto unlock;
+
+                        ret = __inode_ctx_put (inode, this,
+                                               (uint64_t )(unsigned long) *ctx);
+                        if (ret == -1) {
+                                GF_FREE (*ctx);
+                                goto unlock;
+                        }
+                        ret = 0;
+                }
+        }
+unlock: UNLOCK (&inode->lock);
+
+        return ret;
+}
+
+int
+marker_filter_quota_xattr (dict_t *dict, char *key,
+                           data_t *value, void *data)
+{
+        dict_del (dict, key);
+        return 0;
+}
diff --git a/xlators/features/marker/src/marker-common.h b/xlators/features/marker/src/marker-common.h
new file mode 100644
index 000000000..23dd846cb
--- /dev/null
+++ b/xlators/features/marker/src/marker-common.h
@@ -0,0 +1,27 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _MARKER_COMMON_H
+#define _MARKER_COMMON_H
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "inode.h"
+#include "xlator.h"
+#include "marker.h"
+
+int32_t
+marker_force_inode_ctx_get (inode_t *, xlator_t *, marker_inode_ctx_t **);
+
+int
+marker_filter_quota_xattr (dict_t *, char *, data_t *, void *);
+#endif
diff --git a/xlators/features/marker/src/marker-mem-types.h b/xlators/features/marker/src/marker-mem-types.h
new file mode 100644
index 000000000..1f74d5048
--- /dev/null
+++ b/xlators/features/marker/src/marker-mem-types.h
@@ -0,0 +1,25 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef __MARKER_MEM_TYPES_H__
+#define __MARKER_MEM_TYPES_H__
+
+#include "mem-types.h"
+
+enum gf_marker_mem_types_ {
+        gf_marker_mt_marker_conf_t = gf_common_mt_end + 1,
+        gf_marker_mt_loc_t,
+        gf_marker_mt_volume_mark,
+        gf_marker_mt_int64_t,
+        gf_marker_mt_quota_inode_ctx_t,
+        gf_marker_mt_marker_inode_ctx_t,
+        gf_marker_mt_inode_contribution_t,
+        gf_marker_mt_end
+};
+#endif
diff --git a/xlators/features/marker/src/marker-quota-helper.c b/xlators/features/marker/src/marker-quota-helper.c
new file mode 100644
index 000000000..ec0d83316
--- /dev/null
+++ b/xlators/features/marker/src/marker-quota-helper.c
@@ -0,0 +1,423 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "locking.h"
+#include "marker-quota.h"
+#include "marker-common.h"
+#include "marker-quota-helper.h"
+#include "marker-mem-types.h"
+
+int
+mq_loc_fill (loc_t *loc, inode_t *inode, inode_t *parent, char *path)
+{
+        int ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("marker", loc, out);
+        GF_VALIDATE_OR_GOTO ("marker", inode, out);
+        GF_VALIDATE_OR_GOTO ("marker", path, out);
+        /* Not checking for parent because while filling
+         * loc of root, parent will be NULL
+         */
+
+        if (inode) {
+                loc->inode = inode_ref (inode);
+        }
+
+        if (parent)
+                loc->parent = inode_ref (parent);
+
+        loc->path = gf_strdup (path);
+        if (!loc->path) {
+                gf_log ("loc fill", GF_LOG_ERROR, "strdup failed");
+                goto loc_wipe;
+        }
+
+        loc->name = strrchr (loc->path, '/');
+        if (loc->name)
+                loc->name++;
+        else
+                goto loc_wipe;
+
+        ret = 0;
+loc_wipe:
+        if (ret < 0)
+                loc_wipe (loc);
+out:
+        return ret;
+}
+
+
+int32_t
+mq_inode_loc_fill (const char *parent_gfid, inode_t *inode, loc_t *loc)
+{
+        char            *resolvedpath = NULL;
+        inode_t         *parent       = NULL;
+        int              ret          = -1;
+
+        if ((!inode) || (!loc))
+                return ret;
+
+        if ((inode) && __is_root_gfid (inode->gfid)) {
+                loc->parent = NULL;
+                goto ignore_parent;
+        }
+
+        if (parent_gfid == NULL)
+                parent = inode_parent (inode, 0, NULL);
+        else
+                parent = inode_find (inode->table,
+                                     (unsigned char *) parent_gfid);
+
+        if (parent == NULL)
+                goto err;
+
+ignore_parent:
+        ret = inode_path (inode, NULL, &resolvedpath);
+        if (ret < 0)
+                goto err;
+
+        ret = mq_loc_fill (loc, inode, parent, resolvedpath);
+        if (ret < 0)
+                goto err;
+
+err:
+        if (parent)
+                inode_unref (parent);
+
+        GF_FREE (resolvedpath);
+
+        return ret;
+}
+
+
+quota_inode_ctx_t *
+mq_alloc_inode_ctx ()
+{
+        int32_t                  ret    = -1;
+        quota_inode_ctx_t       *ctx    = NULL;
+
+        QUOTA_ALLOC (ctx, quota_inode_ctx_t, ret);
+        if (ret == -1)
+                goto out;
+
+        ctx->size = 0;
+        ctx->dirty = 0;
+        ctx->updation_status = _gf_false;
+        LOCK_INIT (&ctx->lock);
+        INIT_LIST_HEAD (&ctx->contribution_head);
+out:
+        return ctx;
+}
+
+inode_contribution_t *
+mq_get_contribution_node (inode_t *inode, quota_inode_ctx_t *ctx)
+{
+        inode_contribution_t    *contri = NULL;
+        inode_contribution_t    *temp   = NULL;
+
+        if (!inode || !ctx)
+                goto out;
+
+        list_for_each_entry (temp, &ctx->contribution_head, contri_list) {
+                if (uuid_compare (temp->gfid, inode->gfid) == 0) {
+                        contri = temp;
+                        goto out;
+                }
+        }
+out:
+        return contri;
+}
+
+
+int32_t
+mq_delete_contribution_node (dict_t *dict, char *key,
+                             inode_contribution_t *contribution)
+{
+        if (dict_get (dict, key) != NULL)
+                goto out;
+
+        QUOTA_FREE_CONTRIBUTION_NODE (contribution);
+out:
+        return 0;
+}
+
+
+inode_contribution_t *
+__mq_add_new_contribution_node (xlator_t *this, quota_inode_ctx_t *ctx,
+                                loc_t *loc)
+{
+        int32_t               ret          = 0;
+        inode_contribution_t *contribution = NULL;
+
+        if (!loc->parent) {
+                if (!uuid_is_null (loc->pargfid))
+                        loc->parent = inode_find (loc->inode->table,
+                                                  loc->pargfid);
+
+                if (!loc->parent)
+                        loc->parent = inode_parent (loc->inode, loc->pargfid,
+                                                    loc->name);
+                if (!loc->parent)
+                        goto out;
+        }
+
+        list_for_each_entry (contribution, &ctx->contribution_head,
+                             contri_list) {
+                if (loc->parent &&
+                    uuid_compare (contribution->gfid, loc->parent->gfid) == 0) {
+                        goto out;
+                }
+        }
+
+        QUOTA_ALLOC (contribution, inode_contribution_t, ret);
+        if (ret == -1)
+                goto out;
+
+        contribution->contribution = 0;
+
+        uuid_copy (contribution->gfid, loc->parent->gfid);
+
+	LOCK_INIT (&contribution->lock);
+        INIT_LIST_HEAD (&contribution->contri_list);
+
+        list_add_tail (&contribution->contri_list, &ctx->contribution_head);
+
+out:
+        return contribution;
+}
+
+
+inode_contribution_t *
+mq_add_new_contribution_node (xlator_t *this, quota_inode_ctx_t *ctx,
+                              loc_t *loc)
+{
+        inode_contribution_t *contribution = NULL;
+
+        if ((ctx == NULL) || (loc == NULL))
+                return NULL;
+
+        if (((loc->path) && (strcmp (loc->path, "/") == 0))
+            || (!loc->path && uuid_is_null (loc->pargfid)))
+                return NULL;
+
+        LOCK (&ctx->lock);
+        {
+                contribution = __mq_add_new_contribution_node (this, ctx, loc);
+        }
+        UNLOCK (&ctx->lock);
+
+        return contribution;
+}
+
+
+int32_t
+mq_dict_set_contribution (xlator_t *this, dict_t *dict,
+                          loc_t *loc)
+{
+        int32_t ret              = -1;
+        char    contri_key [512] = {0, };
+
+        GF_VALIDATE_OR_GOTO ("marker", this, out);
+        GF_VALIDATE_OR_GOTO ("marker", dict, out);
+        GF_VALIDATE_OR_GOTO ("marker", loc, out);
+
+        if (loc->parent) {
+                GET_CONTRI_KEY (contri_key, loc->parent->gfid, ret);
+                if (ret < 0) {
+                        ret = -1;
+                        goto out;
+                }
+        } else {
+                /* nameless lookup, fetch contributions to all parents */
+                GET_CONTRI_KEY (contri_key, NULL, ret);
+        }
+
+        ret = dict_set_int64 (dict, contri_key, 0);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "unable to set dict value on %s.",
+                        loc->path);
+                goto out;
+        }
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+int32_t
+mq_inode_ctx_get (inode_t *inode, xlator_t *this,
+                  quota_inode_ctx_t **ctx)
+{
+        int32_t             ret      = -1;
+        uint64_t            ctx_int  = 0;
+        marker_inode_ctx_t *mark_ctx = NULL;
+
+        GF_VALIDATE_OR_GOTO ("marker", inode, out);
+        GF_VALIDATE_OR_GOTO ("marker", this, out);
+        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
+
+        ret = inode_ctx_get (inode, this, &ctx_int);
+        if (ret < 0) {
+                ret = -1;
+                *ctx = NULL;
+                goto out;
+        }
+
+        mark_ctx = (marker_inode_ctx_t *) (unsigned long)ctx_int;
+        if (mark_ctx->quota_ctx == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        *ctx = mark_ctx->quota_ctx;
+
+        ret = 0;
+
+out:
+        return ret;
+}
+
+
+quota_inode_ctx_t *
+__mq_inode_ctx_new (inode_t *inode, xlator_t *this)
+{
+        int32_t               ret        = -1;
+        quota_inode_ctx_t    *quota_ctx  = NULL;
+        marker_inode_ctx_t   *mark_ctx   = NULL;
+
+        ret = marker_force_inode_ctx_get (inode, this, &mark_ctx);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_ERROR,
+                        "marker_force_inode_ctx_get() failed");
+                goto out;
+        }
+
+        LOCK (&inode->lock);
+        {
+                if (mark_ctx->quota_ctx == NULL) {
+                        quota_ctx = mq_alloc_inode_ctx ();
+                        if (quota_ctx == NULL) {
+                                ret = -1;
+                                goto unlock;
+                        }
+                        mark_ctx->quota_ctx = quota_ctx;
+                } else {
+                        quota_ctx = mark_ctx->quota_ctx;
+                }
+
+                ret = 0;
+        }
+unlock:
+        UNLOCK (&inode->lock);
+out:
+        return quota_ctx;
+}
+
+
+quota_inode_ctx_t *
+mq_inode_ctx_new (inode_t * inode, xlator_t *this)
+{
+        return __mq_inode_ctx_new (inode, this);
+}
+
+quota_local_t *
+mq_local_new ()
+{
+        quota_local_t  *local   = NULL;
+
+        local = mem_get0 (THIS->local_pool);
+        if (!local)
+                goto out;
+
+        local->ref = 1;
+        LOCK_INIT (&local->lock);
+
+        local->ctx = NULL;
+        local->contri = NULL;
+
+out:
+        return local;
+}
+
+quota_local_t *
+mq_local_ref (quota_local_t *local)
+{
+        LOCK (&local->lock);
+        {
+                local->ref ++;
+        }
+        UNLOCK (&local->lock);
+
+        return local;
+}
+
+
+int32_t
+mq_local_unref (xlator_t *this, quota_local_t *local)
+{
+        int32_t ref = 0;
+        if (local == NULL)
+                goto out;
+
+        QUOTA_SAFE_DECREMENT (&local->lock, local->ref, ref);
+
+        if (ref != 0)
+                goto out;
+
+        if (local->fd != NULL)
+                fd_unref (local->fd);
+
+        loc_wipe (&local->loc);
+
+        loc_wipe (&local->parent_loc);
+
+        LOCK_DESTROY (&local->lock);
+
+        mem_put (local);
+out:
+        return 0;
+}
+
+
+inode_contribution_t *
+mq_get_contribution_from_loc (xlator_t *this, loc_t *loc)
+{
+        int32_t               ret          = 0;
+        quota_inode_ctx_t    *ctx          = NULL;
+        inode_contribution_t *contribution = NULL;
+
+        ret = mq_inode_ctx_get (loc->inode, this, &ctx);
+        if (ret < 0) {
+                gf_log_callingfn (this->name, GF_LOG_WARNING,
+                                  "cannot get marker-quota context from inode "
+                                  "(gfid:%s, path:%s)",
+                                  uuid_utoa (loc->inode->gfid), loc->path);
+                goto err;
+        }
+
+        contribution = mq_get_contribution_node (loc->parent, ctx);
+        if (contribution == NULL) {
+                gf_log_callingfn (this->name, GF_LOG_WARNING,
+                                  "inode (gfid:%s, path:%s) has "
+                                  "no contribution towards parent (gfid:%s)",
+                                  uuid_utoa (loc->inode->gfid),
+                                  loc->path, uuid_utoa (loc->parent->gfid));
+                goto err;
+        }
+
+err:
+        return contribution;
+}
diff --git a/xlators/features/marker/src/marker-quota-helper.h b/xlators/features/marker/src/marker-quota-helper.h
new file mode 100644
index 000000000..b200413b0
--- /dev/null
+++ b/xlators/features/marker/src/marker-quota-helper.h
@@ -0,0 +1,76 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+
+#ifndef _MARKER_QUOTA_HELPER_H
+#define _MARKER_QUOTA_HELPER_H
+
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "marker.h"
+
+#define QUOTA_FREE_CONTRIBUTION_NODE(_contribution)     \
+        do {                                            \
+                list_del (&_contribution->contri_list); \
+                GF_FREE (_contribution);                \
+        } while (0)
+
+#define QUOTA_SAFE_INCREMENT(lock, var)                 \
+        do {                                            \
+                LOCK (lock);                            \
+                        var ++;                         \
+                UNLOCK (lock);                          \
+        } while (0)
+
+#define QUOTA_SAFE_DECREMENT(lock, var, value)  \
+        do {                                    \
+                LOCK (lock);                    \
+                {                               \
+                      value = --var;            \
+                }                               \
+                UNLOCK (lock);                  \
+        } while (0)
+
+inode_contribution_t *
+mq_add_new_contribution_node (xlator_t *, quota_inode_ctx_t *, loc_t *);
+
+int32_t
+mq_dict_set_contribution (xlator_t *, dict_t *, loc_t *);
+
+quota_inode_ctx_t *
+mq_inode_ctx_new (inode_t *, xlator_t *);
+
+int32_t
+mq_inode_ctx_get (inode_t *, xlator_t *, quota_inode_ctx_t **);
+
+int32_t
+mq_delete_contribution_node (dict_t *, char *, inode_contribution_t *);
+
+int32_t
+mq_inode_loc_fill (const char *, inode_t *, loc_t *);
+
+quota_local_t *
+mq_local_new ();
+
+quota_local_t *
+mq_local_ref (quota_local_t *);
+
+int32_t
+mq_local_unref (xlator_t *, quota_local_t *);
+
+inode_contribution_t *
+mq_get_contribution_node (inode_t *, quota_inode_ctx_t *);
+
+inode_contribution_t *
+mq_get_contribution_from_loc (xlator_t *this, loc_t *loc);
+
+#endif
diff --git a/xlators/features/marker/src/marker-quota.c b/xlators/features/marker/src/marker-quota.c
new file mode 100644
index 000000000..1903fdc40
--- /dev/null
+++ b/xlators/features/marker/src/marker-quota.c
@@ -0,0 +1,2705 @@
+/*
+   Copyright (c) 2008-2012 Red Hat, Inc. <http://www.redhat.com>
+   This file is part of GlusterFS.
+
+   This file is licensed to you under your choice of the GNU Lesser
+   General Public License, version 3 or any later version (LGPLv3 or
+   later), or the GNU General Public License, version 2 (GPLv2), in all
+   cases as published by the Free Software Foundation.
+*/
+#ifndef _CONFIG_H
+#define _CONFIG_H
+#include "config.h"
+#endif
+
+#include "dict.h"
+#include "xlator.h"
+#include "defaults.h"
+#include "libxlator.h"
+#include "common-utils.h"
+#include "byte-order.h"
+#include "marker-quota.h"
+#include "marker-quota-helper.h"
+
+int
+mq_loc_copy (loc_t *dst, loc_t *src)
+{
+        int ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("marker", dst, out);
+        GF_VALIDATE_OR_GOTO ("marker", src, out);
+
+        if (src->inode == NULL ||
+            ((src->parent == NULL) && (uuid_is_null (src->pargfid))
+             && !__is_root_gfid (src->inode->gfid))) {
+                gf_log ("marker", GF_LOG_WARNING,
+                        "src loc is not valid");
+                goto out;
+        }
+
+        ret = loc_copy (dst, src);
+out:
+        return ret;
+}
+
+int32_t
+mq_get_local_err (quota_local_t *local,
+                  int32_t *val)
+{
+        int32_t ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("marker", local, out);
+        GF_VALIDATE_OR_GOTO ("marker", val, out);
+
+        LOCK (&local->lock);
+        {
+                *val = local->err;
+        }
+        UNLOCK (&local->lock);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+mq_get_ctx_updation_status (quota_inode_ctx_t *ctx,
+                            gf_boolean_t *status)
+{
+        int32_t   ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
+        GF_VALIDATE_OR_GOTO ("marker", status, out);
+
+        LOCK (&ctx->lock);
+        {
+                *status = ctx->updation_status;
+        }
+        UNLOCK (&ctx->lock);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+
+int32_t
+mq_set_ctx_updation_status (quota_inode_ctx_t *ctx,
+                            gf_boolean_t status)
+{
+        int32_t   ret = -1;
+
+        if (ctx == NULL)
+                goto out;
+
+        LOCK (&ctx->lock);
+        {
+                ctx->updation_status = status;
+        }
+        UNLOCK (&ctx->lock);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+mq_test_and_set_ctx_updation_status (quota_inode_ctx_t *ctx,
+                                     gf_boolean_t *status)
+{
+        int32_t         ret     = -1;
+        gf_boolean_t    temp    = _gf_false;
+
+        GF_VALIDATE_OR_GOTO ("marker", ctx, out);
+        GF_VALIDATE_OR_GOTO ("marker", status, out);
+
+        LOCK (&ctx->lock);
+        {
+                temp = *status;
+                *status = ctx->updation_status;
+                ctx->updation_status = temp;
+        }
+        UNLOCK (&ctx->lock);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+void
+mq_assign_lk_owner (xlator_t *this, call_frame_t *frame)
+{
+        marker_conf_t *conf     = NULL;
+        uint64_t       lk_owner = 0;
+
+        conf = this->private;
+
+        LOCK (&conf->lock);
+        {
+                if (++conf->quota_lk_owner == 0) {
+                        ++conf->quota_lk_owner;
+                }
+
+                lk_owner = conf->quota_lk_owner;
+        }
+        UNLOCK (&conf->lock);
+
+        set_lk_owner_from_uint64 (&frame->root->lk_owner, lk_owner);
+
+        return;
+}
+
+
+int32_t
+mq_loc_fill_from_name (xlator_t *this, loc_t *newloc, loc_t *oldloc,
+                       uint64_t ino, char *name)
+{
+        int32_t   ret  = -1;
+        int32_t   len  = 0;
+        char     *path = NULL;
+
+        GF_VALIDATE_OR_GOTO ("marker", this, out);
+        GF_VALIDATE_OR_GOTO ("marker", newloc, out);
+        GF_VALIDATE_OR_GOTO ("marker", oldloc, out);
+        GF_VALIDATE_OR_GOTO ("marker", name, out);
+
+        newloc->inode = inode_new (oldloc->inode->table);
+
+        if (!newloc->inode) {
+                ret = -1;
+                goto out;
+        }
+
+        newloc->parent = inode_ref (oldloc->inode);
+        uuid_copy (newloc->pargfid, oldloc->inode->gfid);
+
+        len = strlen (oldloc->path);
+
+        if (oldloc->path [len - 1] == '/')
+                ret = gf_asprintf ((char **) &path, "%s%s",
+                                   oldloc->path, name);
+        else
+                ret = gf_asprintf ((char **) &path, "%s/%s",
+                                   oldloc->path, name);
+
+        if (ret < 0)
+                goto out;
+
+        newloc->path = path;
+
+        newloc->name = strrchr (newloc->path, '/');
+
+        if (newloc->name)
+                newloc->name++;
+
+        gf_log (this->name, GF_LOG_DEBUG, "path = %s name =%s",
+                newloc->path, newloc->name);
+out:
+        return ret;
+}
+
+int32_t
+mq_dirty_inode_updation_done (call_frame_t *frame, void *cookie, xlator_t *this,
+                              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        QUOTA_STACK_DESTROY (frame, this);
+
+        return 0;
+}
+
+int32_t
+mq_release_lock_on_dirty_inode (call_frame_t *frame, void *cookie, xlator_t *this,
+                                int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        struct gf_flock   lock  = {0, };
+        quota_local_t    *local = NULL;
+        loc_t             loc = {0, };
+	int               ret = -1;
+
+        local = frame->local;
+
+        if (op_ret == -1) {
+                local->err = -1;
+
+                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0, NULL);
+
+                return 0;
+        }
+
+        if (op_ret == 0)
+                local->ctx->dirty = 0;
+
+        lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+        ret = loc_copy (&loc, &local->loc);
+	if (ret == -1) {
+                local->err = -1;
+                frame->local = NULL;
+                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0, NULL);
+                return 0;
+        }
+
+        if (local->loc.inode == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Inode is NULL, so can't stackwind.");
+                goto out;
+        }
+
+        STACK_WIND (frame,
+                    mq_dirty_inode_updation_done,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->inodelk,
+                    this->name, &loc, F_SETLKW, &lock, NULL);
+
+        loc_wipe (&loc);
+
+        return 0;
+out:
+        mq_dirty_inode_updation_done (frame, NULL, this, -1, 0, NULL);
+
+        return 0;
+}
+
+int32_t
+mq_mark_inode_undirty (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
+{
+        int32_t        ret     = -1;
+        int64_t       *size    = NULL;
+        dict_t        *newdict = NULL;
+        quota_local_t *local   = NULL;
+
+        local = (quota_local_t *) frame->local;
+
+        if (op_ret == -1)
+                goto err;
+
+        if (!dict)
+                goto wind;
+
+        ret = dict_get_bin (dict, QUOTA_SIZE_KEY, (void **) &size);
+        if (ret)
+                goto wind;
+
+        LOCK (&local->ctx->lock);
+        {
+                local->ctx->size = ntoh64 (*size);
+        }
+        UNLOCK (&local->ctx->lock);
+
+wind:
+        newdict = dict_new ();
+        if (!newdict)
+                goto err;
+
+        ret = dict_set_int8 (newdict, QUOTA_DIRTY_KEY, 0);
+        if (ret)
+                goto err;
+
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+
+        GF_UUID_ASSERT (local->loc.gfid);
+        STACK_WIND (frame, mq_release_lock_on_dirty_inode,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->setxattr,
+                    &local->loc, newdict, 0, NULL);
+        ret = 0;
+
+err:
+        if (op_ret == -1 || ret == -1) {
+                local->err = -1;
+
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
+        }
+
+        if (newdict)
+                dict_unref (newdict);
+
+        return 0;
+}
+
+int32_t
+mq_update_size_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
+                      int32_t op_ret, int32_t op_errno, inode_t *inode,
+                      struct iatt *buf, dict_t *dict, struct iatt *postparent)
+{
+        int32_t          ret      = -1;
+        dict_t          *new_dict = NULL;
+        int64_t         *size     = NULL;
+        int64_t         *delta    = NULL;
+        quota_local_t   *local    = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1)
+                goto err;
+
+        if (dict == NULL) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Dict is null while updating the size xattr %s",
+                        local->loc.path?local->loc.path:"");
+                goto err;
+        }
+
+        ret = dict_get_bin (dict, QUOTA_SIZE_KEY, (void **) &size);
+        if (!size) {
+                gf_log (this->name, GF_LOG_WARNING,
+                        "failed to get the size, %s",
+                        local->loc.path?local->loc.path:"");
+                goto err;
+        }
+
+        QUOTA_ALLOC_OR_GOTO (delta, int64_t, ret, err);
+
+        *delta = hton64 (local->sum - ntoh64 (*size));
+
+        gf_log (this->name, GF_LOG_DEBUG, "calculated size = %"PRId64", "
+                "original size = %"PRIu64
+                " path = %s diff = %"PRIu64, local->sum, ntoh64 (*size),
+                local->loc.path, ntoh64 (*delta));
+
+        new_dict = dict_new ();
+        if (!new_dict) {
+		errno = ENOMEM;
+		goto err;
+	}
+
+        ret = dict_set_bin (new_dict, QUOTA_SIZE_KEY, delta, 8);
+        if (ret)
+                goto err;
+
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+
+        GF_UUID_ASSERT (local->loc.gfid);
+
+        STACK_WIND (frame, mq_mark_inode_undirty, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->xattrop, &local->loc,
+                    GF_XATTROP_ADD_ARRAY64, new_dict, NULL);
+
+        ret = 0;
+
+err:
+        if (op_ret == -1 || ret == -1) {
+                local->err = -1;
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
+        }
+
+        if (new_dict)
+                dict_unref (new_dict);
+
+        return 0;
+}
+
+int32_t
+mq_test_and_set_local_err(quota_local_t *local,
+                          int32_t *val)
+{
+        int     tmp = 0;
+        int32_t ret = -1;
+
+        GF_VALIDATE_OR_GOTO ("marker", local, out);
+        GF_VALIDATE_OR_GOTO ("marker", val, out);
+
+        LOCK (&local->lock);
+        {
+                tmp = local->err;
+                local->err = *val;
+                *val = tmp;
+        }
+        UNLOCK (&local->lock);
+
+        ret = 0;
+out:
+        return ret;
+}
+
+int32_t
+mq_get_dirty_inode_size (call_frame_t *frame, xlator_t *this)
+{
+        int32_t        ret   = -1;
+        dict_t        *dict  = NULL;
+        quota_local_t *local = NULL;
+
+        local = (quota_local_t *) frame->local;
+
+        dict = dict_new ();
+        if (!dict) {
+                ret = -1;
+                goto err;
+        }
+
+        ret = dict_set_int64 (dict, QUOTA_SIZE_KEY, 0);
+        if (ret)
+                goto err;
+
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+
+        GF_UUID_ASSERT (local->loc.gfid);
+
+        STACK_WIND (frame, mq_update_size_xattr, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, &local->loc, dict);
+        ret =0;
+
+err:
+        if (ret) {
+                local->err = -1;
+
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
+        }
+
+        if (dict)
+                dict_unref (dict);
+
+        return 0;
+}
+
+int32_t
+mq_get_child_contribution (call_frame_t *frame,
+                           void *cookie,
+                           xlator_t *this,
+                           int32_t op_ret,
+                           int32_t op_errno,
+                           inode_t *inode,
+                           struct iatt *buf,
+                           dict_t *dict,
+                           struct iatt *postparent)
+{
+        int32_t        ret                = -1;
+        int32_t        val                = 0;
+        char           contri_key [512]   = {0, };
+        int64_t       *contri             = NULL;
+        quota_local_t *local              = NULL;
+
+        local = frame->local;
+
+        frame->local = NULL;
+
+        QUOTA_STACK_DESTROY (frame, this);
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR, "%s",
+                        strerror (op_errno));
+                val = -2;
+                if (!mq_test_and_set_local_err (local, &val) &&
+                    val != -2)
+                        mq_release_lock_on_dirty_inode (local->frame, NULL,
+                                                        this, 0, 0, NULL);
+
+                goto exit;
+        }
+
+        ret = mq_get_local_err (local, &val);
+        if (!ret && val == -2)
+                goto exit;
+
+        GET_CONTRI_KEY (contri_key, local->loc.inode->gfid, ret);
+        if (ret < 0)
+                goto out;
+
+        if (!dict)
+                goto out;
+
+        if (dict_get_bin (dict, contri_key, (void **) &contri) == 0)
+                local->sum += ntoh64 (*contri);
+
+out:
+        LOCK (&local->lock);
+        {
+                val = --local->dentry_child_count;
+        }
+        UNLOCK (&local->lock);
+
+        if (val == 0) {
+                mq_dirty_inode_readdir (local->frame, NULL, this,
+                                        0, 0, NULL, NULL);
+        }
+        mq_local_unref (this, local);
+
+        return 0;
+exit:
+        mq_local_unref (this, local);
+        return 0;
+}
+
+int32_t
+mq_readdir_cbk (call_frame_t *frame,
+                void *cookie,
+                xlator_t *this,
+                int32_t op_ret,
+                int32_t op_errno,
+                gf_dirent_t *entries, dict_t *xdata)
+{
+        char           contri_key [512]   = {0, };
+        int32_t        ret                = 0;
+        int32_t        val                = 0;
+        off_t          offset             = 0;
+        int32_t        count              = 0;
+        dict_t        *dict               = NULL;
+        quota_local_t *local              = NULL;
+        gf_dirent_t   *entry              = NULL;
+        call_frame_t  *newframe           = NULL;
+        loc_t          loc                = {0, };
+
+        local = mq_local_ref (frame->local);
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_DEBUG,
+                        "readdir failed %s", strerror (op_errno));
+                local->err = -1;
+
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
+
+                goto end;
+        } else if (op_ret == 0) {
+                mq_get_dirty_inode_size (frame, this);
+
+                goto end;
+        }
+
+        local->dentry_child_count =  0;
+
+        list_for_each_entry (entry, (&entries->list), list) {
+                gf_log (this->name, GF_LOG_DEBUG, "entry  = %s", entry->d_name);
+
+                if ((!strcmp (entry->d_name, ".")) || (!strcmp (entry->d_name,
+                                                                ".."))) {
+                        gf_log (this->name, GF_LOG_DEBUG, "entry  = %s",
+                                entry->d_name);
+                        continue;
+                }
+
+                offset = entry->d_off;
+                count++;
+        }
+
+        if (count == 0) {
+                mq_get_dirty_inode_size (frame, this);
+                goto end;
+
+        }
+
+        local->frame = frame;
+
+        LOCK (&local->lock);
+        {
+                local->dentry_child_count = count;
+                local->d_off = offset;
+        }
+        UNLOCK (&local->lock);
+
+
+        list_for_each_entry (entry, (&entries->list), list) {
+                gf_log (this->name, GF_LOG_DEBUG, "entry  = %s", entry->d_name);
+
+                if ((!strcmp (entry->d_name, ".")) || (!strcmp (entry->d_name,
+                                                                ".."))) {
+                        gf_log (this->name, GF_LOG_DEBUG, "entry  = %s",
+                                entry->d_name);
+                        continue;
+                }
+
+                ret = mq_loc_fill_from_name (this, &loc, &local->loc,
+                                             entry->d_ino, entry->d_name);
+                if (ret < 0)
+                        goto out;
+
+                ret = 0;
+
+                LOCK (&local->lock);
+                {
+                        if (local->err != -2) {
+                                newframe = copy_frame (frame);
+                                if (!newframe) {
+                                        ret = -1;
+                                }
+                        } else
+                                ret = -1;
+                }
+                UNLOCK (&local->lock);
+
+                if (ret == -1)
+                        goto out;
+
+                newframe->local = mq_local_ref (local);
+
+                dict = dict_new ();
+                if (!dict) {
+                        ret = -1;
+                        goto out;
+                }
+
+                GET_CONTRI_KEY (contri_key, local->loc.inode->gfid, ret);
+                if (ret < 0)
+                        goto out;
+
+                ret = dict_set_int64 (dict, contri_key, 0);
+                if (ret)
+                        goto out;
+
+                STACK_WIND (newframe,
+                            mq_get_child_contribution,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->lookup,
+                            &loc, dict);
+
+                offset = entry->d_off;
+
+                loc_wipe (&loc);
+
+                newframe = NULL;
+
+        out:
+                if (dict) {
+                        dict_unref (dict);
+                        dict = NULL;
+                }
+
+                if (ret) {
+                        val = -2;
+                        mq_test_and_set_local_err (local, &val);
+
+                        if (newframe) {
+                                newframe->local = NULL;
+                                mq_local_unref(this, local);
+                                QUOTA_STACK_DESTROY (newframe, this);
+                        }
+
+                        break;
+                }
+        }
+
+        if (ret && val != -2) {
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
+        }
+end:
+        mq_local_unref (this, local);
+
+        return 0;
+}
+
+int32_t
+mq_dirty_inode_readdir (call_frame_t *frame,
+                        void *cookie,
+                        xlator_t *this,
+                        int32_t op_ret,
+                        int32_t op_errno,
+                        fd_t *fd, dict_t *xdata)
+{
+        quota_local_t *local = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1) {
+                local->err = -1;
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
+                return 0;
+        }
+
+        if (local->fd == NULL)
+                local->fd = fd_ref (fd);
+
+        STACK_WIND (frame,
+                    mq_readdir_cbk,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->readdir,
+                    local->fd, READDIR_BUF, local->d_off, xdata);
+
+        return 0;
+}
+
+int32_t
+mq_check_if_still_dirty (call_frame_t *frame,
+                         void *cookie,
+                         xlator_t *this,
+                         int32_t op_ret,
+                         int32_t op_errno,
+                         inode_t *inode,
+                         struct iatt *buf,
+                         dict_t *dict,
+                         struct iatt *postparent)
+{
+        int8_t           dirty          = -1;
+        int32_t          ret            = -1;
+        fd_t            *fd             = NULL;
+        quota_local_t   *local          = NULL;
+
+        local = frame->local;
+
+        if (op_ret == -1) {
+                gf_log (this->name, GF_LOG_ERROR, "failed to get "
+                        "the dirty xattr for %s", local->loc.path);
+                goto err;
+        }
+
+        if (!dict) {
+                ret = -1;
+                goto err;
+        }
+
+        ret = dict_get_int8 (dict, QUOTA_DIRTY_KEY, &dirty);
+        if (ret)
+                goto err;
+
+        //the inode is not dirty anymore
+        if (dirty == 0) {
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
+
+                return 0;
+        }
+
+        fd = fd_create (local->loc.inode, frame->root->pid);
+
+        local->d_off = 0;
+
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+
+        GF_UUID_ASSERT (local->loc.gfid);
+        STACK_WIND(frame,
+                   mq_dirty_inode_readdir,
+                   FIRST_CHILD(this),
+                   FIRST_CHILD(this)->fops->opendir,
+                   &local->loc, fd, NULL);
+
+        ret = 0;
+
+err:
+        if (op_ret == -1 || ret == -1) {
+                local->err = -1;
+                mq_release_lock_on_dirty_inode (frame, NULL, this, 0, 0, NULL);
+        }
+
+        if (fd != NULL) {
+                fd_unref (fd);
+        }
+
+        return 0;
+}
+
+int32_t
+mq_get_dirty_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
+                    int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        int32_t        ret       = -1;
+        dict_t        *xattr_req = NULL;
+        quota_local_t *local     = NULL;
+
+        if (op_ret == -1) {
+                mq_dirty_inode_updation_done (frame, NULL, this, 0, 0, NULL);
+                return 0;
+        }
+
+        local = frame->local;
+
+        xattr_req = dict_new ();
+        if (xattr_req == NULL) {
+                ret = -1;
+                goto err;
+        }
+
+        ret = dict_set_int8 (xattr_req, QUOTA_DIRTY_KEY, 0);
+        if (ret)
+                goto err;
+
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+
+        GF_UUID_ASSERT (local->loc.gfid);
+
+        STACK_WIND (frame,
+                    mq_check_if_still_dirty,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup,
+                    &local->loc,
+                    xattr_req);
+        ret = 0;
+
+err:
+        if (ret) {
+                local->err = -1;
+                mq_release_lock_on_dirty_inode(frame, NULL, this, 0, 0, NULL);
+        }
+
+        if (xattr_req)
+                dict_unref (xattr_req);
+
+        return 0;
+}
+
+/* return 1 when dirty updation started
+ * 0 other wise
+ */
+int32_t
+mq_update_dirty_inode (xlator_t *this,
+                       loc_t *loc,
+                       quota_inode_ctx_t *ctx,
+                       inode_contribution_t *contribution)
+{
+        int32_t          ret        = -1;
+        quota_local_t   *local      = NULL;
+        gf_boolean_t    status     = _gf_false;
+        struct gf_flock  lock       = {0, };
+        call_frame_t    *frame      = NULL;
+
+        ret = mq_get_ctx_updation_status (ctx, &status);
+        if (ret == -1 || status == _gf_true) {
+                ret = 0;
+                goto out;
+        }
+
+        frame = create_frame (this, this->ctx->pool);
+        if (frame == NULL) {
+                ret = -1;
+                goto out;
+        }
+
+        mq_assign_lk_owner (this, frame);
+
+        local = mq_local_new ();
+        if (local == NULL)
+                goto fr_destroy;
+
+        frame->local = local;
+        ret = mq_loc_copy (&local->loc, loc);
+        if (ret < 0)
+                goto fr_destroy;
+
+        local->ctx = ctx;
+
+        local->contri = contribution;
+
+        lock.l_type = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start = 0;
+        lock.l_len = 0;
+
+        if (local->loc.inode == NULL) {
+                ret = -1;
+                gf_log (this->name, GF_LOG_WARNING,
+                        "Inode is NULL, so can't stackwind.");
+                goto fr_destroy;
+        }
+
+        STACK_WIND (frame,
+                    mq_get_dirty_xattr,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->inodelk,
+                    this->name, &local->loc, F_SETLKW, &lock, NULL);
+        return 1;
+
+fr_destroy:
+        QUOTA_STACK_DESTROY (frame, this);
+out:
+
+        return 0;
+}
+
+
+int32_t
+mq_inode_creation_done (call_frame_t *frame, void *cookie, xlator_t *this,
+                        int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        quota_local_t *local = NULL;
+
+        if (frame == NULL)
+                return 0;
+
+        local = frame->local;
+
+        if (local != NULL) {
+                mq_initiate_quota_txn (this, &local->loc);
+        }
+
+        QUOTA_STACK_DESTROY (frame, this);
+
+        return 0;
+}
+
+
+int32_t
+mq_xattr_creation_release_lock (call_frame_t *frame, void *cookie,
+                                xlator_t *this, int32_t op_ret,
+                                int32_t op_errno, dict_t *xdata)
+{
+        struct gf_flock  lock  = {0, };
+        quota_local_t   *local = NULL;
+
+        local = frame->local;
+
+        lock.l_type   = F_UNLCK;
+        lock.l_whence = SEEK_SET;
+        lock.l_start  = 0;
+        lock.l_len    = 0;
+        lock.l_pid    = 0;
+
+        STACK_WIND (frame,
+                    mq_inode_creation_done,
+                    FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->inodelk,
+                    this->name, &local->loc,
+                    F_SETLKW, &lock, NULL);
+
+        return 0;
+}
+
+
+int32_t
+mq_create_dirty_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
+                       int32_t op_ret, int32_t op_errno, dict_t *dict,
+                       dict_t *xdata)
+{
+        int32_t          ret       = -1;
+        dict_t          *newdict   = NULL;
+        quota_local_t   *local     = NULL;
+
+        if (op_ret < 0) {
+                goto err;
+        }
+
+        local = frame->local;
+
+        if (local->loc.inode->ia_type == IA_IFDIR) {
+                newdict = dict_new ();
+                if (!newdict) {
+                        goto err;
+                }
+
+                ret = dict_set_int8 (newdict, QUOTA_DIRTY_KEY, 0);
+                if (ret == -1) {
+                        goto err;
+                }
+
+                uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+                GF_UUID_ASSERT (local->loc.gfid);
+
+                STACK_WIND (frame, mq_xattr_creation_release_lock,
+                            FIRST_CHILD(this),
+                            FIRST_CHILD(this)->fops->setxattr,
+                            &local->loc, newdict, 0, NULL);
+        } else {
+                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
+        }
+
+        ret = 0;
+
+err:
+        if (ret < 0) {
+                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
+        }
+
+        if (newdict != NULL)
+                dict_unref (newdict);
+
+        return 0;
+}
+
+
+int32_t
+mq_create_xattr (xlator_t *this, call_frame_t *frame)
+{
+        int32_t               ret       = 0;
+        int64_t              *value     = NULL;
+        int64_t              *size      = NULL;
+        dict_t               *dict      = NULL;
+        char                  key[512]  = {0, };
+        quota_local_t        *local     = NULL;
+        quota_inode_ctx_t    *ctx       = NULL;
+        inode_contribution_t *contri    = NULL;
+
+        if (frame == NULL || this == NULL)
+                return 0;
+
+        local = frame->local;
+
+        ret = mq_inode_ctx_get (local->loc.inode, this, &ctx);
+        if (ret < 0) {
+                ctx = mq_inode_ctx_new (local->loc.inode, this);
+                if (ctx == NULL) {
+                        gf_log (this->name, GF_LOG_WARNING,
+                                "mq_inode_ctx_new failed");
+                        ret = -1;
+                        goto out;
+                }
+        }
+
+        dict = dict_new ();
+        if (!dict)
+                goto out;
+
+        if (local->loc.inode->ia_type == IA_IFDIR) {
+                QUOTA_ALLOC_OR_GOTO (size, int64_t, ret, err);
+                ret = dict_set_bin (dict, QUOTA_SIZE_KEY, size, 8);
+                if (ret < 0)
+                        goto free_size;
+        }
+
+        if ((local->loc.path && strcmp (local->loc.path, "/") != 0)
+            || (local->loc.inode && !uuid_is_null (local->loc.inode->gfid) &&
+                !__is_root_gfid (local->loc.inode->gfid))
+            || (!uuid_is_null (local->loc.gfid)
+                && !__is_root_gfid (local->loc.gfid))) {
+                contri = mq_add_new_contribution_node (this, ctx, &local->loc);
+                if (contri == NULL)
+                        goto err;
+
+                QUOTA_ALLOC_OR_GOTO (value, int64_t, ret, err);
+                GET_CONTRI_KEY (key, local->loc.parent->gfid, ret);
+
+                ret = dict_set_bin (dict, key, value, 8);
+                if (ret < 0)
+                        goto free_value;
+        }
+
+        GF_UUID_ASSERT (local->loc.gfid);
+
+        STACK_WIND (frame, mq_create_dirty_xattr, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->xattrop, &local->loc,
+                    GF_XATTROP_ADD_ARRAY64, dict, NULL);
+        ret = 0;
+
+free_size:
+        if (ret < 0) {
+                GF_FREE (size);
+        }
+
+free_value:
+        if (ret < 0) {
+                GF_FREE (value);
+        }
+
+err:
+        dict_unref (dict);
+
+out:
+        if (ret < 0) {
+                mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
+        }
+
+        return 0;
+}
+
+
+int32_t
+mq_check_n_set_inode_xattr (call_frame_t *frame, void *cookie,
+                            xlator_t *this, int32_t op_ret, int32_t op_errno,
+                            inode_t *inode, struct iatt *buf, dict_t *dict,
+                            struct iatt *postparent)
+{
+        quota_local_t        *local           = NULL;
+        int64_t              *size            = NULL, *contri = NULL;
+        int8_t                dirty           = 0;
+        int32_t               ret             = 0;
+        char                  contri_key[512] = {0, };
+
+        if (op_ret < 0) {
+                goto out;
+        }
+
+        local = frame->local;
+
+        ret = dict_get_bin (dict, QUOTA_SIZE_KEY, (void **) &size);
+        if (ret < 0)
+                goto create_xattr;
+
+        ret = dict_get_int8 (dict, QUOTA_DIRTY_KEY, &dirty);
+        if (ret < 0)
+                goto create_xattr;
+
+        //check contribution xattr if not root
+        if ((local->loc.path && strcmp (local->loc.path, "/") != 0)
+            || (!uuid_is_null (local->loc.gfid)
+                && !__is_root_gfid (local->loc.gfid))
+            || (local->loc.inode
+                && !uuid_is_null (local->loc.inode->gfid)
+                && !__is_root_gfid (local->loc.inode->gfid))) {
+                GET_CONTRI_KEY (contri_key, local->loc.parent->gfid, ret);
+                if (ret < 0)
+                        goto out;
+
+                ret = dict_get_bin (dict, contri_key, (void **) &contri);
+                if (ret < 0)
+                        goto create_xattr;
+        }
+
+out:
+        mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
+        return 0;
+
+create_xattr:
+        if (uuid_is_null (local->loc.gfid)) {
+                uuid_copy (local->loc.gfid, buf->ia_gfid);
+        }
+
+        mq_create_xattr (this, frame);
+        return 0;
+}
+
+
+int32_t
+mq_get_xattr (call_frame_t *frame, void *cookie, xlator_t *this,
+              int32_t op_ret, int32_t op_errno, dict_t *xdata)
+{
+        dict_t        *xattr_req = NULL;
+        quota_local_t *local     = NULL;
+        int32_t        ret       = 0;
+
+        if (op_ret < 0) {
+                goto lock_err;
+        }
+
+        local = frame->local;
+
+        xattr_req = dict_new ();
+        if (xattr_req == NULL) {
+                goto err;
+        }
+
+        ret = mq_req_xattr (this, &local->loc, xattr_req);
+        if (ret < 0) {
+                gf_log (this->name, GF_LOG_WARNING, "cannot request xattr");
+                goto err;
+        }
+
+        if (uuid_is_null (local->loc.gfid))
+                uuid_copy (local->loc.gfid, local->loc.inode->gfid);
+
+        GF_UUID_ASSERT (local->loc.gfid);
+
+        STACK_WIND (frame, mq_check_n_set_inode_xattr, FIRST_CHILD(this),
+                    FIRST_CHILD(this)->fops->lookup, &local->loc, xattr_req);
+
+        dict_unref (xattr_req);
+
+        return 0;
+
+err:
+        mq_xattr_creation_release_lock (frame, NULL, this, 0, 0, NULL);
+
+        if (xattr_req)
+                dict_unref (xattr_req);
+        return 0;
+
+lock_err:
+        mq_inode_creation_done (frame, NULL, this, 0, 0, NULL);
+        return 0;
+}
+
+
+int32_t
+mq_set_inode_xattr (xlator_t *this, loc_t *loc)
+{
+        struct gf_flock  lock  = {0, };
+        quota_local_t   *local = NULL;
+        int32_t          ret   = 0;
+        call_frame_t    *frame = NULL;
+
+        frame = create_frame (this, this->ctx->pool);
+        if (!frame) {
+                ret = -1;
+                goto err;
+        }
+
+        local = mq_local_new ();
+        if (local == NULL) {
+                goto err;
+        }
+
+        frame->local = local;
+
+        ret = loc_copy (&local->loc, loc);
+        if (ret < 0) {
+                goto err;
+        }
+
+        frame->local = local;
+
+        lock.l_len    = 0;
+        lock.l_start  = 0;
+        lock.l_type   = F_WRLCK;
+        lock.l_whence = SEEK_SET;
+
+        STACK_WIND (frame,
+
author	Pranith Kumar K <pkarampu@redhat.com>	2013-05-15 12:09:28 +0530
committer	Vijay Bellur <vbellur@redhat.com>	2013-05-31 11:22:24 -0700
commit	9e974f9311869fb317c9c691ed09e7dd18047248 (patch)
tree	598cbff63e2a10a3eef4a3b8e4128331f5b5b810 /rpc
parent	114c02e54abae9b3cbf0f84bc48132bf1535118e (diff)