From 4c0593cb6011c8382cf0c29438635f3b41c4e149 Mon Sep 17 00:00:00 2001
From: Martin Krizek <martin.krizek@gmail.com>
Date: Tue, 21 May 2013 17:56:04 -0400
Subject: [PATCH] Add online deduplication

This patch adds online deduplication.

Deduplication is online (or immediate), meaning that duplicates are eliminated
before being written on disk. It is the block level kind of deduplication.
The deduplication block size is variable -- can be set by a mount option. A new 
tree is created for holding hashes of blocks. sha256 is used as hashing algorithm. 
Description of usage follows:

Create a btrfs partition.
mkfs.btrfs -f /dev/sda

Turn on deduplication.
mount -o dedup /dev/sda /mnt

Set deduplication block size to 8192 bytes.
mount -o dedup,dedupsize=8192 /dev/sda /mnt

Turn on byte--by--byte block comparison.
mount -o dedup,dedupverify=on /dev/sda /mnt

Set deduplication block size to 16384 bytes and turn on byte--by--byte block 
comparison.
mount -o dedup,dedupsize=16384,dedupverify=on /dev/sda /mnt

When deduplication turned on a non-empty filesystem, one can run the following
ioctl: BTRFS_IOC_DEDUP_SYNC
(Note that the ioctl does not work when deduplication is turned on, off and on
again.)
---
 fs/btrfs/Makefile          |   2 +-
 fs/btrfs/ctree.h           |  73 ++++++
 fs/btrfs/dedup.c           | 544 +++++++++++++++++++++++++++++++++++++++++++++
 fs/btrfs/dedup.h           |  64 ++++++
 fs/btrfs/disk-io.c         |  25 ++-
 fs/btrfs/extent-tree.c     |   2 +
 fs/btrfs/inode.c           | 363 ++++++++++++++++++++++++++++--
 fs/btrfs/ioctl.c           |  20 ++
 fs/btrfs/ordered-data.c    |  29 ++-
 fs/btrfs/ordered-data.h    |  17 ++
 fs/btrfs/super.c           |  44 +++-
 include/uapi/linux/btrfs.h |   1 +
 12 files changed, 1164 insertions(+), 20 deletions(-)
 create mode 100644 fs/btrfs/dedup.c
 create mode 100644 fs/btrfs/dedup.h

diff --git a/fs/btrfs/Makefile b/fs/btrfs/Makefile
index 3932224..d1e7ce3 100644
--- a/fs/btrfs/Makefile
+++ b/fs/btrfs/Makefile
@@ -8,7 +8,7 @@ btrfs-y += super.o ctree.o extent-tree.o print-tree.o root-tree.o dir-item.o \
 	   extent_io.o volumes.o async-thread.o ioctl.o locking.o orphan.o \
 	   export.o tree-log.o free-space-cache.o zlib.o lzo.o \
 	   compression.o delayed-ref.o relocation.o delayed-inode.o scrub.o \
-	   reada.o backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o
+	   reada.o backref.o ulist.o qgroup.o send.o dev-replace.o raid56.o dedup.o
 
 btrfs-$(CONFIG_BTRFS_FS_POSIX_ACL) += acl.o
 btrfs-$(CONFIG_BTRFS_FS_CHECK_INTEGRITY) += check-integrity.o
diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
index d6dd49b..8492a52 100644
--- a/fs/btrfs/ctree.h
+++ b/fs/btrfs/ctree.h
@@ -91,6 +91,9 @@ struct btrfs_ordered_sum;
 /* holds quota configuration and tracking */
 #define BTRFS_QUOTA_TREE_OBJECTID 8ULL
 
+/* holds hashes of all the data extents for dedup */
+#define BTRFS_DEDUP_TREE_OBJECTID 9ULL
+
 /* for storing balance parameters in the root tree */
 #define BTRFS_BALANCE_OBJECTID -4ULL
 
@@ -892,6 +895,34 @@ struct btrfs_csum_item {
 	u8 csum;
 } __attribute__ ((__packed__));
 
+struct btrfs_dedup_item {
+	__le64 disk_bytenr;
+	__le64 disk_num_bytes;
+	__le64 extent_offset;
+	__le64 len;
+	__le64 hash0;
+	__le64 hash1;
+} __attribute__ ((__packed__));
+
+struct btrfs_dedup_hash_item {
+	__le64 hash0;
+	__le64 hash1;
+	__le64 hash2;
+	__le64 hash3;
+} __attribute__ ((__packed__));
+
+
+/* holds metadata to be inserted into the dedup tree  */
+struct btrfs_duplicate {
+	struct inode *inode;
+	u64 disk_bytenr;
+	u64 num_bytes;
+	u64 extent_offset;
+	u64 file_offset;
+	u64 *inram;
+	struct list_head list;
+};
+
 struct btrfs_dev_stats_item {
 	/*
 	 * grow this item struct at the end for future enhancements and keep
@@ -1290,6 +1321,7 @@ struct btrfs_fs_info {
 	struct btrfs_root *fs_root;
 	struct btrfs_root *csum_root;
 	struct btrfs_root *quota_root;
+	struct btrfs_root *dedup_root;
 
 	/* the log root tree is a directory of all the other log roots */
 	struct btrfs_root *log_root_tree;
@@ -1603,6 +1635,18 @@ struct btrfs_fs_info {
 	/* used by btrfs_qgroup_record_ref for an efficient tree traversal */
 	u64 qgroup_seq;
 
+	/* holds hashes of in-ram blocks */
+	struct rb_root dedup_hashes;
+
+	/* holds duplicates metadata */
+	struct btrfs_duplicate duplicates;
+
+	/* dedup block size */
+	u64 dedupsize;
+
+	/* dedup byte-by-byte block verification flag */
+	int dedupverify;
+
 	/* qgroup rescan items */
 	struct mutex qgroup_rescan_lock; /* protects the progress item */
 	struct btrfs_key qgroup_rescan_progress;
@@ -1811,6 +1855,17 @@ struct btrfs_ioctl_defrag_range_args {
 #define BTRFS_ROOT_ITEM_KEY	132
 
 /*
+ * extent hashes are stored in a separate tree and hold hashes of
+ * dedup-sized block for an entire extent on disk.
+ */
+#define BTRFS_DEDUP_ITEM_KEY	136
+
+/*
+ * dedup hash key maps on-disk location of a block to its hash
+ */
+#define BTRFS_DEDUP_HASH_KEY	140
+
+/*
  * root backrefs tie subvols and snapshots to the directory entries that
  * reference them
  */
@@ -1927,6 +1982,7 @@ struct btrfs_ioctl_defrag_range_args {
 #define BTRFS_MOUNT_CHECK_INTEGRITY	(1 << 20)
 #define BTRFS_MOUNT_CHECK_INTEGRITY_INCLUDING_EXTENT_DATA (1 << 21)
 #define BTRFS_MOUNT_PANIC_ON_FATAL_ERROR	(1 << 22)
+#define BTRFS_MOUNT_DEDUP		(1 << 23)
 
 #define btrfs_clear_opt(o, opt)		((o) &= ~BTRFS_MOUNT_##opt)
 #define btrfs_set_opt(o, opt)		((o) |= BTRFS_MOUNT_##opt)
@@ -2896,6 +2952,22 @@ BTRFS_SETGET_FUNCS(qgroup_status_flags, struct btrfs_qgroup_status_item,
 BTRFS_SETGET_FUNCS(qgroup_status_rescan, struct btrfs_qgroup_status_item,
 		   rescan, 64);
 
+/* btrfs_dedup_item */
+BTRFS_SETGET_FUNCS(dedup_disk_bytenr, struct btrfs_dedup_item, disk_bytenr, 64);
+BTRFS_SETGET_FUNCS(dedup_disk_num_bytes, struct btrfs_dedup_item, disk_num_bytes, 64);
+BTRFS_SETGET_FUNCS(dedup_extent_offset, struct btrfs_dedup_item, extent_offset, 64);
+BTRFS_SETGET_FUNCS(dedup_len, struct btrfs_dedup_item, len, 64);
+BTRFS_SETGET_FUNCS(dedup_hash0, struct btrfs_dedup_item, hash0, 64);
+BTRFS_SETGET_FUNCS(dedup_hash1, struct btrfs_dedup_item, hash1, 64);
+
+
+/* btrfs_dedup_hash_item */
+BTRFS_SETGET_FUNCS(dedup_hash_hash0, struct btrfs_dedup_hash_item, hash0, 64);
+BTRFS_SETGET_FUNCS(dedup_hash_hash1, struct btrfs_dedup_hash_item, hash1, 64);
+BTRFS_SETGET_FUNCS(dedup_hash_hash2, struct btrfs_dedup_hash_item, hash2, 64);
+BTRFS_SETGET_FUNCS(dedup_hash_hash3, struct btrfs_dedup_hash_item, hash3, 64);
+
+
 /* btrfs_qgroup_info_item */
 BTRFS_SETGET_FUNCS(qgroup_info_generation, struct btrfs_qgroup_info_item,
 		   generation, 64);
@@ -3321,6 +3393,7 @@ static inline void free_fs_info(struct btrfs_fs_info *fs_info)
 	kfree(fs_info->dev_root);
 	kfree(fs_info->csum_root);
 	kfree(fs_info->quota_root);
+	kfree(fs_info->dedup_root);
 	kfree(fs_info->super_copy);
 	kfree(fs_info->super_for_commit);
 	kfree(fs_info);
diff --git a/fs/btrfs/dedup.c b/fs/btrfs/dedup.c
new file mode 100644
index 0000000..c0b8cd1
--- /dev/null
+++ b/fs/btrfs/dedup.c
@@ -0,0 +1,544 @@
+/*
+ * Copyright (C) 2013.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License v2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 021110-1307, USA.
+ */
+
+#include <linux/rbtree.h>
+#include <crypto/hash.h>
+#include "ctree.h"
+#include "disk-io.h"
+#include "dedup.h"
+#include "btrfs_inode.h"
+
+int btrfs_dedup_hash_data(char *data, u64 *hash, u32 blocksize)
+{
+	struct crypto_shash *dedup_shash_tfm;
+	dedup_shash_tfm = crypto_alloc_shash("sha256", 0, 0);
+	BUG_ON(IS_ERR(dedup_shash_tfm));
+	struct {
+		struct shash_desc shash;
+		char ctx[crypto_shash_descsize(dedup_shash_tfm)];
+	} desc;
+
+	desc.shash.tfm = dedup_shash_tfm;
+	desc.shash.flags = 0;
+
+	crypto_shash_init(&desc.shash);
+	crypto_shash_update(&desc.shash, data, blocksize);
+	crypto_shash_final(&desc.shash, (char *) hash);
+	crypto_free_shash(dedup_shash_tfm);
+
+	return 0;
+}
+
+
+static int get_data(struct inode *inode, u64 start, u64 end,
+	     char *buffer, u32 blocksize)
+{
+	struct page *page;
+	char *data;
+	u64 start_new = start;
+	u32 pos = 0;
+	int ret = 0;
+
+	BUG_ON(!buffer);
+	if (start + blocksize > end + 1)
+		return 1;
+
+	while (pos < blocksize) {
+		page = find_or_create_page(inode->i_mapping,
+			start_new >> PAGE_CACHE_SHIFT, GFP_NOFS);
+		if (!page) {
+			ret = -EINVAL;
+			break;
+		}
+		if (!PageUptodate(page)) {
+			btrfs_readpage(NULL, page);
+			lock_page(page);
+			if (!PageUptodate(page)) {
+				unlock_page(page);
+				page_cache_release(page);
+				ret = -EINVAL;
+				break;
+			}
+		}
+
+		data = kmap(page);
+		memcpy(buffer+pos, data, PAGE_CACHE_SIZE);
+		kunmap(page);
+		unlock_page(page);
+		page_cache_release(page);
+		start_new += PAGE_CACHE_SIZE;
+		pos += PAGE_CACHE_SIZE;
+	}
+
+	return 0;
+}
+static int hash_data(struct btrfs_trans_handle *trans,
+		     struct btrfs_root *root,
+		     struct inode *inode)
+{
+	int ret = 0;
+	pgoff_t index = 0;
+	pgoff_t last_index = (inode->i_size - 1) >> PAGE_CACHE_SHIFT;
+	u64 bytes_read = 0;
+	struct btrfs_file_extent_item *fi;
+	u64 disk_bytenr = 0;
+	u64 num_bytes = 0;
+	u64 extent_offset = 0;
+	u64 *hash;
+	int extent_type;
+	u64 ino = btrfs_ino(inode);
+	struct btrfs_path *path;
+	struct extent_buffer *leaf;
+	u32 blocksize = root->fs_info->dedupsize;
+	char *data;
+	u64 tmp;
+
+	while (index <= last_index) {
+		if (num_bytes - bytes_read < blocksize) {
+			bytes_read = num_bytes;
+			index += (num_bytes - bytes_read) / PAGE_CACHE_SIZE;
+		}
+
+		if (bytes_read == num_bytes) {
+			path = btrfs_alloc_path();
+			if (!path)
+				return -ENOMEM;
+
+			ret = btrfs_lookup_file_extent(trans, root, path, ino,
+							index*PAGE_CACHE_SIZE, 0);
+			if (ret) {
+				btrfs_free_path(path);
+				break;
+			}
+			leaf = path->nodes[0];
+			fi = btrfs_item_ptr(leaf, path->slots[0],
+						struct btrfs_file_extent_item);
+			extent_type = btrfs_file_extent_type(leaf, fi);
+			if (extent_type == BTRFS_FILE_EXTENT_REG ||
+			    extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
+				disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
+				num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
+				extent_offset = btrfs_file_extent_offset(leaf, fi);
+			}
+			else {
+				btrfs_free_path(path);
+				break;
+			}
+			bytes_read = 0;
+			btrfs_free_path(path);
+		}
+
+		data = (char *) kzalloc(blocksize, GFP_NOFS);
+		BUG_ON(!data);
+		if (get_data(inode, index * PAGE_CACHE_SIZE,
+			     index * PAGE_CACHE_SIZE + blocksize, data,
+			     blocksize))
+			goto out;
+
+		hash = kzalloc(sizeof(u64) * 4, GFP_NOFS);
+		BUG_ON(!hash);
+		btrfs_dedup_hash_data(data, hash, blocksize);
+
+		if (!(ret = btrfs_lookup_dedup_item(root, hash, &tmp, &tmp, &tmp)))
+			goto next;
+
+		ret = btrfs_insert_dedup_item(trans,
+					      root,
+					      hash,
+					      disk_bytenr,
+					      num_bytes,
+					      extent_offset+bytes_read);
+		BUG_ON(ret);
+		ret = btrfs_insert_dedup_hash_item(trans,
+						   root,
+						   hash, disk_bytenr,
+						   extent_offset+bytes_read);
+		BUG_ON(ret);
+next:
+		kfree(hash);
+		kfree(data);
+		index += blocksize / PAGE_CACHE_SIZE;
+		bytes_read += blocksize;
+	}
+
+out:
+	return ret;
+}
+
+int btrfs_insert_dedup_items(struct btrfs_trans_handle *trans,
+			     struct btrfs_root *root)
+{
+	int ret;
+	struct btrfs_key key;
+	struct btrfs_key found_key;
+	struct btrfs_path *path;
+	struct inode *inode = NULL;
+	struct extent_buffer *leaf;
+
+	path = btrfs_alloc_path();
+	if (!path)
+		return -ENOMEM;
+
+	key.objectid = (u64)-1;
+	key.offset = 0;
+	btrfs_set_key_type(&key, BTRFS_INODE_ITEM_KEY);
+
+	ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
+	if (ret < 0)
+		goto err;
+
+	BUG_ON(ret == 0);
+	while (1) {
+		ret = btrfs_previous_item(root, path, 0,
+					  BTRFS_INODE_ITEM_KEY);
+		if (ret < 0)
+			goto err;
+		if (ret)
+			break;
+
+		leaf = path->nodes[0];
+		btrfs_item_key_to_cpu(path->nodes[0], &found_key,
+				      path->slots[0]);
+		inode = btrfs_iget(root->fs_info->sb, &found_key,
+				   root, NULL);
+
+		if (IS_ERR(inode)) {
+			ret = PTR_ERR(inode);
+			goto err;
+		}
+
+		if (!S_ISREG(inode->i_mode) ||
+		    BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW ||
+		    BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM ||
+		    BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS ||
+		    BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC ||
+		    BTRFS_I(inode)->force_compress)
+		{
+			iput(inode);
+			continue;
+		}
+
+		ret = hash_data(trans, root, inode);
+		iput(inode);
+
+		if (ret < 0)
+			goto err;
+	}
+
+	ret = 0;
+err:
+	btrfs_free_path(path);
+	return ret;
+}
+
+int btrfs_insert_dedup_item(struct btrfs_trans_handle *trans,
+			    struct btrfs_root *root,
+			    u64 *hash, u64 disk_bytenr, u64 disk_num_bytes,
+			    u64 extent_offset)
+{
+	int ret = 0;
+	struct btrfs_root *dedup_root = root->fs_info->dedup_root;
+	struct btrfs_dedup_item *item;
+	struct btrfs_key key;
+	struct btrfs_path *path;
+	struct extent_buffer *leaf;
+
+	path = btrfs_alloc_path();
+	if (!path)
+		return -ENOMEM;
+
+	key.objectid = hash[2];
+	key.offset = hash[3];
+	btrfs_set_key_type(&key, BTRFS_DEDUP_ITEM_KEY);
+
+	ret = btrfs_insert_empty_item(trans, dedup_root, path, &key,
+				      sizeof(*item));
+	if (ret == -EEXIST) {
+		ret = 0;
+		goto out;
+	}
+	if (ret < 0)
+		goto out;
+	BUG_ON(ret);
+	leaf = path->nodes[0];
+	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dedup_item);
+
+	btrfs_set_dedup_disk_bytenr(leaf, item, disk_bytenr);
+	btrfs_set_dedup_disk_num_bytes(leaf, item, disk_num_bytes);
+	btrfs_set_dedup_extent_offset(leaf, item, extent_offset);
+	btrfs_set_dedup_len(leaf, item, root->fs_info->dedupsize);
+	btrfs_set_dedup_hash0(leaf, item, hash[0]);
+	btrfs_set_dedup_hash1(leaf, item, hash[1]);
+
+	btrfs_mark_buffer_dirty(leaf);
+out:
+	btrfs_free_path(path);
+	return ret;
+}
+
+int btrfs_lookup_dedup_item(struct btrfs_root *root,
+			    u64 *hash, u64 *disk_bytenr,
+			    u64 *disk_num_bytes, u64 *extent_offset)
+{
+	int ret;
+	struct btrfs_root *dedup_root = root->fs_info->dedup_root;
+	struct btrfs_dedup_item *item;
+	struct btrfs_key key;
+	struct btrfs_path *path;
+	struct extent_buffer *leaf;
+
+	path = btrfs_alloc_path();
+	if (!path) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	key.objectid = hash[2];
+	key.offset = hash[3];
+	btrfs_set_key_type(&key, BTRFS_DEDUP_ITEM_KEY);
+	ret = btrfs_search_slot(NULL, dedup_root, &key, path, 0, 0);
+	if (ret) {
+		ret = 1;
+		goto out;
+	}
+	leaf = path->nodes[0];
+	item = btrfs_item_ptr(leaf, path->slots[0],
+			       struct btrfs_dedup_item);
+	if (hash[0] != btrfs_dedup_hash0(leaf, item) ||
+	    hash[1] != btrfs_dedup_hash1(leaf, item) ||
+	    btrfs_dedup_len(leaf, item) != root->fs_info->dedupsize) {
+		ret = 1;
+		goto out;
+	}
+
+	ret = 0;
+	*disk_bytenr = btrfs_dedup_disk_bytenr(leaf, item);
+	*disk_num_bytes = btrfs_dedup_disk_num_bytes(leaf, item);
+	*extent_offset = btrfs_dedup_extent_offset(leaf, item);
+out:
+	btrfs_free_path(path);
+	return ret;
+}
+
+static int btrfs_del_dedup_item(struct btrfs_trans_handle *trans,
+			 struct btrfs_root *root,
+			 u64 *hash)
+{
+	int ret;
+	struct btrfs_root *dedup_root = root->fs_info->dedup_root;
+	struct btrfs_dedup_item *item;
+	struct btrfs_key key;
+	struct btrfs_path *path;
+	struct extent_buffer *leaf;
+
+	path = btrfs_alloc_path();
+	if (!path) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	key.objectid = hash[2];
+	key.offset = hash[3];
+	btrfs_set_key_type(&key, BTRFS_DEDUP_ITEM_KEY);
+	ret = btrfs_search_slot(trans, dedup_root, &key, path, -1, 1);
+	if (ret)
+		goto out;
+	leaf = path->nodes[0];
+	item = btrfs_item_ptr(leaf, path->slots[0],
+			       struct btrfs_dedup_item);
+	if (hash[0] != btrfs_dedup_hash0(leaf, item) ||
+	    hash[1] != btrfs_dedup_hash1(leaf, item) ||
+	    btrfs_dedup_len(leaf, item) != root->fs_info->dedupsize) {
+		ret = 1;
+		goto out;
+	}
+
+	ret = btrfs_del_item(trans, dedup_root, path);
+out:
+	btrfs_free_path(path);
+
+	return ret;
+}
+
+int btrfs_insert_dedup_hash_item(struct btrfs_trans_handle *trans,
+				 struct btrfs_root *root,
+				 u64 *hash, u64 disk_bytenr,
+				 u64 extent_offset)
+{
+	int ret = 0;
+	struct btrfs_root *dedup_root = root->fs_info->dedup_root;
+	struct btrfs_dedup_hash_item *item;
+	struct btrfs_key key;
+	struct btrfs_path *path;
+	struct extent_buffer *leaf;
+
+	path = btrfs_alloc_path();
+	if (!path)
+		return -ENOMEM;
+
+	key.objectid = disk_bytenr;
+	key.offset = extent_offset;
+	btrfs_set_key_type(&key, BTRFS_DEDUP_HASH_KEY);
+
+	ret = btrfs_insert_empty_item(trans, dedup_root, path, &key,
+				      sizeof(*item));
+	if (ret == -EEXIST) {
+		ret = 0;
+		goto out;
+	}
+	if (ret < 0)
+		goto out;
+	BUG_ON(ret);
+	leaf = path->nodes[0];
+	item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dedup_hash_item);
+
+	btrfs_set_dedup_hash_hash0(leaf, item, hash[0]);
+	btrfs_set_dedup_hash_hash1(leaf, item, hash[1]);
+	btrfs_set_dedup_hash_hash2(leaf, item, hash[2]);
+	btrfs_set_dedup_hash_hash3(leaf, item, hash[3]);
+
+	btrfs_mark_buffer_dirty(leaf);
+out:
+	btrfs_free_path(path);
+	return ret;
+}
+
+static int btrfs_del_dedup_hash_item(struct btrfs_trans_handle *trans,
+			  struct btrfs_root *root,
+			  u64 disk_bytenr, u64 extent_offset)
+{
+	int ret;
+	struct btrfs_root *dedup_root = root->fs_info->dedup_root;
+	struct btrfs_dedup_hash_item *item;
+	struct btrfs_key key;
+	struct btrfs_path *path;
+	struct extent_buffer *leaf;
+	u64 hash[4];
+
+	path = btrfs_alloc_path();
+	if (!path) {
+		ret = -ENOMEM;
+		btrfs_free_path(path);
+		return ret;
+	}
+
+	key.objectid = disk_bytenr;
+	key.offset = extent_offset;
+	btrfs_set_key_type(&key, BTRFS_DEDUP_HASH_KEY);
+	ret = btrfs_search_slot(trans, dedup_root, &key, path, -1, 1);
+	if (ret) {
+		btrfs_free_path(path);
+		return ret;
+	}
+	leaf = path->nodes[0];
+	item = btrfs_item_ptr(leaf, path->slots[0],
+			       struct btrfs_dedup_hash_item);
+	hash[0] = btrfs_dedup_hash_hash0(leaf, item);
+	hash[1] = btrfs_dedup_hash_hash1(leaf, item);
+	hash[2] = btrfs_dedup_hash_hash2(leaf, item);
+	hash[3] = btrfs_dedup_hash_hash3(leaf, item);
+
+	ret = btrfs_del_item(trans, dedup_root, path);
+	btrfs_free_path(path);
+	btrfs_del_dedup_item(trans, dedup_root, hash);
+	return ret;
+}
+
+int btrfs_free_dedup_items(struct btrfs_trans_handle *trans,
+			   struct btrfs_root *root,
+			   u64 disk_bytenr, u64 num_bytes)
+{
+	int i = 0;
+
+	for (i = 0; i != num_bytes; i += root->fs_info->dedupsize)
+		btrfs_del_dedup_hash_item(trans, root, disk_bytenr, i);
+
+	return 0;
+}
+
+struct btrfs_rb_dedup_hash *btrfs_rb_search_dedup_hash(
+						struct btrfs_fs_info *fs_info,
+						u64 *hash)
+{
+	struct rb_node *n = fs_info->dedup_hashes.rb_node;
+	struct btrfs_rb_dedup_hash *dedup_hash;
+
+	while (n) {
+		dedup_hash = rb_entry(n, struct btrfs_rb_dedup_hash, node);
+		if (memcmp(dedup_hash->hash, hash, sizeof(u64)*4) < 0)
+			n = n->rb_left;
+		else if (memcmp(dedup_hash->hash, hash, sizeof(u64)*4) > 0)
+			n = n->rb_right;
+		else
+			return dedup_hash;
+	}
+	return NULL;
+}
+
+struct btrfs_rb_dedup_hash *btrfs_rb_insert_dedup_hash(
+						struct btrfs_fs_info *fs_info,
+						u64 *hash, struct inode *inode,
+						u64 file_offset)
+{
+	struct rb_node **p = &fs_info->dedup_hashes.rb_node;
+	struct rb_node *parent = NULL;
+	struct btrfs_rb_dedup_hash *dedup_hash;
+
+	while (*p) {
+		parent = *p;
+		dedup_hash = rb_entry(parent, struct btrfs_rb_dedup_hash, node);
+		if (memcmp(dedup_hash->hash, hash, sizeof(u64)*4) < 0)
+			p = &(*p)->rb_left;
+		else if (memcmp(dedup_hash->hash, hash, sizeof(u64)*4) > 0)
+			p = &(*p)->rb_right;
+		else
+			return dedup_hash;
+	}
+
+	dedup_hash = kzalloc(sizeof(*dedup_hash), GFP_ATOMIC);
+	if (!dedup_hash)
+		return ERR_PTR(-ENOMEM);
+
+	dedup_hash->hash = kzalloc(sizeof(u64)*4, GFP_ATOMIC);
+	if (!dedup_hash->hash)
+		return ERR_PTR(-ENOMEM);
+
+	dedup_hash->inode = inode;
+	memcpy(dedup_hash->hash, hash, sizeof(u64)*4);
+	dedup_hash->file_offset = file_offset;
+
+	rb_link_node(&dedup_hash->node, parent, p);
+	rb_insert_color(&dedup_hash->node, &fs_info->dedup_hashes);
+
+	return dedup_hash;
+}
+
+int btrfs_rb_erase_dedup_hash(struct btrfs_fs_info *fs_info, u64 *hash)
+{
+	struct btrfs_rb_dedup_hash *dedup_hash =
+		btrfs_rb_search_dedup_hash(fs_info, hash);
+
+	if (!dedup_hash)
+		return -ENOENT;
+
+	rb_erase(&dedup_hash->node, &fs_info->dedup_hashes);
+	kfree(dedup_hash->hash);
+	kfree(dedup_hash);
+
+	return 0;
+}
diff --git a/fs/btrfs/dedup.h b/fs/btrfs/dedup.h
new file mode 100644
index 0000000..48542e8
--- /dev/null
+++ b/fs/btrfs/dedup.h
@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2013.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License v2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this program; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 021110-1307, USA.
+ */
+
+#ifndef __BTRFS_DEDUP_
+#define __BTRFS_DEDUP_
+
+/* holds metadata for in-ram duplicates in rb tree */
+struct btrfs_rb_dedup_hash {
+	struct inode *inode;
+	u64 *hash;
+	u64 file_offset;
+	struct rb_node node;
+};
+
+int btrfs_dedup_hash_data(char *data, u64 *hash, u32 blocksize);
+
+int btrfs_insert_dedup_items(struct btrfs_trans_handle *trans,
+			     struct btrfs_root *root);
+
+int btrfs_insert_dedup_item(struct btrfs_trans_handle *trans,
+			    struct btrfs_root *root,
+			    u64 *hash, u64 disk_bytenr, u64 disk_num_bytes,
+			    u64 extent_offset);
+
+int btrfs_lookup_dedup_item(struct btrfs_root *root,
+			    u64 *hash, u64 *disk_bytenr,
+			    u64 *disk_num_bytes, u64 *extent_offset);
+
+int btrfs_insert_dedup_hash_item(struct btrfs_trans_handle *trans,
+				 struct btrfs_root *root,
+				 u64 *hash, u64 disk_bytenr,
+				 u64 extent_offset);
+
+int btrfs_free_dedup_items(struct btrfs_trans_handle *trans,
+			   struct btrfs_root *root,
+			   u64 disk_bytenr, u64 num_bytes);
+
+struct btrfs_rb_dedup_hash *btrfs_rb_search_dedup_hash(
+						struct btrfs_fs_info *fs_info,
+						u64 *hash);
+
+struct btrfs_rb_dedup_hash *btrfs_rb_insert_dedup_hash(
+						struct btrfs_fs_info *fs_info,
+						u64 *hash, struct inode *inode,
+						u64 file_offset);
+
+int btrfs_rb_erase_dedup_hash(struct btrfs_fs_info *fs_info, u64 *hash);
+
+#endif
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index e7b3cb5..fb7989d 100644
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -1538,6 +1538,8 @@ struct btrfs_root *btrfs_read_fs_root_no_name(struct btrfs_fs_info *fs_info,
 		return fs_info->dev_root;
 	if (location->objectid == BTRFS_CSUM_TREE_OBJECTID)
 		return fs_info->csum_root;
+	if (location->objectid == BTRFS_DEDUP_TREE_OBJECTID)
+		return fs_info->dedup_root;
 	if (location->objectid == BTRFS_QUOTA_TREE_OBJECTID)
 		return fs_info->quota_root ? fs_info->quota_root :
 					     ERR_PTR(-ENOENT);
@@ -2008,6 +2010,12 @@ static void free_root_pointers(struct btrfs_fs_info *info, int chunk_root)
 		info->csum_root->node = NULL;
 		info->csum_root->commit_root = NULL;
 	}
+	if (info->dedup_root) {
+		free_extent_buffer(info->dedup_root->node);
+		free_extent_buffer(info->dedup_root->commit_root);
+		info->dedup_root->node = NULL;
+		info->dedup_root->commit_root = NULL;
+	}
 	if (info->quota_root) {
 		free_extent_buffer(info->quota_root->node);
 		free_extent_buffer(info->quota_root->commit_root);
@@ -2071,6 +2079,7 @@ int open_ctree(struct super_block *sb,
 	struct btrfs_root *tree_root;
 	struct btrfs_root *extent_root;
 	struct btrfs_root *csum_root;
+	struct btrfs_root *dedup_root;
 	struct btrfs_root *chunk_root;
 	struct btrfs_root *dev_root;
 	struct btrfs_root *quota_root;
@@ -2083,12 +2092,14 @@ int open_ctree(struct super_block *sb,
 	tree_root = fs_info->tree_root = btrfs_alloc_root(fs_info);
 	extent_root = fs_info->extent_root = btrfs_alloc_root(fs_info);
 	csum_root = fs_info->csum_root = btrfs_alloc_root(fs_info);
+	dedup_root = fs_info->dedup_root = btrfs_alloc_root(fs_info);
 	chunk_root = fs_info->chunk_root = btrfs_alloc_root(fs_info);
 	dev_root = fs_info->dev_root = btrfs_alloc_root(fs_info);
 	quota_root = fs_info->quota_root = btrfs_alloc_root(fs_info);
 
 	if (!tree_root || !extent_root || !csum_root ||
-	    !chunk_root || !dev_root || !quota_root) {
+	    !dedup_root || !chunk_root || !dev_root ||
+	    !quota_root) {
 		err = -ENOMEM;
 		goto fail;
 	}
@@ -2276,6 +2287,11 @@ int open_ctree(struct super_block *sb,
 	fs_info->pending_quota_state = 0;
 	mutex_init(&fs_info->qgroup_rescan_lock);
 
+	fs_info->dedup_hashes = RB_ROOT;
+	INIT_LIST_HEAD(&fs_info->duplicates.list);
+	fs_info->dedupsize = 8192;
+	fs_info->dedupverify = 0;
+
 	btrfs_init_free_cluster(&fs_info->meta_alloc_cluster);
 	btrfs_init_free_cluster(&fs_info->data_alloc_cluster);
 
@@ -2657,6 +2673,13 @@ retry_root_backup:
 	csum_root->track_dirty = 1;
 
 	ret = find_and_setup_root(tree_root, fs_info,
+				  BTRFS_DEDUP_TREE_OBJECTID, dedup_root);
+	if (ret)
+		goto recovery_tree_root;
+	dedup_root->track_dirty = 1;
+
+
+	ret = find_and_setup_root(tree_root, fs_info,
 				  BTRFS_QUOTA_TREE_OBJECTID, quota_root);
 	if (ret) {
 		kfree(quota_root);
diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c
index df472ab..4fe783e 100644
--- a/fs/btrfs/extent-tree.c
+++ b/fs/btrfs/extent-tree.c
@@ -35,6 +35,7 @@
 #include "locking.h"
 #include "free-space-cache.h"
 #include "math.h"
+#include "dedup.h"
 
 #undef SCRAMBLE_DELAYED_REFS
 
@@ -5778,6 +5779,7 @@ int btrfs_free_extent(struct btrfs_trans_handle *trans, struct btrfs_root *root,
 					parent, root_objectid, (int)owner,
 					BTRFS_DROP_DELAYED_REF, NULL, for_cow);
 	} else {
+		btrfs_free_dedup_items(trans, root, bytenr, num_bytes);
 		ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
 						num_bytes,
 						parent, root_objectid, owner,
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index 23c596c..0e1241b 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -56,6 +56,7 @@
 #include "free-space-cache.h"
 #include "inode-map.h"
 #include "backref.h"
+#include "dedup.h"
 
 struct btrfs_iget_args {
 	u64 ino;
@@ -96,7 +97,8 @@ static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent);
 static noinline int cow_file_range(struct inode *inode,
 				   struct page *locked_page,
 				   u64 start, u64 end, int *page_started,
-				   unsigned long *nr_written, int unlock);
+				   unsigned long *nr_written, int unlock,
+				   struct btrfs_dedup_metadata *dedup_metadata);
 static struct extent_map *create_pinned_em(struct inode *inode, u64 start,
 					   u64 len, u64 orig_start,
 					   u64 block_start, u64 block_len,
@@ -648,7 +650,7 @@ retry:
 					     async_extent->start,
 					     async_extent->start +
 					     async_extent->ram_size - 1,
-					     &page_started, &nr_written, 0);
+					     &page_started, &nr_written, 0, NULL);
 
 			/* JDM XXX */
 
@@ -855,8 +857,8 @@ static noinline int __cow_file_range(struct btrfs_trans_handle *trans,
 				     struct btrfs_root *root,
 				     struct page *locked_page,
 				     u64 start, u64 end, int *page_started,
-				     unsigned long *nr_written,
-				     int unlock)
+				     unsigned long *nr_written, int unlock,
+				     struct btrfs_dedup_metadata *dedup_metadata)
 {
 	u64 alloc_hint = 0;
 	u64 num_bytes;
@@ -958,8 +960,13 @@ static noinline int __cow_file_range(struct btrfs_trans_handle *trans,
 			goto out_reserve;
 
 		cur_alloc_size = ins.offset;
-		ret = btrfs_add_ordered_extent(inode, start, ins.objectid,
-					       ram_size, cur_alloc_size, 0);
+		if (dedup_metadata)
+			ret = btrfs_add_ordered_extent_dedup(inode, start, ins.objectid,
+						       ram_size, cur_alloc_size, 0,
+						       dedup_metadata);
+		else
+			ret = btrfs_add_ordered_extent(inode, start, ins.objectid,
+						       ram_size, cur_alloc_size, 0);
 		if (ret)
 			goto out_reserve;
 
@@ -1017,8 +1024,8 @@ out_unlock:
 static noinline int cow_file_range(struct inode *inode,
 				   struct page *locked_page,
 				   u64 start, u64 end, int *page_started,
-				   unsigned long *nr_written,
-				   int unlock)
+				   unsigned long *nr_written, int unlock,
+				   struct btrfs_dedup_metadata *dedup_metadata)
 {
 	struct btrfs_trans_handle *trans;
 	struct btrfs_root *root = BTRFS_I(inode)->root;
@@ -1040,7 +1047,7 @@ static noinline int cow_file_range(struct inode *inode,
 	trans->block_rsv = &root->fs_info->delalloc_block_rsv;
 
 	ret = __cow_file_range(trans, inode, root, locked_page, start, end,
-			       page_started, nr_written, unlock);
+			       page_started, nr_written, unlock, dedup_metadata);
 
 	btrfs_end_transaction(trans, root);
 
@@ -1364,7 +1371,7 @@ out_check:
 		if (cow_start != (u64)-1) {
 			ret = __cow_file_range(trans, inode, root, locked_page,
 					       cow_start, found_key.offset - 1,
-					       page_started, nr_written, 1);
+					       page_started, nr_written, 1, NULL);
 			if (ret) {
 				btrfs_abort_transaction(trans, root, ret);
 				goto error;
@@ -1440,7 +1447,7 @@ out_check:
 	if (cow_start != (u64)-1) {
 		ret = __cow_file_range(trans, inode, root, locked_page,
 				       cow_start, end,
-				       page_started, nr_written, 1);
+				       page_started, nr_written, 1, NULL);
 		if (ret) {
 			btrfs_abort_transaction(trans, root, ret);
 			goto error;
@@ -1467,6 +1474,300 @@ error:
 	return ret;
 }
 
+static int link_duplicates(struct inode *inode, struct btrfs_root *root,
+			struct btrfs_trans_handle *trans)
+{
+	int ret = 0;
+	u32 dedup_block = root->fs_info->dedupsize;
+	struct btrfs_duplicate *duplicate, *next;
+
+	list_for_each_entry_safe(duplicate, next,
+				&root->fs_info->duplicates.list, list)
+	{
+		struct inode *inode = duplicate->inode;
+		if (!duplicate->disk_bytenr) {
+			u64 disk_bytenr;
+			u64 disk_num_bytes;
+			u64 extent_offset;
+			ret = btrfs_lookup_dedup_item(root,
+					duplicate->inram, &disk_bytenr,
+					&disk_num_bytes, &extent_offset);
+			BUG_ON(ret);
+
+			duplicate->disk_bytenr = disk_bytenr;
+			duplicate->num_bytes = disk_num_bytes;
+			duplicate->extent_offset = extent_offset;
+		}
+
+		ret = btrfs_inc_extent_ref(trans, root, duplicate->disk_bytenr,
+				duplicate->num_bytes, 0,
+				root->objectid,
+				inode->i_ino,
+				duplicate->extent_offset,
+				0);
+		ret = btrfs_drop_extents(trans, root, inode,
+				duplicate->file_offset,
+				duplicate->file_offset + dedup_block,
+				0);
+		ret = btrfs_insert_file_extent(trans, root,
+				inode->i_ino, duplicate->file_offset,
+				duplicate->disk_bytenr, dedup_block,
+				dedup_block, duplicate->extent_offset,
+				dedup_block, 0, 0, 0);
+		inode_add_bytes(inode, dedup_block);
+		btrfs_i_size_write(inode, inode->i_size);
+		btrfs_update_inode(trans, root, inode);
+		list_del(&duplicate->list);
+		kfree(duplicate);
+	}
+	return ret;
+}
+
+static void dedup_complete_bio_end_io(struct bio *bio, int err)
+{
+	complete((struct completion *)bio->bi_private);
+}
+
+int get_data_bytenr(struct inode *inode, u64 disk_bytenr, char *data, u32 blocksize)
+{
+	struct bio *bio;
+	struct page *page;
+	char *buffer;
+	u32 pos = 0;
+	DECLARE_COMPLETION_ONSTACK(complete);
+
+	while (pos < blocksize) {
+		page = alloc_page(GFP_NOFS);
+		BUG_ON(!page);
+
+		bio = bio_alloc(GFP_NOFS, 1);
+		BUG_ON(!bio);
+
+		bio->bi_bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
+		bio->bi_sector = (disk_bytenr+pos) >> 9;
+		bio->bi_end_io = dedup_complete_bio_end_io;
+		bio->bi_private = &complete;
+
+		if (!bio_add_page(bio, page, PAGE_CACHE_SIZE, 0)) {
+			bio_put(bio);
+			return -EIO;
+		}
+		submit_bio(READ, bio);
+
+		wait_for_completion(&complete);
+
+		if (unlikely(!test_bit(BIO_UPTODATE, &bio->bi_flags))) {
+			bio_put(bio);
+			__free_page(page);
+			return -EIO;
+		} else {
+			buffer = kmap(page);
+			memcpy(data + pos, buffer, PAGE_CACHE_SIZE);
+			kunmap(page);
+			pos += PAGE_CACHE_SIZE;
+		}
+		__free_page(page);
+		bio_put(bio);
+	}
+
+	return 0;
+}
+
+int get_data(struct inode *inode, u64 start, u64 end,
+	     char *buffer, u32 blocksize)
+{
+	struct page *page;
+	char *data;
+	u64 start_new = start;
+	u32 pos = 0;
+
+	BUG_ON(!buffer);
+	if (start + blocksize > end + 1)
+		return 1;
+
+	while (pos < blocksize) {
+		page = find_get_page(inode->i_mapping,
+			start_new >> PAGE_CACHE_SHIFT);
+		data = kmap(page);
+		memcpy(buffer+pos, data, PAGE_CACHE_SIZE);
+		kunmap(page);
+		page_cache_release(page);
+		start_new += PAGE_CACHE_SIZE;
+		pos += PAGE_CACHE_SIZE;
+	}
+
+	return 0;
+}
+
+static noinline int cow_file_range_dedup(struct inode *inode,
+					struct page *locked_page,
+					u64 start, u64 end, int *page_started,
+					unsigned long *nr_written,
+					int unlock)
+{
+	struct btrfs_root *root = BTRFS_I(inode)->root;
+	struct btrfs_dedup_metadata *dedup_metadata = NULL;
+	struct btrfs_duplicate *duplicate;
+	struct btrfs_dedup_hash *hash_entry;
+	struct btrfs_rb_dedup_hash *rb_dedup_hash;
+	int ret = 0;
+	char *data;
+	char *dup_data = NULL;
+	unsigned long written;
+	u32 blocksize = root->fs_info->dedupsize;
+	u64 bytes_left = end - start + 1;
+	u64 start_write = start;
+	u64 start_new = start;
+	u64 len_write = 0;
+	u64 *hash;
+	u64 disk_bytenr;
+	u64 disk_num_bytes;
+	u64 extent_offset;
+	int disk_dups = 1; // 0 if in-ram duplicates only
+	bool dup = false;
+
+/* - *BUG* cow_file_range could could split the extent!!! */
+
+	/*
+	 * go through the extent range and find duplicate blocks,
+	 * contigous ranges of unique blocks are written to disk,
+	 * duplicate blocks are thrown away and linked later in
+	 * btrfs_finish_ordered_io (or here in case the whole
+	 * range consists of duplicate blocks only)
+	 */
+	while (bytes_left > 0) {
+		dup = false;
+		data = (char *) kzalloc(blocksize, GFP_NOFS);
+		BUG_ON(!data);
+		if (get_data(inode, start_new, end, data, blocksize)) {
+			if (!len_write)
+				start_write = start_new;
+			len_write += end - start_new + 1;
+			goto out;
+		}
+		hash = kzalloc(sizeof(u64) * 4, GFP_NOFS);
+		BUG_ON(!hash);
+		btrfs_dedup_hash_data(data, hash, blocksize);
+
+		if (root->fs_info->dedupverify)
+			dup_data = (char *) kzalloc(blocksize, GFP_NOFS);
+		if (!(ret = btrfs_lookup_dedup_item(root, hash,
+						&disk_bytenr, &disk_num_bytes,
+						&extent_offset))) {
+			if (root->fs_info->dedupverify) {
+				get_data_bytenr(inode,
+						disk_bytenr+extent_offset,
+						dup_data, blocksize);
+			} else {
+				dup = true;
+			}
+		}
+		else if ((rb_dedup_hash = btrfs_rb_search_dedup_hash(root->fs_info, hash)) != NULL) {
+			if (root->fs_info->dedupverify) {
+				get_data(rb_dedup_hash->inode,
+					rb_dedup_hash->file_offset,
+					rb_dedup_hash->file_offset+blocksize,
+					dup_data, blocksize);
+			} else {
+				dup = true;
+			}
+		}
+
+		if (root->fs_info->dedupverify) {
+			if (!memcmp(data, dup_data, blocksize))
+				dup = true;
+			kfree(dup_data);
+		}
+		kfree(data);
+
+		if (dup) {
+			duplicate = kzalloc(sizeof(struct btrfs_duplicate),
+						GFP_NOFS);
+			BUG_ON(!duplicate);
+
+			if (!ret) {
+				duplicate->disk_bytenr = disk_bytenr;
+				duplicate->num_bytes = disk_num_bytes;
+				duplicate->extent_offset = extent_offset;
+			} else {
+				disk_dups = 0;
+				duplicate->disk_bytenr = 0;
+				duplicate->inram = hash;
+			}
+
+			duplicate->inode = inode;
+			duplicate->file_offset = start_new;
+			INIT_LIST_HEAD(&duplicate->list);
+			list_add_tail(&duplicate->list,
+					&root->fs_info->duplicates.list);
+
+			extent_clear_unlock_delalloc(inode,
+				     &BTRFS_I(inode)->io_tree,
+				     start_new, start_new + blocksize - 1,
+				     locked_page,
+				     EXTENT_CLEAR_UNLOCK_PAGE |
+				     EXTENT_CLEAR_UNLOCK |
+				     EXTENT_CLEAR_DELALLOC |
+				     EXTENT_CLEAR_DIRTY |
+				     EXTENT_SET_WRITEBACK |
+				     EXTENT_END_WRITEBACK);
+
+			if (len_write) {
+				ret = cow_file_range(inode, locked_page,
+						start_write, start_write + len_write - 1,
+						page_started, &written,
+						unlock, dedup_metadata);
+				*nr_written += written;
+				len_write = 0;
+			}
+			start_write = start_new + blocksize;
+		} else {
+			disk_dups = 0;
+			if (!len_write) {
+				dedup_metadata = kzalloc(
+					sizeof(struct btrfs_dedup_metadata),
+					GFP_NOFS);
+				BUG_ON(!dedup_metadata);
+				INIT_LIST_HEAD(&dedup_metadata->hashes.list);
+			}
+
+			hash_entry = kzalloc(
+				sizeof(struct btrfs_dedup_hash),
+				GFP_NOFS);
+			BUG_ON(!hash_entry);
+			hash_entry->extent_offset = len_write;
+			hash_entry->hash = hash;
+			INIT_LIST_HEAD(&hash_entry->list);
+			list_add_tail(&hash_entry->list,
+					&dedup_metadata->hashes.list);
+			rb_dedup_hash = btrfs_rb_insert_dedup_hash(root->fs_info,
+							hash, inode, start_new);
+			BUG_ON(!rb_dedup_hash);
+			len_write += blocksize;
+		}
+
+		bytes_left -= blocksize;
+		start_new += blocksize;
+	}
+
+
+out:
+	if (!len_write && disk_dups) {
+		struct btrfs_trans_handle *trans;
+		trans = btrfs_join_transaction(root);
+		link_duplicates(inode, root, trans);
+		btrfs_end_transaction(trans, root);
+	} else if (len_write) {
+		ret = cow_file_range(inode, locked_page,
+				start_write, start_write + len_write - 1,
+				page_started, &written,
+				unlock, dedup_metadata);
+		*nr_written += written;
+	}
+
+	return ret;
+}
+
 /*
  * extent_io.c call back to do delayed allocation processing
  */
@@ -1485,9 +1786,13 @@ static int run_delalloc_range(struct inode *inode, struct page *locked_page,
 					 page_started, 0, nr_written);
 	} else if (!btrfs_test_opt(root, COMPRESS) &&
 		   !(BTRFS_I(inode)->force_compress) &&
-		   !(BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS)) {
+		   !(BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS) &&
+		   !btrfs_test_opt(root, DEDUP)) {
 		ret = cow_file_range(inode, locked_page, start, end,
-				      page_started, nr_written, 1);
+				      page_started, nr_written, 1, NULL);
+	} else if (btrfs_test_opt(root, DEDUP)) {
+		ret = cow_file_range_dedup(inode, locked_page, start, end,
+					   page_started, nr_written, 1);
 	} else {
 		set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
 			&BTRFS_I(inode)->runtime_flags);
@@ -2594,6 +2899,34 @@ out_kfree:
 	return NULL;
 }
 
+static int insert_dedup_metadata(struct inode *inode, struct btrfs_root *root,
+			struct btrfs_trans_handle *trans,
+			struct btrfs_ordered_extent *ordered_extent)
+{
+	int ret = 0;
+	struct btrfs_dedup_hash *hash_entry, *next;
+
+	list_for_each_entry_safe(hash_entry, next,
+				&ordered_extent->dedup_metadata->hashes.list,
+				list)
+	{
+		btrfs_insert_dedup_item(trans, root,
+					hash_entry->hash,
+					ordered_extent->start,
+					ordered_extent->len,
+					hash_entry->extent_offset);
+		btrfs_insert_dedup_hash_item(trans, root,
+					hash_entry->hash,
+					ordered_extent->start,
+					hash_entry->extent_offset);
+		list_del(&hash_entry->list);
+		btrfs_rb_erase_dedup_hash(root->fs_info, hash_entry->hash);
+		kfree(hash_entry->hash);
+		kfree(hash_entry);
+	}
+	return ret;
+}
+
 /*
  * helper function for btrfs_finish_ordered_io, this
  * just reads in some of the csum leaves to prime them into ram
@@ -2689,6 +3022,10 @@ static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent)
 						ordered_extent->len,
 						compress_type, 0, 0,
 						BTRFS_FILE_EXTENT_REG);
+		if (ordered_extent->dedup_metadata)
+			ret = insert_dedup_metadata(inode, root, trans, ordered_extent);
+		if (!rb_first(&root->fs_info->dedup_hashes))
+			ret = link_duplicates(inode, root, trans);
 	}
 	unpin_extent_cache(&BTRFS_I(inode)->extent_tree,
 			   ordered_extent->file_offset, ordered_extent->len,
diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
index 0f81d67..1e381b3 100644
--- a/fs/btrfs/ioctl.c
+++ b/fs/btrfs/ioctl.c
@@ -57,6 +57,8 @@
 #include "send.h"
 #include "dev-replace.h"
 
+#include "dedup.h"
+
 /* Mask out flags that are inappropriate for the given type of inode. */
 static inline __u32 btrfs_mask_flags(umode_t mode, __u32 flags)
 {
@@ -4077,6 +4079,22 @@ out_unlock:
 	return ret;
 }
 
+static long btrfs_ioctl_dedup_sync(struct file *file,
+				   void __user *arg)
+{
+	struct btrfs_root *root = BTRFS_I(fdentry(file)->d_inode)->root;
+	struct btrfs_trans_handle *trans;
+
+	trans = btrfs_join_transaction(root);
+	if (IS_ERR(trans)) {
+		return PTR_ERR(trans);
+	}
+	btrfs_insert_dedup_items(trans, root);
+	btrfs_end_transaction(trans, root);
+
+	return 0;
+}
+
 long btrfs_ioctl(struct file *file, unsigned int
 		cmd, unsigned long arg)
 {
@@ -4185,6 +4203,8 @@ long btrfs_ioctl(struct file *file, unsigned int
 		return btrfs_ioctl_get_fslabel(file, argp);
 	case BTRFS_IOC_SET_FSLABEL:
 		return btrfs_ioctl_set_fslabel(file, argp);
+	case BTRFS_IOC_DEDUP_SYNC:
+		return btrfs_ioctl_dedup_sync(file, argp);
 	}
 
 	return -ENOTTY;
diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c
index 1ddd728..d42d483 100644
--- a/fs/btrfs/ordered-data.c
+++ b/fs/btrfs/ordered-data.c
@@ -182,7 +182,8 @@ static inline struct rb_node *tree_search(struct btrfs_ordered_inode_tree *tree,
  */
 static int __btrfs_add_ordered_extent(struct inode *inode, u64 file_offset,
 				      u64 start, u64 len, u64 disk_len,
-				      int type, int dio, int compress_type)
+				      int type, int dio, int compress_type,
+				      struct btrfs_dedup_metadata *dedup_metadata)
 {
 	struct btrfs_ordered_inode_tree *tree;
 	struct rb_node *node;
@@ -203,6 +204,7 @@ static int __btrfs_add_ordered_extent(struct inode *inode, u64 file_offset,
 	entry->bytes_left = len;
 	entry->inode = igrab(inode);
 	entry->compress_type = compress_type;
+	entry->dedup_metadata = dedup_metadata;
 	if (type != BTRFS_ORDERED_IO_DONE && type != BTRFS_ORDERED_COMPLETE)
 		set_bit(type, &entry->flags);
 
@@ -240,7 +242,16 @@ int btrfs_add_ordered_extent(struct inode *inode, u64 file_offset,
 {
 	return __btrfs_add_ordered_extent(inode, file_offset, start, len,
 					  disk_len, type, 0,
-					  BTRFS_COMPRESS_NONE);
+					  BTRFS_COMPRESS_NONE, NULL);
+}
+
+int btrfs_add_ordered_extent_dedup(struct inode *inode, u64 file_offset,
+			     u64 start, u64 len, u64 disk_len, int type,
+			     struct btrfs_dedup_metadata *dedup_metadata)
+{
+	return __btrfs_add_ordered_extent(inode, file_offset, start, len,
+					  disk_len, type, 0,
+					  BTRFS_COMPRESS_NONE, dedup_metadata);
 }
 
 int btrfs_add_ordered_extent_dio(struct inode *inode, u64 file_offset,
@@ -248,7 +259,7 @@ int btrfs_add_ordered_extent_dio(struct inode *inode, u64 file_offset,
 {
 	return __btrfs_add_ordered_extent(inode, file_offset, start, len,
 					  disk_len, type, 1,
-					  BTRFS_COMPRESS_NONE);
+					  BTRFS_COMPRESS_NONE, NULL);
 }
 
 int btrfs_add_ordered_extent_compress(struct inode *inode, u64 file_offset,
@@ -257,7 +268,7 @@ int btrfs_add_ordered_extent_compress(struct inode *inode, u64 file_offset,
 {
 	return __btrfs_add_ordered_extent(inode, file_offset, start, len,
 					  disk_len, type, 0,
-					  compress_type);
+					  compress_type, NULL);
 }
 
 /*
@@ -481,6 +492,7 @@ void btrfs_put_ordered_extent(struct btrfs_ordered_extent *entry)
 {
 	struct list_head *cur;
 	struct btrfs_ordered_sum *sum;
+	struct btrfs_dedup_hash *hash;
 
 	trace_btrfs_ordered_extent_put(entry->inode, entry);
 
@@ -493,6 +505,15 @@ void btrfs_put_ordered_extent(struct btrfs_ordered_extent *entry)
 			list_del(&sum->list);
 			kfree(sum);
 		}
+		if (entry->dedup_metadata) {
+			while (!list_empty(&entry->dedup_metadata->hashes.list)) {
+				cur = entry->dedup_metadata->hashes.list.next;
+				hash = list_entry(cur, struct btrfs_dedup_hash, list);
+				list_del(&hash->list);
+				kfree(hash);
+			}
+			kfree(entry->dedup_metadata);
+		}
 		kmem_cache_free(btrfs_ordered_extent_cache, entry);
 	}
 }
diff --git a/fs/btrfs/ordered-data.h b/fs/btrfs/ordered-data.h
index 58b0e3b..a8d7784 100644
--- a/fs/btrfs/ordered-data.h
+++ b/fs/btrfs/ordered-data.h
@@ -51,6 +51,17 @@ struct btrfs_ordered_sum {
 	struct btrfs_sector_sum sums[];
 };
 
+/* holds hashes of extent's blocks to be inserted into the dedup tree  */
+struct btrfs_dedup_hash {
+	u64 extent_offset;
+	u64 *hash;
+	struct list_head list;
+};
+
+struct btrfs_dedup_metadata {
+	struct btrfs_dedup_hash hashes;
+};
+
 /*
  * bits for the flags field:
  *
@@ -114,6 +125,9 @@ struct btrfs_ordered_extent {
 	/* compression algorithm */
 	int compress_type;
 
+	/* dedup metadata */
+	struct btrfs_dedup_metadata *dedup_metadata;
+
 	/* reference count */
 	atomic_t refs;
 
@@ -176,6 +190,9 @@ int btrfs_dec_test_first_ordered_pending(struct inode *inode,
 				   int uptodate);
 int btrfs_add_ordered_extent(struct inode *inode, u64 file_offset,
 			     u64 start, u64 len, u64 disk_len, int type);
+int btrfs_add_ordered_extent_dedup(struct inode *inode, u64 file_offset,
+			     u64 start, u64 len, u64 disk_len, int type,
+			     struct btrfs_dedup_metadata *dedup_metadata);
 int btrfs_add_ordered_extent_dio(struct inode *inode, u64 file_offset,
 				 u64 start, u64 len, u64 disk_len, int type);
 int btrfs_add_ordered_extent_compress(struct inode *inode, u64 file_offset,
diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
index f0857e0..86ac211 100644
--- a/fs/btrfs/super.c
+++ b/fs/btrfs/super.c
@@ -319,7 +319,7 @@ enum {
 	Opt_no_space_cache, Opt_recovery, Opt_skip_balance,
 	Opt_check_integrity, Opt_check_integrity_including_extent_data,
 	Opt_check_integrity_print_mask, Opt_fatal_errors,
-	Opt_err,
+	Opt_err, Opt_dedup, Opt_dedupsize, Opt_dedupverify,
 };
 
 static match_table_t tokens = {
@@ -337,6 +337,9 @@ static match_table_t tokens = {
 	{Opt_compress_type, "compress=%s"},
 	{Opt_compress_force, "compress-force"},
 	{Opt_compress_force_type, "compress-force=%s"},
+	{Opt_dedup, "dedup"},
+	{Opt_dedupsize, "dedupsize=%d"},
+	{Opt_dedupverify, "dedupverify=%s"},
 	{Opt_ssd, "ssd"},
 	{Opt_ssd_spread, "ssd_spread"},
 	{Opt_nossd, "nossd"},
@@ -472,6 +475,38 @@ int btrfs_parse_options(struct btrfs_root *root, char *options)
 				pr_info("btrfs: use %s compression\n",
 					compress_type);
 			break;
+		case Opt_dedup:
+			printk(KERN_INFO "btrfs: use online deduplication\n");
+			btrfs_set_opt(info->mount_opt, DEDUP);
+			if (btrfs_test_opt(root, COMPRESS) ||
+			    btrfs_test_opt(root, FORCE_COMPRESS)) {
+				printk(KERN_INFO "btrfs: disabling compression\n");
+				info->compress_type = BTRFS_COMPRESS_NONE;
+				btrfs_clear_opt(info->mount_opt, COMPRESS);
+				btrfs_clear_opt(info->mount_opt, FORCE_COMPRESS);
+			}
+			if (btrfs_test_opt(root, NODATACOW)) {
+				printk(KERN_INFO "btrfs: disabling nodatacow\n");
+				btrfs_clear_opt(info->mount_opt, NODATACOW);
+			}
+			break;
+		case Opt_dedupsize:
+			intarg = 0;
+			match_int(&args[0], &intarg);
+			if (intarg) {
+				info->dedupsize = intarg;
+				printk(KERN_INFO
+					"btrfs: dedup block size = %llu\n",
+					info->dedupsize);
+			}
+			break;
+		case Opt_dedupverify:
+			if (strncmp(args[0].from, "on", 2) == 0) {
+				printk(KERN_INFO "btrfs: use byte-by-byte dedup "
+						"block verification\n");
+				info->dedupverify = 1;
+			}
+			break;
 		case Opt_ssd:
 			printk(KERN_INFO "btrfs: use ssd allocation scheme\n");
 			btrfs_set_opt(info->mount_opt, SSD);
@@ -911,6 +946,13 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry)
 		else
 			seq_printf(seq, ",compress=%s", compress_type);
 	}
+	if (btrfs_test_opt(root, DEDUP)) {
+		seq_puts(seq, ",dedup");
+		if (info->dedupsize)
+			seq_printf(seq, ",dedupsize=%llu", info->dedupsize);
+		if (info->dedupverify)
+			seq_puts(seq, ",dedupverify=on");
+	}
 	if (btrfs_test_opt(root, NOSSD))
 		seq_puts(seq, ",nossd");
 	if (btrfs_test_opt(root, SSD_SPREAD))
diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h
index 5ef0df5..3f3ccd37 100644
--- a/include/uapi/linux/btrfs.h
+++ b/include/uapi/linux/btrfs.h
@@ -538,5 +538,6 @@ struct btrfs_ioctl_send_args {
 				      struct btrfs_ioctl_get_dev_stats)
 #define BTRFS_IOC_DEV_REPLACE _IOWR(BTRFS_IOCTL_MAGIC, 53, \
 				    struct btrfs_ioctl_dev_replace_args)
+#define BTRFS_IOC_DEDUP_SYNC _IO(BTRFS_IOCTL_MAGIC, 54)
 
 #endif /* _UAPI_LINUX_BTRFS_H */
-- 
1.8.1.4

