diff --git a/bcwc_drv.c b/bcwc_drv.c
index 7430a28..f60a6f8 100644
--- a/bcwc_drv.c
+++ b/bcwc_drv.c
@@ -25,6 +25,7 @@
 #include <linux/workqueue.h>
 #include "bcwc_drv.h"
 #include "bcwc_hw.h"
+#include "isp.h"
 
 static int bcwc_pci_reserve_mem(struct bcwc_private *dev_priv)
 {
@@ -39,11 +40,7 @@ static int bcwc_pci_reserve_mem(struct bcwc_private *dev_priv)
 		return ret;
 	}
 
-	ret = pci_request_region(dev_priv->pdev, BCWC_PCI_S2_MEM, "S2 MEM");
-	if (ret) {
-		dev_err(&dev_priv->pdev->dev, "Failed to request S2 MEM\n");
-		return ret;
-	}
+	/* S2 MEM is reserved by the isp memory manager */
 
 	ret = pci_request_region(dev_priv->pdev, BCWC_PCI_ISP_IO, "ISP IO");
 	if (ret) {
@@ -208,6 +205,7 @@ static void bcwc_pci_remove(struct pci_dev *pdev)
 	dev_priv = pci_get_drvdata(pdev);
 
 	if (dev_priv) {
+		isp_mem_destroy(dev_priv->firmware);
 		bcwc_irq_disable(dev_priv);
 		pci_disable_msi(pdev);
 
diff --git a/bcwc_drv.h b/bcwc_drv.h
index fdc7db9..0360146 100644
--- a/bcwc_drv.h
+++ b/bcwc_drv.h
@@ -57,6 +57,10 @@ struct bcwc_private {
 	/* DDR_PHY saved registers. Offsets need to be initialized somewhere */
 	u32 ddr_phy_num_regs;
 	struct bcwc_reg ddr_phy_reg_map[DDR_PHY_NUM_REGS];
+
+	/* Root resource for memory management */
+	struct resource *mem;
+	struct isp_mem_obj *firmware;
 };
 
 #endif
diff --git a/isp.c b/isp.c
index 0cd7193..de2728f 100644
--- a/isp.c
+++ b/isp.c
@@ -18,6 +18,53 @@
 #include "bcwc_reg.h"
 #include "isp.h"
 
+int isp_mem_init(struct bcwc_private *dev_priv)
+{
+	dev_priv->mem = &dev_priv->pdev->resource[BCWC_PCI_S2_MEM];
+	if (!dev_priv->mem)
+		return -EIO;
+
+	return 0;
+}
+
+struct isp_mem_obj *isp_mem_create(struct bcwc_private *dev_priv,
+				   unsigned int type, resource_size_t size)
+{
+	struct isp_mem_obj *obj;
+	struct resource *root = dev_priv->mem;
+	int ret;
+
+	obj = kzalloc(sizeof(struct isp_mem_obj), GFP_KERNEL);
+	if (!obj)
+		return NULL;
+
+	obj->type = type;
+	obj->base.name = "S2 ISP";
+	ret = allocate_resource(root, &obj->base, size, root->start, root->end,
+				PAGE_SIZE, NULL, NULL);
+	if (ret) {
+		dev_err(&dev_priv->pdev->dev,
+			"Failed to allocate resource (size: %Ld, start: %Ld, end: %Ld, ret: %d)\n", size, root->start, root->end, ret);
+		kfree(obj);
+		obj = NULL;
+	}
+
+	obj->offset = obj->base.start - root->start;
+
+	return obj;
+}
+
+int isp_mem_destroy(struct isp_mem_obj *obj)
+{
+	if (obj) {
+		release_resource(&obj->base);
+		kfree(obj);
+		obj = NULL;
+	}
+
+	return 0;
+}
+
 int isp_acpi_set_power(struct bcwc_private *dev_priv, int power)
 {
 	acpi_status status;
@@ -71,17 +118,36 @@ static int isp_enable_sensor(struct bcwc_private *dev_priv)
 static int isp_load_firmware(struct bcwc_private *dev_priv)
 {
 	const struct firmware *fw;
+	struct resource *res;
 	int ret = 0;
 
 	ret = request_firmware(&fw, "fthd.bin", &dev_priv->pdev->dev);
-
-	if (fw == NULL)
+	if (ret)
 		return ret;
 
-	memcpy(dev_priv->s2_mem, fw->data, fw->size);
+	/*
+	 * The firmware must be the first thing we allocate since it needs to
+	 * be at offset 0. Would be better if we preallocated memory for it and
+	 * then resized it to not waste any space.
+	 */
+	dev_priv->firmware = isp_mem_create(dev_priv, FTHD_MEM_FIRMWARE,
+					    fw->size);
+	if (!dev_priv->firmware)
+		return -ENOMEM;
 
-	dev_info(&dev_priv->pdev->dev, "Loaded firmware (size: %lukb)\n",
-		 fw->size / 1024);
+	if (res->start != dev_priv->mem->start) {
+		dev_err(&dev_priv->pdev->dev,
+			"Misaligned firmware memory object (offset: %Ld)\n",
+			res->start - dev_priv->mem->start);
+		isp_mem_destroy(dev_priv->firmware);
+		return -EBUSY;
+	}
+
+	memcpy(dev_priv->s2_mem + dev_priv->firmware->offset, fw->data,
+	       fw->size);
+
+	dev_info(&dev_priv->pdev->dev, "Loaded firmware, size: %lukb (%Lukb)\n",
+		 fw->size / 1024, (res->end - res->start) / 1024);
 
 	release_firmware(fw);
 
@@ -92,9 +158,15 @@ int isp_init(struct bcwc_private *dev_priv)
 {
 	u32 num_channels, queue_size;
 	u32 reg;
-	int i, retries;
+	int i, retries, ret;
 
-	isp_load_firmware(dev_priv);
+	ret = isp_mem_init(dev_priv);
+	if (ret)
+		return ret;
+
+	ret = isp_load_firmware(dev_priv);
+	if (ret)
+		return ret;
 
 	isp_acpi_set_power(dev_priv, 1);
 	mdelay(20);
diff --git a/isp.h b/isp.h
index 9e21252..8812d8d 100644
--- a/isp.h
+++ b/isp.h
@@ -13,6 +13,26 @@
 #ifndef _ISP_H
 #define _ISP_H
 
+/* ISP memory types */
+#define FTHD_MEM_FIRMWARE	1
+#define FTHD_MEM_HEAP		2
+#define FTHD_MEM_IPC_QUEUE	3
+
+struct isp_mem_obj {
+	struct resource base;
+	unsigned int type;
+	resource_size_t size;
+	unsigned long offset;
+};
+
+#define to_isp_mem_obj(x) container_of((x), struct isp_mem_obj, base)
+
 extern int isp_init(struct bcwc_private *dev_priv);
 
+extern int isp_mem_init(struct bcwc_private *dev_priv);
+extern struct isp_mem_obj *isp_mem_create(struct bcwc_private *dev_priv,
+					  unsigned int type,
+					  resource_size_t size);
+extern int isp_mem_destroy(struct isp_mem_obj *obj);
+
 #endif