drm/ttm: convert page allocation to use page ptr array instead of list V4

Use the ttm_tt page ptr array for page allocation, move the list to
array unwinding into the page allocation functions.

V2 split the fix to use ttm put page as a separate fix
properly fill pages array when TTM_PAGE_FLAG_ZERO_ALLOC is not
set
V3 Added back page_count()==1 check when freeing page
V4 Rebase on top of memory accounting overhaul

Signed-off-by: Jerome Glisse <jglisse@redhat.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>

5 files changed, 120 insertions, 115 deletions

diff --git a/drivers/gpu/drm/ttm/ttm_memory.c b/drivers/gpu/drm/ttm/ttm_memory.c
index b550baf0201d0a..98f68994c9603f 100644
--- a/drivers/gpu/drm/ttm/ttm_memory.c
+++ b/drivers/gpu/drm/ttm/ttm_memory.c
@@ -326,32 +326,45 @@ int ttm_mem_global_alloc(struct ttm_mem_global *glob,
 }
 EXPORT_SYMBOL(ttm_mem_global_alloc);
 
-int ttm_mem_global_alloc_page(struct ttm_mem_global *glob,
-			      struct page *page,
-			      bool no_wait)
+int ttm_mem_global_alloc_pages(struct ttm_mem_global *glob,
+			       unsigned npages,
+			       bool no_wait)
 {
-
-	if (ttm_mem_global_alloc(glob, PAGE_SIZE, no_wait))
+	if (ttm_mem_global_alloc(glob, PAGE_SIZE * npages, no_wait))
 		return -ENOMEM;
+	ttm_check_swapping(glob);
+	return 0;
+}
+
+void ttm_mem_global_account_pages(struct ttm_mem_global *glob,
+				  struct page **pages,
+				  unsigned npages)
+{
+	unsigned i;
 
 	/* check if page is dma32 */
-	if (page_to_pfn(page) > 0x00100000UL) {
-		spin_lock(&glob->lock);
-		glob->used_mem -= PAGE_SIZE;
-		glob->used_dma32_mem += PAGE_SIZE;
-		spin_unlock(&glob->lock);
+	spin_lock(&glob->lock);
+	for (i = 0; i < npages; i++) {
+		if (page_to_pfn(pages[i]) > 0x00100000UL) {
+			glob->used_mem -= PAGE_SIZE;
+			glob->used_dma32_mem += PAGE_SIZE;
+		}
 	}
-	ttm_check_swapping(glob);
-	return 0;
+	spin_unlock(&glob->lock);
 }
 
-void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page)
+void ttm_mem_global_free_pages(struct ttm_mem_global *glob,
+			       struct page **pages, unsigned npages)
 {
+	unsigned i;
+
 	spin_lock(&glob->lock);
-	if (page_to_pfn(page) > 0x00100000UL) {
-		glob->used_dma32_mem -= PAGE_SIZE;
-	} else {
-		glob->used_mem -= PAGE_SIZE;
+	for (i = 0; i < npages; i++) {
+		if (page_to_pfn(pages[i]) > 0x00100000UL) {
+			glob->used_dma32_mem -= PAGE_SIZE;
+		} else {
+			glob->used_mem -= PAGE_SIZE;
+		}
 	}
 	spin_unlock(&glob->lock);
 }
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc.c b/drivers/gpu/drm/ttm/ttm_page_alloc.c
index 727e93daac3b04..c4f18b9e4ba6c2 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc.c
@@ -619,8 +619,10 @@ static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
  * @return count of pages still required to fulfill the request.
  */
 static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
-		struct list_head *pages, int ttm_flags,
-		enum ttm_caching_state cstate, unsigned count)
+					struct list_head *pages,
+					int ttm_flags,
+					enum ttm_caching_state cstate,
+					unsigned count)
 {
 	unsigned long irq_flags;
 	struct list_head *p;
@@ -664,13 +666,14 @@ out:
  * On success pages list will hold count number of correctly
  * cached pages.
  */
-int ttm_get_pages(struct list_head *pages, int flags,
-		  enum ttm_caching_state cstate, unsigned count,
-		  dma_addr_t *dma_address)
+int ttm_get_pages(struct page **pages, unsigned npages, int flags,
+		  enum ttm_caching_state cstate, dma_addr_t *dma_address)
 {
 	struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
 	struct page *p = NULL;
+	struct list_head plist;
 	gfp_t gfp_flags = GFP_USER;
+	unsigned count = 0;
 	int r;
 
 	/* set zero flag for page allocation if required */
@@ -684,94 +687,107 @@ int ttm_get_pages(struct list_head *pages, int flags,
 		else
 			gfp_flags |= GFP_HIGHUSER;
 
-		for (r = 0; r < count; ++r) {
-			p = alloc_page(gfp_flags);
-			if (!p) {
-
+		for (count = 0; count < npages; ++count) {
+			pages[count] = alloc_page(gfp_flags);
+			if (pages[count] == NULL) {
 				printk(KERN_ERR TTM_PFX
 				       "Unable to allocate page.");
 				return -ENOMEM;
 			}
-
-			list_add(&p->lru, pages);
 		}
 		return 0;
 	}
 
-
 	/* combine zero flag to pool flags */
 	gfp_flags |= pool->gfp_flags;
 
 	/* First we take pages from the pool */
-	count = ttm_page_pool_get_pages(pool, pages, flags, cstate, count);
+	INIT_LIST_HEAD(&plist);
+	npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages);
 
 	/* clear the pages coming from the pool if requested */
+	count = 0;
+	list_for_each_entry(p, &plist, lru) {
+		pages[count++] = p;
+	}
 	if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) {
-		list_for_each_entry(p, pages, lru) {
+		list_for_each_entry(p, &plist, lru) {
 			clear_page(page_address(p));
 		}
 	}
 
 	/* If pool didn't have enough pages allocate new one. */
-	if (count > 0) {
+	if (npages > 0) {
 		/* ttm_alloc_new_pages doesn't reference pool so we can run
 		 * multiple requests in parallel.
 		 **/
-		r = ttm_alloc_new_pages(pages, gfp_flags, flags, cstate, count);
+		INIT_LIST_HEAD(&plist);
+		r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, npages);
+		list_for_each_entry(p, &plist, lru) {
+			pages[count++] = p;
+		}
 		if (r) {
 			/* If there is any pages in the list put them back to
 			 * the pool. */
 			printk(KERN_ERR TTM_PFX
 			       "Failed to allocate extra pages "
 			       "for large request.");
-			ttm_put_pages(pages, 0, flags, cstate, NULL);
+			ttm_put_pages(pages, count, flags, cstate, NULL);
 			return r;
 		}
 	}
 
-
 	return 0;
 }
 
 /* Put all pages in pages list to correct pool to wait for reuse */
-void ttm_put_pages(struct list_head *pages, unsigned page_count, int flags,
+void ttm_put_pages(struct page **pages, unsigned npages, int flags,
 		   enum ttm_caching_state cstate, dma_addr_t *dma_address)
 {
 	unsigned long irq_flags;
 	struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
-	struct page *p, *tmp;
+	unsigned i;
 
 	if (pool == NULL) {
 		/* No pool for this memory type so free the pages */
 
-		list_for_each_entry_safe(p, tmp, pages, lru) {
-			__free_page(p);
+		for (i = 0; i < npages; i++) {
+			if (pages[i]) {
+				if (page_count(pages[i]) != 1)
+					printk(KERN_ERR TTM_PFX
+						"Erroneous page count. "
+						"Leaking pages.\n");
+				__free_page(pages[i]);
+				pages[i] = NULL;
+			}
 		}
-		/* Make the pages list empty */
-		INIT_LIST_HEAD(pages);
 		return;
 	}
-	if (page_count == 0) {
-		list_for_each_entry_safe(p, tmp, pages, lru) {
-			++page_count;
-		}
-	}
 
 	spin_lock_irqsave(&pool->lock, irq_flags);
-	list_splice_init(pages, &pool->list);
-	pool->npages += page_count;
+	for (i = 0; i < npages; i++) {
+		if (pages[i]) {
+			if (page_count(pages[i]) != 1)
+				printk(KERN_ERR TTM_PFX
+					"Erroneous page count. "
+					"Leaking pages.\n");
+			list_add_tail(&pages[i]->lru, &pool->list);
+			pages[i] = NULL;
+			pool->npages++;
+		}
+	}
 	/* Check that we don't go over the pool limit */
-	page_count = 0;
+	npages = 0;
 	if (pool->npages > _manager->options.max_size) {
-		page_count = pool->npages - _manager->options.max_size;
+		npages = pool->npages - _manager->options.max_size;
 		/* free at least NUM_PAGES_TO_ALLOC number of pages
 		 * to reduce calls to set_memory_wb */
-		if (page_count < NUM_PAGES_TO_ALLOC)
-			page_count = NUM_PAGES_TO_ALLOC;
+		if (npages < NUM_PAGES_TO_ALLOC)
+			npages = NUM_PAGES_TO_ALLOC;
 	}
 	spin_unlock_irqrestore(&pool->lock, irq_flags);
-	if (page_count)
-		ttm_page_pool_free(pool, page_count);
+	if (npages)
+		ttm_page_pool_free(pool, npages);
 }
 
 static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags,
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index d0169af7a53add..872d6cb1d93c02 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -66,35 +66,23 @@ static void ttm_tt_free_page_directory(struct ttm_tt *ttm)
 static struct page *__ttm_tt_get_page(struct ttm_tt *ttm, int index)
 {
 	struct page *p;
-	struct list_head h;
 	struct ttm_mem_global *mem_glob = ttm->glob->mem_glob;
 	int ret;
 
 	if (NULL == (p = ttm->pages[index])) {
+		ret = ttm_mem_global_alloc_pages(mem_glob, 1, false);
+		if (unlikely(ret != 0))
+			return NULL;
 
-		INIT_LIST_HEAD(&h);
-
-		ret = ttm_get_pages(&h, ttm->page_flags, ttm->caching_state, 1,
+		ret = ttm_get_pages(&ttm->pages[index], 1, ttm->page_flags,
+				    ttm->caching_state,
 				    &ttm->dma_address[index]);
-
 		if (ret != 0)
 			return NULL;
 
-		p = list_first_entry(&h, struct page, lru);
-
-		ret = ttm_mem_global_alloc_page(mem_glob, p, false);
-		if (unlikely(ret != 0))
-			goto out_err;
-
-		ttm->pages[index] = p;
+		ttm_mem_global_account_pages(mem_glob, &ttm->pages[index], 1);
 	}
 	return p;
-out_err:
-	INIT_LIST_HEAD(&h);
-	list_add(&p->lru, &h);
-	ttm_put_pages(&h, 1, ttm->page_flags,
-		      ttm->caching_state, &ttm->dma_address[index]);
-	return NULL;
 }
 
 struct page *ttm_tt_get_page(struct ttm_tt *ttm, int index)
@@ -111,8 +99,7 @@ struct page *ttm_tt_get_page(struct ttm_tt *ttm, int index)
 
 int ttm_tt_populate(struct ttm_tt *ttm)
 {
-	struct page *page;
-	unsigned long i;
+	struct ttm_mem_global *mem_glob = ttm->glob->mem_glob;
 	struct ttm_backend *be;
 	int ret;
 
@@ -127,11 +114,16 @@ int ttm_tt_populate(struct ttm_tt *ttm)
 
 	be = ttm->be;
 
-	for (i = 0; i < ttm->num_pages; ++i) {
-		page = __ttm_tt_get_page(ttm, i);
-		if (!page)
-			return -ENOMEM;
-	}
+	ret = ttm_mem_global_alloc_pages(mem_glob, ttm->num_pages, false);
+	if (unlikely(ret != 0))
+		return -ENOMEM;
+
+	ret = ttm_get_pages(ttm->pages, ttm->num_pages, ttm->page_flags,
+			    ttm->caching_state, ttm->dma_address);
+	if (ret != 0)
+		return -ENOMEM;
+
+	ttm_mem_global_account_pages(mem_glob, ttm->pages, ttm->num_pages);
 
 	be->func->populate(be, ttm->num_pages, ttm->pages,
 			   ttm->dummy_read_page, ttm->dma_address);
@@ -243,33 +235,15 @@ EXPORT_SYMBOL(ttm_tt_set_placement_caching);
 
 static void ttm_tt_free_alloced_pages(struct ttm_tt *ttm)
 {
-	int i;
-	unsigned count = 0;
-	struct list_head h;
-	struct page *cur_page;
 	struct ttm_backend *be = ttm->be;
-
-	INIT_LIST_HEAD(&h);
+	struct ttm_mem_global *glob = ttm->glob->mem_glob;
 
 	if (be)
 		be->func->clear(be);
-	for (i = 0; i < ttm->num_pages; ++i) {
 
-		cur_page = ttm->pages[i];
-		ttm->pages[i] = NULL;
-		if (cur_page) {
-			if (page_count(cur_page) != 1)
-				printk(KERN_ERR TTM_PFX
-				       "Erroneous page count. "
-				       "Leaking pages.\n");
-			ttm_mem_global_free_page(ttm->glob->mem_glob,
-						 cur_page);
-			list_add(&cur_page->lru, &h);
-			count++;
-		}
-	}
-	ttm_put_pages(&h, count, ttm->page_flags, ttm->caching_state,
-		      ttm->dma_address);
+	ttm_mem_global_free_pages(glob, ttm->pages, ttm->num_pages);
+	ttm_put_pages(ttm->pages, ttm->num_pages, ttm->page_flags,
+			ttm->caching_state, ttm->dma_address);
 	ttm->state = tt_unpopulated;
 }
 
diff --git a/include/drm/ttm/ttm_memory.h b/include/drm/ttm/ttm_memory.h
index d24792b371f148..c0c732d42cbcdb 100644
--- a/include/drm/ttm/ttm_memory.h
+++ b/include/drm/ttm/ttm_memory.h
@@ -150,11 +150,14 @@ extern int ttm_mem_global_alloc(struct ttm_mem_global *glob,
 				bool no_wait);
 extern void ttm_mem_global_free(struct ttm_mem_global *glob,
 				uint64_t amount);
-extern int ttm_mem_global_alloc_page(struct ttm_mem_global *glob,
-				     struct page *page,
-				     bool no_wait);
-extern void ttm_mem_global_free_page(struct ttm_mem_global *glob,
-				     struct page *page);
+extern int ttm_mem_global_alloc_pages(struct ttm_mem_global *glob,
+				      unsigned npages,
+				      bool no_wait);
+extern void ttm_mem_global_free_pages(struct ttm_mem_global *glob,
+				      struct page **pages, unsigned npages);
+extern void ttm_mem_global_account_pages(struct ttm_mem_global *glob,
+					 struct page **pages,
+					 unsigned npages);
 extern size_t ttm_round_pot(size_t size);
 
 #endif
diff --git a/include/drm/ttm/ttm_page_alloc.h b/include/drm/ttm/ttm_page_alloc.h
index 129de12353f191..fffb3bd2c887af 100644
--- a/include/drm/ttm/ttm_page_alloc.h
+++ b/include/drm/ttm/ttm_page_alloc.h
@@ -32,29 +32,28 @@
 /**
  * Get count number of pages from pool to pages list.
  *
- * @pages: head of empty linked list where pages are filled.
+ * @pages: array of pages ptr
+ * @npages: number of pages to allocate.
  * @flags: ttm flags for page allocation.
  * @cstate: ttm caching state for the page.
- * @count: number of pages to allocate.
  * @dma_address: The DMA (bus) address of pages (if TTM_PAGE_FLAG_DMA32 set).
  */
-int ttm_get_pages(struct list_head *pages,
+int ttm_get_pages(struct page **pages,
+		  unsigned npages,
 		  int flags,
 		  enum ttm_caching_state cstate,
-		  unsigned count,
 		  dma_addr_t *dma_address);
 /**
  * Put linked list of pages to pool.
  *
- * @pages: list of pages to free.
- * @page_count: number of pages in the list. Zero can be passed for unknown
- * count.
+ * @pages: array of pages ptr
+ * @npages: number of pages to free.
  * @flags: ttm flags for page allocation.
  * @cstate: ttm caching state.
  * @dma_address: The DMA (bus) address of pages (if TTM_PAGE_FLAG_DMA32 set).
  */
-void ttm_put_pages(struct list_head *pages,
-		   unsigned page_count,
+void ttm_put_pages(struct page **pages,
+		   unsigned npages,
 		   int flags,
 		   enum ttm_caching_state cstate,
 		   dma_addr_t *dma_address);