From: Roland Dreier Clean up the computation of the HCA context memory map. This serves two purposes: - make it easier to change the HCA "profile" (eg add more QPs) - make it easier to implement mem-free Arbel support Signed-off-by: Roland Dreier Signed-off-by: Andrew Morton --- 25-akpm/drivers/infiniband/hw/mthca/mthca_cmd.c | 8 + 25-akpm/drivers/infiniband/hw/mthca/mthca_cmd.h | 6 - 25-akpm/drivers/infiniband/hw/mthca/mthca_dev.h | 7 + 25-akpm/drivers/infiniband/hw/mthca/mthca_main.c | 21 ++++ 25-akpm/drivers/infiniband/hw/mthca/mthca_profile.c | 91 ++++++++++++-------- 25-akpm/drivers/infiniband/hw/mthca/mthca_profile.h | 26 ++--- 6 files changed, 102 insertions(+), 57 deletions(-) diff -puN drivers/infiniband/hw/mthca/mthca_cmd.c~infiniband-mthca-clean-up-computation-of-hca-memory-map drivers/infiniband/hw/mthca/mthca_cmd.c --- 25/drivers/infiniband/hw/mthca/mthca_cmd.c~infiniband-mthca-clean-up-computation-of-hca-memory-map Wed Jan 12 16:31:52 2005 +++ 25-akpm/drivers/infiniband/hw/mthca/mthca_cmd.c Wed Jan 12 16:31:52 2005 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004 Topspin Communications. All rights reserved. + * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -959,9 +959,9 @@ int mthca_QUERY_DEV_LIM(struct mthca_dev MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSZ_SRQ_OFFSET); dev_lim->hca.arbel.resize_srq = field & 1; MTHCA_GET(size, outbox, QUERY_DEV_LIM_MTT_ENTRY_SZ_OFFSET); - dev_lim->hca.arbel.mtt_entry_sz = size; + dev_lim->mtt_seg_sz = size; MTHCA_GET(size, outbox, QUERY_DEV_LIM_MPT_ENTRY_SZ_OFFSET); - dev_lim->hca.arbel.mpt_entry_sz = size; + dev_lim->mpt_entry_sz = size; MTHCA_GET(field, outbox, QUERY_DEV_LIM_PBL_SZ_OFFSET); dev_lim->hca.arbel.max_pbl_sz = 1 << (field & 0x3f); MTHCA_GET(dev_lim->hca.arbel.bmme_flags, outbox, @@ -987,6 +987,8 @@ int mthca_QUERY_DEV_LIM(struct mthca_dev } else { MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_AV_OFFSET); dev_lim->hca.tavor.max_avs = 1 << (field & 0x3f); + dev_lim->mtt_seg_sz = MTHCA_MTT_SEG_SIZE; + dev_lim->mpt_entry_sz = MTHCA_MPT_ENTRY_SIZE; } out: diff -puN drivers/infiniband/hw/mthca/mthca_cmd.h~infiniband-mthca-clean-up-computation-of-hca-memory-map drivers/infiniband/hw/mthca/mthca_cmd.h --- 25/drivers/infiniband/hw/mthca/mthca_cmd.h~infiniband-mthca-clean-up-computation-of-hca-memory-map Wed Jan 12 16:31:52 2005 +++ 25-akpm/drivers/infiniband/hw/mthca/mthca_cmd.h Wed Jan 12 16:31:52 2005 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004 Topspin Communications. All rights reserved. + * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -148,14 +148,14 @@ struct mthca_dev_lim { int cqc_entry_sz; int srq_entry_sz; int uar_scratch_entry_sz; + int mtt_seg_sz; + int mpt_entry_sz; union { struct { int max_avs; } tavor; struct { int resize_srq; - int mtt_entry_sz; - int mpt_entry_sz; int max_pbl_sz; u8 bmme_flags; u32 reserved_lkey; diff -puN drivers/infiniband/hw/mthca/mthca_dev.h~infiniband-mthca-clean-up-computation-of-hca-memory-map drivers/infiniband/hw/mthca/mthca_dev.h --- 25/drivers/infiniband/hw/mthca/mthca_dev.h~infiniband-mthca-clean-up-computation-of-hca-memory-map Wed Jan 12 16:31:52 2005 +++ 25-akpm/drivers/infiniband/hw/mthca/mthca_dev.h Wed Jan 12 16:31:52 2005 @@ -70,13 +70,16 @@ enum { }; enum { - MTHCA_MPT_ENTRY_SIZE = 0x40, MTHCA_EQ_CONTEXT_SIZE = 0x40, MTHCA_CQ_CONTEXT_SIZE = 0x40, MTHCA_QP_CONTEXT_SIZE = 0x200, MTHCA_RDB_ENTRY_SIZE = 0x20, MTHCA_AV_SIZE = 0x20, - MTHCA_MGM_ENTRY_SIZE = 0x40 + MTHCA_MGM_ENTRY_SIZE = 0x40, + + /* Arbel FW gives us these, but we need them for Tavor */ + MTHCA_MPT_ENTRY_SIZE = 0x40, + MTHCA_MTT_SEG_SIZE = 0x40, }; enum { diff -puN drivers/infiniband/hw/mthca/mthca_main.c~infiniband-mthca-clean-up-computation-of-hca-memory-map drivers/infiniband/hw/mthca/mthca_main.c --- 25/drivers/infiniband/hw/mthca/mthca_main.c~infiniband-mthca-clean-up-computation-of-hca-memory-map Wed Jan 12 16:31:52 2005 +++ 25-akpm/drivers/infiniband/hw/mthca/mthca_main.c Wed Jan 12 16:31:52 2005 @@ -76,6 +76,20 @@ static const char mthca_version[] __devi "ib_mthca: Mellanox InfiniBand HCA driver v" DRV_VERSION " (" DRV_RELDATE ")\n"; +static struct mthca_profile default_profile = { + .num_qp = 1 << 16, + .rdb_per_qp = 4, + .num_cq = 1 << 16, + .num_mcg = 1 << 13, + .num_mpt = 1 << 17, + .num_mtt = 1 << 20 +}; + +enum { + MTHCA_TAVOR_NUM_UDAV = 1 << 15, + MTHCA_ARBEL_UARC_SIZE = 1 << 18 +}; + static int __devinit mthca_tune_pci(struct mthca_dev *mdev) { int cap; @@ -175,6 +189,7 @@ static int __devinit mthca_init_tavor(st u8 status; int err; struct mthca_dev_lim dev_lim; + struct mthca_profile profile; struct mthca_init_hca_param init_hca; struct mthca_adapter adapter; @@ -214,7 +229,11 @@ static int __devinit mthca_init_tavor(st err = mthca_dev_lim(mdev, &dev_lim); - err = mthca_make_profile(mdev, &dev_lim, &init_hca); + profile = default_profile; + profile.num_uar = dev_lim.uar_size / PAGE_SIZE; + profile.num_udav = MTHCA_TAVOR_NUM_UDAV; + + err = mthca_make_profile(mdev, &profile, &dev_lim, &init_hca); if (err) goto err_out_disable; diff -puN drivers/infiniband/hw/mthca/mthca_profile.c~infiniband-mthca-clean-up-computation-of-hca-memory-map drivers/infiniband/hw/mthca/mthca_profile.c --- 25/drivers/infiniband/hw/mthca/mthca_profile.c~infiniband-mthca-clean-up-computation-of-hca-memory-map Wed Jan 12 16:31:52 2005 +++ 25-akpm/drivers/infiniband/hw/mthca/mthca_profile.c Wed Jan 12 16:31:52 2005 @@ -37,31 +37,34 @@ #include "mthca_profile.h" -static int default_profile[MTHCA_RES_NUM] = { - [MTHCA_RES_QP] = 1 << 16, - [MTHCA_RES_EQP] = 1 << 16, - [MTHCA_RES_CQ] = 1 << 16, - [MTHCA_RES_EQ] = 32, - [MTHCA_RES_RDB] = 1 << 18, - [MTHCA_RES_MCG] = 1 << 13, - [MTHCA_RES_MPT] = 1 << 17, - [MTHCA_RES_MTT] = 1 << 20, - [MTHCA_RES_UDAV] = 1 << 15 -}; - enum { - MTHCA_MTT_SEG_SIZE = 64 + MTHCA_RES_QP, + MTHCA_RES_EEC, + MTHCA_RES_SRQ, + MTHCA_RES_CQ, + MTHCA_RES_EQP, + MTHCA_RES_EEEC, + MTHCA_RES_EQ, + MTHCA_RES_RDB, + MTHCA_RES_MCG, + MTHCA_RES_MPT, + MTHCA_RES_MTT, + MTHCA_RES_UAR, + MTHCA_RES_UDAV, + MTHCA_RES_UARC, + MTHCA_RES_NUM }; enum { + MTHCA_NUM_EQS = 32, MTHCA_NUM_PDS = 1 << 15 }; int mthca_make_profile(struct mthca_dev *dev, + struct mthca_profile *request, struct mthca_dev_lim *dev_lim, struct mthca_init_hca_param *init_hca) { - /* just use default profile for now */ struct mthca_resource { u64 size; u64 start; @@ -70,17 +73,18 @@ int mthca_make_profile(struct mthca_dev int log_num; }; + u64 mem_base, mem_avail; u64 total_size = 0; struct mthca_resource *profile; struct mthca_resource tmp; int i, j; - default_profile[MTHCA_RES_UAR] = dev_lim->uar_size / PAGE_SIZE; - profile = kmalloc(MTHCA_RES_NUM * sizeof *profile, GFP_KERNEL); if (!profile) return -ENOMEM; + memset(profile, 0, MTHCA_RES_NUM * sizeof *profile); + profile[MTHCA_RES_QP].size = dev_lim->qpc_entry_sz; profile[MTHCA_RES_EEC].size = dev_lim->eec_entry_sz; profile[MTHCA_RES_SRQ].size = dev_lim->srq_entry_sz; @@ -90,16 +94,36 @@ int mthca_make_profile(struct mthca_dev profile[MTHCA_RES_EQ].size = dev_lim->eqc_entry_sz; profile[MTHCA_RES_RDB].size = MTHCA_RDB_ENTRY_SIZE; profile[MTHCA_RES_MCG].size = MTHCA_MGM_ENTRY_SIZE; - profile[MTHCA_RES_MPT].size = MTHCA_MPT_ENTRY_SIZE; - profile[MTHCA_RES_MTT].size = MTHCA_MTT_SEG_SIZE; + profile[MTHCA_RES_MPT].size = dev_lim->mpt_entry_sz; + profile[MTHCA_RES_MTT].size = dev_lim->mtt_seg_sz; profile[MTHCA_RES_UAR].size = dev_lim->uar_scratch_entry_sz; profile[MTHCA_RES_UDAV].size = MTHCA_AV_SIZE; + profile[MTHCA_RES_UARC].size = request->uarc_size; + + profile[MTHCA_RES_QP].num = request->num_qp; + profile[MTHCA_RES_EQP].num = request->num_qp; + profile[MTHCA_RES_RDB].num = request->num_qp * request->rdb_per_qp; + profile[MTHCA_RES_CQ].num = request->num_cq; + profile[MTHCA_RES_EQ].num = MTHCA_NUM_EQS; + profile[MTHCA_RES_MCG].num = request->num_mcg; + profile[MTHCA_RES_MPT].num = request->num_mpt; + profile[MTHCA_RES_MTT].num = request->num_mtt; + profile[MTHCA_RES_UAR].num = request->num_uar; + profile[MTHCA_RES_UARC].num = request->num_uar; + profile[MTHCA_RES_UDAV].num = request->num_udav; for (i = 0; i < MTHCA_RES_NUM; ++i) { profile[i].type = i; - profile[i].num = default_profile[i]; - profile[i].log_num = max(ffs(default_profile[i]) - 1, 0); - profile[i].size *= default_profile[i]; + profile[i].log_num = max(ffs(profile[i].num) - 1, 0); + profile[i].size *= profile[i].num; + } + + if (dev->hca_type == ARBEL_NATIVE) { + mem_base = 0; + mem_avail = dev_lim->hca.arbel.max_icm_sz; + } else { + mem_base = dev->ddr_start; + mem_avail = dev->fw.tavor.fw_start - dev->ddr_start; } /* @@ -119,16 +143,14 @@ int mthca_make_profile(struct mthca_dev for (i = 0; i < MTHCA_RES_NUM; ++i) { if (profile[i].size) { - profile[i].start = dev->ddr_start + total_size; + profile[i].start = mem_base + total_size; total_size += profile[i].size; } - if (total_size > dev->fw.tavor.fw_start - dev->ddr_start) { + if (total_size > mem_avail) { mthca_err(dev, "Profile requires 0x%llx bytes; " - "won't fit between DDR start at 0x%016llx " - "and FW start at 0x%016llx.\n", + "won't in 0x%llx bytes of context memory.\n", (unsigned long long) total_size, - (unsigned long long) dev->ddr_start, - (unsigned long long) dev->fw.tavor.fw_start); + (unsigned long long) mem_avail); kfree(profile); return -ENOMEM; } @@ -141,10 +163,13 @@ int mthca_make_profile(struct mthca_dev (unsigned long long) profile[i].size); } - mthca_dbg(dev, "HCA memory: allocated %d KB/%d KB (%d KB free)\n", - (int) (total_size >> 10), - (int) ((dev->fw.tavor.fw_start - dev->ddr_start) >> 10), - (int) ((dev->fw.tavor.fw_start - dev->ddr_start - total_size) >> 10)); + if (dev->hca_type == ARBEL_NATIVE) + mthca_dbg(dev, "HCA context memory: reserving %d KB\n", + (int) (total_size >> 10)); + else + mthca_dbg(dev, "HCA memory: allocated %d KB/%d KB (%d KB free)\n", + (int) (total_size >> 10), (int) (mem_avail >> 10), + (int) ((mem_avail - total_size) >> 10)); for (i = 0; i < MTHCA_RES_NUM; ++i) { switch (profile[i].type) { @@ -203,10 +228,10 @@ int mthca_make_profile(struct mthca_dev break; case MTHCA_RES_MTT: dev->limits.num_mtt_segs = profile[i].num; - dev->limits.mtt_seg_size = MTHCA_MTT_SEG_SIZE; + dev->limits.mtt_seg_size = dev_lim->mtt_seg_sz; dev->mr_table.mtt_base = profile[i].start; init_hca->mtt_base = profile[i].start; - init_hca->mtt_seg_sz = ffs(MTHCA_MTT_SEG_SIZE) - 7; + init_hca->mtt_seg_sz = ffs(dev_lim->mtt_seg_sz) - 7; break; case MTHCA_RES_UAR: init_hca->uar_scratch_base = profile[i].start; diff -puN drivers/infiniband/hw/mthca/mthca_profile.h~infiniband-mthca-clean-up-computation-of-hca-memory-map drivers/infiniband/hw/mthca/mthca_profile.h --- 25/drivers/infiniband/hw/mthca/mthca_profile.h~infiniband-mthca-clean-up-computation-of-hca-memory-map Wed Jan 12 16:31:52 2005 +++ 25-akpm/drivers/infiniband/hw/mthca/mthca_profile.h Wed Jan 12 16:31:52 2005 @@ -38,24 +38,20 @@ #include "mthca_dev.h" #include "mthca_cmd.h" -enum { - MTHCA_RES_QP, - MTHCA_RES_EEC, - MTHCA_RES_SRQ, - MTHCA_RES_CQ, - MTHCA_RES_EQP, - MTHCA_RES_EEEC, - MTHCA_RES_EQ, - MTHCA_RES_RDB, - MTHCA_RES_MCG, - MTHCA_RES_MPT, - MTHCA_RES_MTT, - MTHCA_RES_UAR, - MTHCA_RES_UDAV, - MTHCA_RES_NUM +struct mthca_profile { + int num_qp; + int rdb_per_qp; + int num_cq; + int num_mcg; + int num_mpt; + int num_mtt; + int num_udav; + int num_uar; + int uarc_size; }; int mthca_make_profile(struct mthca_dev *mdev, + struct mthca_profile *request, struct mthca_dev_lim *dev_lim, struct mthca_init_hca_param *init_hca); _