SIS_DEPTHsched-sisdepth-v1r3

5 files changed, 21 insertions, 56 deletions

diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h
index 8f0f778b7c9111..681a5fe19a3065 100644
--- a/include/linux/sched/topology.h
+++ b/include/linux/sched/topology.h
@@ -100,7 +100,7 @@ struct sched_domain {
 	u64 max_newidle_lb_cost;
 	unsigned long next_decay_max_lb_cost;
 
-	u64 avg_scan_cost;		/* select_idle_sibling */
+	int sis_scan_depth;		/* select_idle_sibling */
 
 #ifdef CONFIG_SCHEDSTATS
 	/* load_balance() stats */
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index ff235c15eb631b..c74ae2260ecb47 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -3545,9 +3545,6 @@ static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags,
 		if (rq->avg_idle > max)
 			rq->avg_idle = max;
 
-		rq->wake_stamp = jiffies;
-		rq->wake_avg_idle = rq->avg_idle / 2;
-
 		rq->idle_stamp = 0;
 	}
 #endif
@@ -9412,8 +9409,6 @@ void __init sched_init(void)
 		rq->online = 0;
 		rq->idle_stamp = 0;
 		rq->avg_idle = 2*sysctl_sched_migration_cost;
-		rq->wake_stamp = jiffies;
-		rq->wake_avg_idle = rq->avg_idle;
 		rq->max_idle_balance_cost = sysctl_sched_migration_cost;
 
 		INIT_LIST_HEAD(&rq->cfs_tasks);
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 93acb537923c9b..91bdd6c3af1373 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -6265,17 +6265,14 @@ static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd
 
 /*
  * Scan the LLC domain for idle CPUs; this is dynamically regulated by
- * comparing the average scan cost (tracked in sd->avg_scan_cost) against the
- * average idle time for this rq (as found in rq->avg_idle).
+ * the success or failure of previous scans;
  */
 static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool has_idle_core, int target)
 {
 	struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_idle_mask);
 	int i, cpu, idle_cpu = -1, nr = INT_MAX;
-	struct rq *this_rq = this_rq();
-	int this = smp_processor_id();
 	struct sched_domain *this_sd;
-	u64 time = 0;
+	int depth = 0;
 
 	this_sd = rcu_dereference(*this_cpu_ptr(&sd_llc));
 	if (!this_sd)
@@ -6283,39 +6280,21 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool
 
 	cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr);
 
-	if (sched_feat(SIS_PROP)) {
-		u64 avg_cost, avg_idle, span_avg;
-		unsigned long now = jiffies;
-
-		/*
-		 * If we're busy, the assumption that the last idle period
-		 * predicts the future is flawed; age away the remaining
-		 * predicted idle time.
-		 */
-		if (unlikely(this_rq->wake_stamp < now)) {
-			while (this_rq->wake_stamp < now && this_rq->wake_avg_idle) {
-				this_rq->wake_stamp++;
-				this_rq->wake_avg_idle >>= 1;
-			}
-		}
-
-		avg_idle = this_rq->wake_avg_idle;
-		avg_cost = this_sd->avg_scan_cost + 1;
-
-		span_avg = sd->span_weight * avg_idle;
-		if (span_avg > sis_min_cores * avg_cost)
-			nr = div_u64(span_avg, avg_cost);
-		else
-			nr = sis_min_cores;
+	if (sched_feat(SIS_DEPTH)) {
+		depth = min_t(int, this_sd->sis_scan_depth, nr);
 
 		if (!has_idle_core)
 			nr *= sched_smt_weight;
-		time = cpu_clock(this);
+		nr = depth;
 	}
 
+	cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr);
+
 	for_each_cpu_wrap(cpu, cpus, target + 1) {
-		if (--nr < 0)
+		if (--nr < 0) {
+			depth = clamp(depth - 1, 1, sis_min_cores * sched_smt_weight);
 			break;
+		}
 
 		if (has_idle_core) {
 			i = select_idle_core(p, cpu, cpus, &idle_cpu);
@@ -6328,21 +6307,14 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool
 		}
 	}
 
-	if ((unsigned int)idle_cpu < nr_cpumask_bits) {
-		if (has_idle_core)
-			set_idle_cores(target, false);
-
-		if (sched_feat(SIS_PROP)) {
-			time = cpu_clock(this) - time;
+	if ((unsigned int)idle_cpu < nr_cpumask_bits && has_idle_core)
+		set_idle_cores(target, false);
 
-			/*
-			 * Account for the scan cost of wakeups against the average
-			 * idle time.
-			 */
-			this_rq->wake_avg_idle -= min(this_rq->wake_avg_idle, time);
-
-			update_avg(&this_sd->avg_scan_cost, time);
-		}
+	if (sched_feat(SIS_DEPTH)) {
+		if (idle_cpu)
+			depth = min_t(int, depth + 1, sd->span_weight / sched_smt_weight);
+		if (this_sd->sis_scan_depth != depth)
+			this_sd->sis_scan_depth = depth;
 	}
 
 	return idle_cpu;
diff --git a/kernel/sched/features.h b/kernel/sched/features.h
index 7f8dace0964c22..4e685a8752277b 100644
--- a/kernel/sched/features.h
+++ b/kernel/sched/features.h
@@ -53,9 +53,10 @@ SCHED_FEAT(NONTASK_CAPACITY, true)
 SCHED_FEAT(TTWU_QUEUE, true)
 
 /*
- * When doing wakeups, attempt to limit superfluous scans of the LLC domain.
+ * When doing wakeups, attempt to limit superfluous scans of the LLC domain
+ * based on previous scans success or failure.
  */
-SCHED_FEAT(SIS_PROP, true)
+SCHED_FEAT(SIS_DEPTH, true)
 
 /*
  * Issue a WARN when we do multiple update_rq_clock() calls
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 4541dab9afa4e8..056ff1c457330f 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1033,9 +1033,6 @@ struct rq {
 	u64			idle_stamp;
 	u64			avg_idle;
 
-	unsigned long		wake_stamp;
-	u64			wake_avg_idle;
-
 	/* This is used to determine avg_idle's max value */
 	u64			max_idle_balance_cost;