extend algo wip

Author: eddyz87

include/linux/bpf_verifier.h

@@ -714,19 +714,14 @@ struct bpf_scc_info {
 	 * Indicates that read and precision marks are incomplete for
 	 * states with insn_idx in this SCC.
 	 */
-	u32 state_loops_possible:1;
-	/* Number of verifier states with .branches > 0 that have
-	 * state->parent->insn_idx within this SCC.
-	 * In other words, the number of states originating from this
-	 * SCC that have not yet been fully explored.
-	 */
-	u32 branches:31;
+	bool state_loops_possible;
 	/* Number of times this SCC was entered by some verifier state
 	 * and that state was fully explored.
 	 * In other words, number of times .branches became non-zero
 	 * and then zero again.
 	 */
 	u32 scc_epoch;
+	struct bpf_verifier_state *entry_state;
 };
 
 /* single container for all structs

kernel/bpf/verifier.c

@@ -1787,6 +1787,8 @@ static struct bpf_scc_info *insn_scc(struct bpf_verifier_env *env, int insn_idx)
 	return scc ? &env->scc_info[scc] : NULL;
 }
 
+static u32 frame_insn_idx(struct bpf_verifier_state *st, u32 frame);
+
 /* Returns true iff:
  * - verifier is currently exploring states with origins in some CFG SCCs;
  * - st->insn_idx belongs to one of these SCCs;
@@ -1803,12 +1805,19 @@ static bool incomplete_read_marks(struct bpf_verifier_env *env,
 				  struct bpf_verifier_state *st)
 {
 	struct bpf_scc_info *scc_info;
+	u32 insn_idx, i;
 
-	scc_info = insn_scc(env, st->insn_idx);
-	return scc_info &&
-	       scc_info->state_loops_possible &&
-	       scc_info->scc_epoch == st->scc_epoch &&
-	       scc_info->branches > 0;
+	for (i = 0; i <= st->curframe; i++) {
+		insn_idx = frame_insn_idx(st, i);
+		scc_info = insn_scc(env, st->insn_idx);
+		if (scc_info &&
+		    scc_info->state_loops_possible &&
+		    scc_info->scc_epoch == st->scc_epoch &&
+		    scc_info->entry_state)
+			return true;
+	}
+
+	return false;
 }
 
 static void mark_state_loops_possible(struct bpf_verifier_env *env,
@@ -1821,42 +1830,12 @@ static void mark_state_loops_possible(struct bpf_verifier_env *env,
 		scc_info->state_loops_possible = 1;
 }
 
-/* See comments for bpf_scc_info->{branches,visit_count} and
- * mark_all_regs_read_and_precise().
- */
-static void parent_scc_enter(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
-{
-	struct bpf_scc_info *scc_info;
-
-	if (!st->parent)
-		return;
-	scc_info = insn_scc(env, st->parent->insn_idx);
-	if (scc_info)
-		scc_info->branches++;
-}
-
-/* See comments for bpf_scc_info->{branches,visit_count} and
- * mark_all_regs_read_and_precise().
- */
-static void parent_scc_exit(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
-{
-	struct bpf_scc_info *scc_info;
-
-	if (!st->parent)
-		return;
-	scc_info = insn_scc(env, st->parent->insn_idx);
-	if (scc_info) {
-		WARN_ON_ONCE(scc_info->branches == 0);
-		scc_info->branches--;
-		if (scc_info->branches == 0)
-			scc_info->scc_epoch++;
-	}
-}
-
 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
 {
 	struct bpf_verifier_state_list *sl = NULL, *parent_sl;
 	struct bpf_verifier_state *parent;
+	struct bpf_scc_info *scc_info;
+	u32 insn_idx, i;
 
 	while (st) {
 		u32 br = --st->branches;
@@ -1869,7 +1848,17 @@ static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifi
 			  br);
 		if (br)
 			break;
-		parent_scc_exit(env, st);
+		for (i = 0; i <= st->curframe; i++) {
+			insn_idx = frame_insn_idx(st, i);
+			scc_info = insn_scc(env, insn_idx);
+			if (scc_info && scc_info->entry_state == st) {
+				scc_info->entry_state = NULL;
+				scc_info->scc_epoch++;
+				if (env->log.level & BPF_LOG_LEVEL2)
+					verbose(env, "insn #%d, scc #%ld new epoch %d\n",
+						st->insn_idx, scc_info - env->scc_info, scc_info->scc_epoch);
+			}
+		}
 		parent = st->parent;
 		parent_sl = state_parent_as_list(st);
 		if (sl)
@@ -1947,7 +1936,6 @@ static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
 		 * which might have large 'branches' count.
 		 */
 	}
-	parent_scc_enter(env, &elem->st);
 	return &elem->st;
 err:
 	free_verifier_state(env->cur_state, true);
@@ -18240,7 +18228,7 @@ static void mark_all_regs_read_and_precise(struct bpf_verifier_env *env,
 
 	for (i = 0; i <= st->curframe; i++) {
 		insn_idx = frame_insn_idx(st, i);
-		live_regs = env->insn_aux_data[st->insn_idx].live_regs_before;
+		live_regs = env->insn_aux_data[insn_idx].live_regs_before;
 		func = st->frame[i];
 		for (j = 0; j < BPF_REG_FP; j++) {
 			reg = &func->regs[j];
@@ -18298,8 +18286,8 @@ static void clean_live_states(struct bpf_verifier_env *env, int insn,
 	struct bpf_verifier_state_list *sl;
 	struct bpf_scc_info *scc_info;
 	struct list_head *pos, *head;
+	u32 insn_idx, i;
 
-	scc_info = insn_scc(env, insn);
 	head = explored_state(env, insn);
 	list_for_each(pos, head) {
 		sl = container_of(pos, struct bpf_verifier_state_list, node);
@@ -18312,10 +18300,25 @@ static void clean_live_states(struct bpf_verifier_env *env, int insn,
 			continue;
 		if (incomplete_read_marks(env, &sl->state))
 			continue;
-		if (scc_info &&
-		    scc_info->state_loops_possible &&
-		    scc_info->scc_epoch > sl->state.scc_epoch)
-			mark_all_regs_read_and_precise(env, &sl->state);
+		for (i = 0; i <= sl->state.curframe; i++) {
+			insn_idx = frame_insn_idx(&sl->state, i);
+			scc_info = insn_scc(env, insn_idx);
+			if (env->log.level & BPF_LOG_LEVEL2)
+				verbose(env, "clean_live_states: insn #%d, scc #%ld epoch %d vs %d, loops possible? %d\n",
+					insn,
+					scc_info ? scc_info - env->scc_info : -1,
+					sl->state.scc_epoch,
+					scc_info ? scc_info->scc_epoch : -1,
+					scc_info ? scc_info->state_loops_possible : -1);
+			if (scc_info &&
+			    scc_info->state_loops_possible &&
+			    scc_info->scc_epoch > sl->state.scc_epoch) {
+				if (env->log.level & BPF_LOG_LEVEL2)
+					verbose(env, "mark_all_regs_read_and_precise\n");
+				mark_all_regs_read_and_precise(env, &sl->state);
+				break;
+			}
+		}
 		clean_verifier_state(env, &sl->state);
 	}
 }
@@ -19044,13 +19047,27 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)
 
 	clean_live_states(env, insn_idx, cur);
 
+	if (env->log.level & BPF_LOG_LEVEL2) {
+		verbose(env, "is_state_visited: cur_state: ");
+		for (i = cur->curframe; i >= 0; i--)
+			print_verifier_state(env, cur, i, true);
+	}
+
 	head = explored_state(env, insn_idx);
 	list_for_each_safe(pos, tmp, head) {
 		sl = container_of(pos, struct bpf_verifier_state_list, node);
 		states_cnt++;
 		if (sl->state.insn_idx != insn_idx)
 			continue;
 
+		if (env->log.level & BPF_LOG_LEVEL2) {
+			verbose(env, "is_state_visited: candidate: ");
+			if (!same_callsites(cur, &sl->state))
+				continue;
+			for (i = cur->curframe; i >= 0; i--)
+				print_verifier_state(env, &sl->state, i, true);
+		}
+
 		if (sl->state.branches) {
 			struct bpf_func_state *frame = sl->state.frame[sl->state.curframe];
 
@@ -19275,8 +19292,11 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)
 	}
 	new->insn_idx = insn_idx;
 	scc_info = insn_scc(env, insn_idx);
-	if (scc_info)
+	if (scc_info) {
 		new->scc_epoch = scc_info->scc_epoch;
+		if (!scc_info->entry_state)
+			scc_info->entry_state = new;
+	}
 	WARN_ONCE(new->branches != 1,
 		  "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx);
 
@@ -19285,7 +19305,6 @@ static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)
 	cur->insn_hist_start = cur->insn_hist_end;
 	cur->dfs_depth = new->dfs_depth + 1;
 	list_add(&new_sl->node, head);
-	parent_scc_enter(env, env->cur_state);
 
 	/* connect new state to parentage chain. Current frame needs all
 	 * registers connected. Only r6 - r9 of the callers are alive (pushed