forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-05-10 cde9070d9970eef1f7ec2360586c802a16230ad8 
 kernel/drivers/gpu/drm/i915/i915_cmd_parser.c
....@@ -25,8 +25,10 @@
2525  *
2626  */
2727 
28
+#include "gt/intel_engine.h"
29
+
2830 #include "i915_drv.h"
29
-#include "intel_ringbuffer.h"
31
+#include "i915_memcpy.h"
3032 
3133 /**
3234  * DOC: batch buffer command parser
....@@ -233,7 +235,7 @@
233235 	/*
234236 	 * MI_BATCH_BUFFER_START requires some special handling. It's not
235237 	 * really a 'skip' action but it doesn't seem like it's worth adding
236
-	 * a new action. See i915_parse_cmds().
238
+	 * a new action. See intel_engine_cmd_parser().
237239 	 */
238240 	CMD(  MI_BATCH_BUFFER_START,            SMI,   !F,  0xFF,   S  ),
239241 };
....@@ -733,7 +735,7 @@
733735 			return 0xFF;
734736 	}
735737 
736
-	DRM_DEBUG_DRIVER("CMD: Abnormal rcs cmd length! 0x%08X\n", cmd_header);
738
+	DRM_DEBUG("CMD: Abnormal rcs cmd length! 0x%08X\n", cmd_header);
737739 	return 0;
738740 }
739741 
....@@ -756,7 +758,7 @@
756758 			return 0xFF;
757759 	}
758760 
759
-	DRM_DEBUG_DRIVER("CMD: Abnormal bsd cmd length! 0x%08X\n", cmd_header);
761
+	DRM_DEBUG("CMD: Abnormal bsd cmd length! 0x%08X\n", cmd_header);
760762 	return 0;
761763 }
762764 
....@@ -769,7 +771,7 @@
769771 	else if (client == INSTR_BC_CLIENT)
770772 		return 0xFF;
771773 
772
-	DRM_DEBUG_DRIVER("CMD: Abnormal blt cmd length! 0x%08X\n", cmd_header);
774
+	DRM_DEBUG("CMD: Abnormal blt cmd length! 0x%08X\n", cmd_header);
773775 	return 0;
774776 }
775777 
....@@ -780,7 +782,7 @@
780782 	if (client == INSTR_MI_CLIENT || client == INSTR_BC_CLIENT)
781783 		return 0xFF;
782784 
783
-	DRM_DEBUG_DRIVER("CMD: Abnormal blt cmd length! 0x%08X\n", cmd_header);
785
+	DRM_DEBUG("CMD: Abnormal blt cmd length! 0x%08X\n", cmd_header);
784786 	return 0;
785787 }
786788 
....@@ -805,10 +807,11 @@
805807 			u32 curr = desc->cmd.value & desc->cmd.mask;
806808 
807809 			if (curr < previous) {
808
-				DRM_ERROR("CMD: %s [%d] command table not sorted: "
809
-					  "table=%d entry=%d cmd=0x%08X prev=0x%08X\n",
810
-					  engine->name, engine->id,
811
-					  i, j, curr, previous);
810
+				drm_err(&engine->i915->drm,
811
+					"CMD: %s [%d] command table not sorted: "
812
+					"table=%d entry=%d cmd=0x%08X prev=0x%08X\n",
813
+					engine->name, engine->id,
814
+					i, j, curr, previous);
812815 				ret = false;
813816 			}
814817 
....@@ -831,10 +834,11 @@
831834 		u32 curr = i915_mmio_reg_offset(reg_table[i].addr);
832835 
833836 		if (curr < previous) {
834
-			DRM_ERROR("CMD: %s [%d] register table not sorted: "
835
-				  "entry=%d reg=0x%08X prev=0x%08X\n",
836
-				  engine->name, engine->id,
837
-				  i, curr, previous);
837
+			drm_err(&engine->i915->drm,
838
+				"CMD: %s [%d] register table not sorted: "
839
+				"entry=%d reg=0x%08X prev=0x%08X\n",
840
+				engine->name, engine->id,
841
+				i, curr, previous);
838842 			ret = false;
839843 		}
840844 
....@@ -935,18 +939,18 @@
935939  * struct intel_engine_cs based on whether the platform requires software
936940  * command parsing.
937941  */
938
-void intel_engine_init_cmd_parser(struct intel_engine_cs *engine)
942
+int intel_engine_init_cmd_parser(struct intel_engine_cs *engine)
939943 {
940944 	const struct drm_i915_cmd_table *cmd_tables;
941945 	int cmd_table_count;
942946 	int ret;
943947 
944
-	if (!IS_GEN7(engine->i915) && !(IS_GEN9(engine->i915) &&
945
-					engine->id == BCS))
946
-		return;
948
+	if (!IS_GEN(engine->i915, 7) && !(IS_GEN(engine->i915, 9) &&
949
+					  engine->class == COPY_ENGINE_CLASS))
950
+		return 0;
947951 
948
-	switch (engine->id) {
949
-	case RCS:
952
+	switch (engine->class) {
953
+	case RENDER_CLASS:
950954 		if (IS_HASWELL(engine->i915)) {
951955 			cmd_tables = hsw_render_ring_cmd_table;
952956 			cmd_table_count =
....@@ -965,14 +969,14 @@
965969 		}
966970 		engine->get_cmd_length_mask = gen7_render_get_cmd_length_mask;
967971 		break;
968
-	case VCS:
972
+	case VIDEO_DECODE_CLASS:
969973 		cmd_tables = gen7_video_cmd_table;
970974 		cmd_table_count = ARRAY_SIZE(gen7_video_cmd_table);
971975 		engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;
972976 		break;
973
-	case BCS:
977
+	case COPY_ENGINE_CLASS:
974978 		engine->get_cmd_length_mask = gen7_blt_get_cmd_length_mask;
975
-		if (IS_GEN9(engine->i915)) {
979
+		if (IS_GEN(engine->i915, 9)) {
976980 			cmd_tables = gen9_blt_cmd_table;
977981 			cmd_table_count = ARRAY_SIZE(gen9_blt_cmd_table);
978982 			engine->get_cmd_length_mask =
....@@ -988,7 +992,7 @@
988992 			cmd_table_count = ARRAY_SIZE(gen7_blt_cmd_table);
989993 		}
990994 
991
-		if (IS_GEN9(engine->i915)) {
995
+		if (IS_GEN(engine->i915, 9)) {
992996 			engine->reg_tables = gen9_blt_reg_tables;
993997 			engine->reg_table_count =
994998 				ARRAY_SIZE(gen9_blt_reg_tables);
....@@ -1000,35 +1004,45 @@
10001004 			engine->reg_table_count = ARRAY_SIZE(ivb_blt_reg_tables);
10011005 		}
10021006 		break;
1003
-	case VECS:
1007
+	case VIDEO_ENHANCEMENT_CLASS:
10041008 		cmd_tables = hsw_vebox_cmd_table;
10051009 		cmd_table_count = ARRAY_SIZE(hsw_vebox_cmd_table);
10061010 		/* VECS can use the same length_mask function as VCS */
10071011 		engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;
10081012 		break;
10091013 	default:
1010
-		MISSING_CASE(engine->id);
1011
-		return;
1014
+		MISSING_CASE(engine->class);
1015
+		goto out;
10121016 	}
10131017 
10141018 	if (!validate_cmds_sorted(engine, cmd_tables, cmd_table_count)) {
1015
-		DRM_ERROR("%s: command descriptions are not sorted\n",
1016
-			  engine->name);
1017
-		return;
1019
+		drm_err(&engine->i915->drm,
1020
+			"%s: command descriptions are not sorted\n",
1021
+			engine->name);
1022
+		goto out;
10181023 	}
10191024 	if (!validate_regs_sorted(engine)) {
1020
-		DRM_ERROR("%s: registers are not sorted\n", engine->name);
1021
-		return;
1025
+		drm_err(&engine->i915->drm,
1026
+			"%s: registers are not sorted\n", engine->name);
1027
+		goto out;
10221028 	}
10231029 
10241030 	ret = init_hash_table(engine, cmd_tables, cmd_table_count);
10251031 	if (ret) {
1026
-		DRM_ERROR("%s: initialised failed!\n", engine->name);
1032
+		drm_err(&engine->i915->drm,
1033
+			"%s: initialised failed!\n", engine->name);
10271034 		fini_hash_table(engine);
1028
-		return;
1035
+		goto out;
10291036 	}
10301037 
10311038 	engine->flags |= I915_ENGINE_USING_CMD_PARSER;
1039
+
1040
+out:
1041
+	if (intel_engine_requires_cmd_parser(engine) &&
1042
+	    !intel_engine_using_cmd_parser(engine))
1043
+		return -EINVAL;
1044
+
1045
+	return 0;
10321046 }
10331047 
10341048 /**
....@@ -1129,81 +1143,83 @@
11291143 /* Returns a vmap'd pointer to dst_obj, which the caller must unmap */
11301144 static u32 *copy_batch(struct drm_i915_gem_object *dst_obj,
11311145 		       struct drm_i915_gem_object *src_obj,
1132
-		       u32 batch_start_offset,
1133
-		       u32 batch_len,
1134
-		       bool *needs_clflush_after)
1146
+		       u32 offset, u32 length)
11351147 {
11361148 	unsigned int src_needs_clflush;
11371149 	unsigned int dst_needs_clflush;
11381150 	void *dst, *src;
11391151 	int ret;
11401152 
1141
-	ret = i915_gem_obj_prepare_shmem_read(src_obj, &src_needs_clflush);
1153
+	ret = i915_gem_object_prepare_write(dst_obj, &dst_needs_clflush);
11421154 	if (ret)
11431155 		return ERR_PTR(ret);
11441156 
1145
-	ret = i915_gem_obj_prepare_shmem_write(dst_obj, &dst_needs_clflush);
1157
+	dst = i915_gem_object_pin_map(dst_obj, I915_MAP_FORCE_WB);
1158
+	i915_gem_object_finish_access(dst_obj);
1159
+	if (IS_ERR(dst))
1160
+		return dst;
1161
+
1162
+	ret = i915_gem_object_prepare_read(src_obj, &src_needs_clflush);
11461163 	if (ret) {
1147
-		dst = ERR_PTR(ret);
1148
-		goto unpin_src;
1164
+		i915_gem_object_unpin_map(dst_obj);
1165
+		return ERR_PTR(ret);
11491166 	}
11501167 
1151
-	dst = i915_gem_object_pin_map(dst_obj, I915_MAP_FORCE_WB);
1152
-	if (IS_ERR(dst))
1153
-		goto unpin_dst;
1154
-
11551168 	src = ERR_PTR(-ENODEV);
1156
-	if (src_needs_clflush &&
1157
-	    i915_can_memcpy_from_wc(NULL, batch_start_offset, 0)) {
1169
+	if (src_needs_clflush && i915_has_memcpy_from_wc()) {
11581170 		src = i915_gem_object_pin_map(src_obj, I915_MAP_WC);
11591171 		if (!IS_ERR(src)) {
1160
-			i915_memcpy_from_wc(dst,
1161
-					    src + batch_start_offset,
1162
-					    ALIGN(batch_len, 16));
1172
+			i915_unaligned_memcpy_from_wc(dst,
1173
+						      src + offset,
1174
+						      length);
11631175 			i915_gem_object_unpin_map(src_obj);
11641176 		}
11651177 	}
11661178 	if (IS_ERR(src)) {
1179
+		unsigned long x, n, remain;
11671180 		void *ptr;
1168
-		int offset, n;
11691181 
1170
-		offset = offset_in_page(batch_start_offset);
1171
-
1172
-		/* We can avoid clflushing partial cachelines before the write
1182
+		/*
1183
+		 * We can avoid clflushing partial cachelines before the write
11731184 		 * if we only every write full cache-lines. Since we know that
11741185 		 * both the source and destination are in multiples of
11751186 		 * PAGE_SIZE, we can simply round up to the next cacheline.
11761187 		 * We don't care about copying too much here as we only
11771188 		 * validate up to the end of the batch.
11781189 		 */
1190
+		remain = length;
11791191 		if (dst_needs_clflush & CLFLUSH_BEFORE)
1180
-			batch_len = roundup(batch_len,
1181
-					    boot_cpu_data.x86_clflush_size);
1192
+			remain = round_up(remain,
1193
+					  boot_cpu_data.x86_clflush_size);
11821194 
11831195 		ptr = dst;
1184
-		for (n = batch_start_offset >> PAGE_SHIFT; batch_len; n++) {
1185
-			int len = min_t(int, batch_len, PAGE_SIZE - offset);
1196
+		x = offset_in_page(offset);
1197
+		for (n = offset >> PAGE_SHIFT; remain; n++) {
1198
+			int len = min(remain, PAGE_SIZE - x);
11861199 
11871200 			src = kmap_atomic(i915_gem_object_get_page(src_obj, n));
11881201 			if (src_needs_clflush)
1189
-				drm_clflush_virt_range(src + offset, len);
1190
-			memcpy(ptr, src + offset, len);
1202
+				drm_clflush_virt_range(src + x, len);
1203
+			memcpy(ptr, src + x, len);
11911204 			kunmap_atomic(src);
11921205 
11931206 			ptr += len;
1194
-			batch_len -= len;
1195
-			offset = 0;
1207
+			remain -= len;
1208
+			x = 0;
11961209 		}
11971210 	}
11981211 
1199
-	/* dst_obj is returned with vmap pinned */
1200
-	*needs_clflush_after = dst_needs_clflush & CLFLUSH_AFTER;
1212
+	i915_gem_object_finish_access(src_obj);
12011213 
1202
-unpin_dst:
1203
-	i915_gem_obj_finish_shmem_access(dst_obj);
1204
-unpin_src:
1205
-	i915_gem_obj_finish_shmem_access(src_obj);
1214
+	memset32(dst + length, 0, (dst_obj->base.size - length) / sizeof(u32));
1215
+
12061216 	return dst;
1217
+}
1218
+
1219
+static inline bool cmd_desc_is(const struct drm_i915_cmd_descriptor * const desc,
1220
+			       const u32 cmd)
1221
+{
1222
+	return desc->cmd.value == (cmd & desc->cmd.mask);
12071223 }
12081224 
12091225 static bool check_cmd(const struct intel_engine_cs *engine,
....@@ -1214,7 +1230,7 @@
12141230 		return true;
12151231 
12161232 	if (desc->flags & CMD_DESC_REJECT) {
1217
-		DRM_DEBUG_DRIVER("CMD: Rejected command: 0x%08X\n", *cmd);
1233
+		DRM_DEBUG("CMD: Rejected command: 0x%08X\n", *cmd);
12181234 		return false;
12191235 	}
12201236 
....@@ -1234,8 +1250,8 @@
12341250 				find_reg(engine, reg_addr);
12351251 
12361252 			if (!reg) {
1237
-				DRM_DEBUG_DRIVER("CMD: Rejected register 0x%08X in command: 0x%08X (%s)\n",
1238
-						 reg_addr, *cmd, engine->name);
1253
+				DRM_DEBUG("CMD: Rejected register 0x%08X in command: 0x%08X (%s)\n",
1254
+					  reg_addr, *cmd, engine->name);
12391255 				return false;
12401256 			}
12411257 
....@@ -1244,23 +1260,23 @@
12441260 			 * allowed mask/value pair given in the whitelist entry.
12451261 			 */
12461262 			if (reg->mask) {
1247
-				if (desc->cmd.value == MI_LOAD_REGISTER_MEM) {
1248
-					DRM_DEBUG_DRIVER("CMD: Rejected LRM to masked register 0x%08X\n",
1249
-							 reg_addr);
1263
+				if (cmd_desc_is(desc, MI_LOAD_REGISTER_MEM)) {
1264
+					DRM_DEBUG("CMD: Rejected LRM to masked register 0x%08X\n",
1265
+						  reg_addr);
12501266 					return false;
12511267 				}
12521268 
1253
-				if (desc->cmd.value == MI_LOAD_REGISTER_REG) {
1254
-					DRM_DEBUG_DRIVER("CMD: Rejected LRR to masked register 0x%08X\n",
1255
-							 reg_addr);
1269
+				if (cmd_desc_is(desc, MI_LOAD_REGISTER_REG)) {
1270
+					DRM_DEBUG("CMD: Rejected LRR to masked register 0x%08X\n",
1271
+						  reg_addr);
12561272 					return false;
12571273 				}
12581274 
1259
-				if (desc->cmd.value == MI_LOAD_REGISTER_IMM(1) &&
1275
+				if (cmd_desc_is(desc, MI_LOAD_REGISTER_IMM(1)) &&
12601276 				    (offset + 2 > length ||
12611277 				     (cmd[offset + 1] & reg->mask) != reg->value)) {
1262
-					DRM_DEBUG_DRIVER("CMD: Rejected LRI to masked register 0x%08X\n",
1263
-							 reg_addr);
1278
+					DRM_DEBUG("CMD: Rejected LRI to masked register 0x%08X\n",
1279
+						  reg_addr);
12641280 					return false;
12651281 				}
12661282 			}
....@@ -1287,8 +1303,8 @@
12871303 			}
12881304 
12891305 			if (desc->bits[i].offset >= length) {
1290
-				DRM_DEBUG_DRIVER("CMD: Rejected command 0x%08X, too short to check bitmask (%s)\n",
1291
-						 *cmd, engine->name);
1306
+				DRM_DEBUG("CMD: Rejected command 0x%08X, too short to check bitmask (%s)\n",
1307
+					  *cmd, engine->name);
12921308 				return false;
12931309 			}
12941310 
....@@ -1296,11 +1312,11 @@
12961312 				desc->bits[i].mask;
12971313 
12981314 			if (dword != desc->bits[i].expected) {
1299
-				DRM_DEBUG_DRIVER("CMD: Rejected command 0x%08X for bitmask 0x%08X (exp=0x%08X act=0x%08X) (%s)\n",
1300
-						 *cmd,
1301
-						 desc->bits[i].mask,
1302
-						 desc->bits[i].expected,
1303
-						 dword, engine->name);
1315
+				DRM_DEBUG("CMD: Rejected command 0x%08X for bitmask 0x%08X (exp=0x%08X act=0x%08X) (%s)\n",
1316
+					  *cmd,
1317
+					  desc->bits[i].mask,
1318
+					  desc->bits[i].expected,
1319
+					  dword, engine->name);
13041320 				return false;
13051321 			}
13061322 		}
....@@ -1309,17 +1325,17 @@
13091325 	return true;
13101326 }
13111327 
1312
-static int check_bbstart(const struct i915_gem_context *ctx,
1313
-			 u32 *cmd, u32 offset, u32 length,
1314
-			 u32 batch_len,
1315
-			 u64 batch_start,
1316
-			 u64 shadow_batch_start)
1328
+static int check_bbstart(u32 *cmd, u32 offset, u32 length,
1329
+			 u32 batch_length,
1330
+			 u64 batch_addr,
1331
+			 u64 shadow_addr,
1332
+			 const unsigned long *jump_whitelist)
13171333 {
13181334 	u64 jump_offset, jump_target;
13191335 	u32 target_cmd_offset, target_cmd_index;
13201336 
13211337 	/* For igt compatibility on older platforms */
1322
-	if (CMDPARSER_USES_GGTT(ctx->i915)) {
1338
+	if (!jump_whitelist) {
13231339 		DRM_DEBUG("CMD: Rejecting BB_START for ggtt based submission\n");
13241340 		return -EACCES;
13251341 	}
....@@ -1330,14 +1346,14 @@
13301346 		return -EINVAL;
13311347 	}
13321348 
1333
-	jump_target = *(u64*)(cmd+1);
1334
-	jump_offset = jump_target - batch_start;
1349
+	jump_target = *(u64 *)(cmd + 1);
1350
+	jump_offset = jump_target - batch_addr;
13351351 
13361352 	/*
13371353 	 * Any underflow of jump_target is guaranteed to be outside the range
13381354 	 * of a u32, so >= test catches both too large and too small
13391355 	 */
1340
-	if (jump_offset >= batch_len) {
1356
+	if (jump_offset >= batch_length) {
13411357 		DRM_DEBUG("CMD: BB_START to 0x%llx jumps out of BB\n",
13421358 			  jump_target);
13431359 		return -EINVAL;
....@@ -1345,20 +1361,20 @@
13451361 
13461362 	/*
13471363 	 * This cannot overflow a u32 because we already checked jump_offset
1348
-	 * is within the BB, and the batch_len is a u32
1364
+	 * is within the BB, and the batch_length is a u32
13491365 	 */
13501366 	target_cmd_offset = lower_32_bits(jump_offset);
13511367 	target_cmd_index = target_cmd_offset / sizeof(u32);
13521368 
1353
-	*(u64*)(cmd + 1) = shadow_batch_start + target_cmd_offset;
1369
+	*(u64 *)(cmd + 1) = shadow_addr + target_cmd_offset;
13541370 
13551371 	if (target_cmd_index == offset)
13561372 		return 0;
13571373 
1358
-	if (ctx->jump_whitelist_cmds <= target_cmd_index) {
1359
-		DRM_DEBUG("CMD: Rejecting BB_START - truncated whitelist array\n");
1360
-		return -EINVAL;
1361
-	} else if (!test_bit(target_cmd_index, ctx->jump_whitelist)) {
1374
+	if (IS_ERR(jump_whitelist))
1375
+		return PTR_ERR(jump_whitelist);
1376
+
1377
+	if (!test_bit(target_cmd_index, jump_whitelist)) {
13621378 		DRM_DEBUG("CMD: BB_START to 0x%llx not a previously executed cmd\n",
13631379 			  jump_target);
13641380 		return -EINVAL;
....@@ -1367,54 +1383,35 @@
13671383 	return 0;
13681384 }
13691385 
1370
-static void init_whitelist(struct i915_gem_context *ctx, u32 batch_len)
1386
+static unsigned long *alloc_whitelist(u32 batch_length)
13711387 {
1372
-	const u32 batch_cmds = DIV_ROUND_UP(batch_len, sizeof(u32));
1373
-	const u32 exact_size = BITS_TO_LONGS(batch_cmds);
1374
-	u32 next_size = BITS_TO_LONGS(roundup_pow_of_two(batch_cmds));
1375
-	unsigned long *next_whitelist;
1388
+	unsigned long *jmp;
13761389 
1377
-	if (CMDPARSER_USES_GGTT(ctx->i915))
1378
-		return;
1390
+	/*
1391
+	 * We expect batch_length to be less than 256KiB for known users,
1392
+	 * i.e. we need at most an 8KiB bitmap allocation which should be
1393
+	 * reasonably cheap due to kmalloc caches.
1394
+	 */
13791395 
1380
-	if (batch_cmds <= ctx->jump_whitelist_cmds) {
1381
-		bitmap_zero(ctx->jump_whitelist, batch_cmds);
1382
-		return;
1383
-	}
1396
+	/* Prefer to report transient allocation failure rather than hit oom */
1397
+	jmp = bitmap_zalloc(DIV_ROUND_UP(batch_length, sizeof(u32)),
1398
+			    GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
1399
+	if (!jmp)
1400
+		return ERR_PTR(-ENOMEM);
13841401 
1385
-again:
1386
-	next_whitelist = kcalloc(next_size, sizeof(long), GFP_KERNEL);
1387
-	if (next_whitelist) {
1388
-		kfree(ctx->jump_whitelist);
1389
-		ctx->jump_whitelist = next_whitelist;
1390
-		ctx->jump_whitelist_cmds =
1391
-			next_size * BITS_PER_BYTE * sizeof(long);
1392
-		return;
1393
-	}
1394
-
1395
-	if (next_size > exact_size) {
1396
-		next_size = exact_size;
1397
-		goto again;
1398
-	}
1399
-
1400
-	DRM_DEBUG("CMD: Failed to extend whitelist. BB_START may be disallowed\n");
1401
-	bitmap_zero(ctx->jump_whitelist, ctx->jump_whitelist_cmds);
1402
-
1403
-	return;
1402
+	return jmp;
14041403 }
14051404 
14061405 #define LENGTH_BIAS 2
14071406 
14081407 /**
1409
- * i915_parse_cmds() - parse a submitted batch buffer for privilege violations
1410
- * @ctx: the context in which the batch is to execute
1408
+ * intel_engine_cmd_parser() - parse a batch buffer for privilege violations
14111409  * @engine: the engine on which the batch is to execute
1412
- * @batch_obj: the batch buffer in question
1413
- * @batch_start: Canonical base address of batch
1414
- * @batch_start_offset: byte offset in the batch at which execution starts
1415
- * @batch_len: length of the commands in batch_obj
1416
- * @shadow_batch_obj: copy of the batch buffer in question
1417
- * @shadow_batch_start: Canonical base address of shadow_batch_obj
1410
+ * @batch: the batch buffer in question
1411
+ * @batch_offset: byte offset in the batch at which execution starts
1412
+ * @batch_length: length of the commands in batch_obj
1413
+ * @shadow: validated copy of the batch buffer in question
1414
+ * @trampoline: whether to emit a conditional trampoline at the end of the batch
14181415  *
14191416  * Parses the specified batch buffer looking for privilege violations as
14201417  * described in the overview.
....@@ -1423,37 +1420,47 @@
14231420  * if the batch appears legal but should use hardware parsing
14241421  */
14251422 
1426
-int intel_engine_cmd_parser(struct i915_gem_context *ctx,
1427
-			    struct intel_engine_cs *engine,
1428
-			    struct drm_i915_gem_object *batch_obj,
1429
-			    u64 batch_start,
1430
-			    u32 batch_start_offset,
1431
-			    u32 batch_len,
1432
-			    struct drm_i915_gem_object *shadow_batch_obj,
1433
-			    u64 shadow_batch_start)
1423
+int intel_engine_cmd_parser(struct intel_engine_cs *engine,
1424
+			    struct i915_vma *batch,
1425
+			    unsigned long batch_offset,
1426
+			    unsigned long batch_length,
1427
+			    struct i915_vma *shadow,
1428
+			    bool trampoline)
14341429 {
14351430 	u32 *cmd, *batch_end, offset = 0;
14361431 	struct drm_i915_cmd_descriptor default_desc = noop_desc;
14371432 	const struct drm_i915_cmd_descriptor *desc = &default_desc;
1438
-	bool needs_clflush_after = false;
1433
+	unsigned long *jump_whitelist;
1434
+	u64 batch_addr, shadow_addr;
14391435 	int ret = 0;
14401436 
1441
-	cmd = copy_batch(shadow_batch_obj, batch_obj,
1442
-			 batch_start_offset, batch_len,
1443
-			 &needs_clflush_after);
1437
+	GEM_BUG_ON(!IS_ALIGNED(batch_offset, sizeof(*cmd)));
1438
+	GEM_BUG_ON(!IS_ALIGNED(batch_length, sizeof(*cmd)));
1439
+	GEM_BUG_ON(range_overflows_t(u64, batch_offset, batch_length,
1440
+				     batch->size));
1441
+	GEM_BUG_ON(!batch_length);
1442
+
1443
+	cmd = copy_batch(shadow->obj, batch->obj,
1444
+			 batch_offset, batch_length);
14441445 	if (IS_ERR(cmd)) {
1445
-		DRM_DEBUG_DRIVER("CMD: Failed to copy batch\n");
1446
+		DRM_DEBUG("CMD: Failed to copy batch\n");
14461447 		return PTR_ERR(cmd);
14471448 	}
14481449 
1449
-	init_whitelist(ctx, batch_len);
1450
+	jump_whitelist = NULL;
1451
+	if (!trampoline)
1452
+		/* Defer failure until attempted use */
1453
+		jump_whitelist = alloc_whitelist(batch_length);
1454
+
1455
+	shadow_addr = gen8_canonical_addr(shadow->node.start);
1456
+	batch_addr = gen8_canonical_addr(batch->node.start + batch_offset);
14501457 
14511458 	/*
14521459 	 * We use the batch length as size because the shadow object is as
14531460 	 * large or larger and copy_batch() will write MI_NOPs to the extra
14541461 	 * space. Parsing should be faster in some cases this way.
14551462 	 */
1456
-	batch_end = cmd + (batch_len / sizeof(*batch_end));
1463
+	batch_end = cmd + batch_length / sizeof(*batch_end);
14571464 	do {
14581465 		u32 length;
14591466 
....@@ -1462,61 +1469,92 @@
14621469 
14631470 		desc = find_cmd(engine, *cmd, desc, &default_desc);
14641471 		if (!desc) {
1465
-			DRM_DEBUG_DRIVER("CMD: Unrecognized command: 0x%08X\n",
1466
-					 *cmd);
1472
+			DRM_DEBUG("CMD: Unrecognized command: 0x%08X\n", *cmd);
14671473 			ret = -EINVAL;
1468
-			goto err;
1474
+			break;
14691475 		}
14701476 
14711477 		if (desc->flags & CMD_DESC_FIXED)
14721478 			length = desc->length.fixed;
14731479 		else
1474
-			length = ((*cmd & desc->length.mask) + LENGTH_BIAS);
1480
+			length = (*cmd & desc->length.mask) + LENGTH_BIAS;
14751481 
14761482 		if ((batch_end - cmd) < length) {
1477
-			DRM_DEBUG_DRIVER("CMD: Command length exceeds batch length: 0x%08X length=%u batchlen=%td\n",
1478
-					 *cmd,
1479
-					 length,
1480
-					 batch_end - cmd);
1483
+			DRM_DEBUG("CMD: Command length exceeds batch length: 0x%08X length=%u batchlen=%td\n",
1484
+				  *cmd,
1485
+				  length,
1486
+				  batch_end - cmd);
14811487 			ret = -EINVAL;
1482
-			goto err;
1488
+			break;
14831489 		}
14841490 
14851491 		if (!check_cmd(engine, desc, cmd, length)) {
14861492 			ret = -EACCES;
1487
-			goto err;
1488
-		}
1489
-
1490
-		if (desc->cmd.value == MI_BATCH_BUFFER_START) {
1491
-			ret = check_bbstart(ctx, cmd, offset, length,
1492
-					    batch_len, batch_start,
1493
-					    shadow_batch_start);
1494
-
1495
-			if (ret)
1496
-				goto err;
14971493 			break;
14981494 		}
14991495 
1500
-		if (ctx->jump_whitelist_cmds > offset)
1501
-			set_bit(offset, ctx->jump_whitelist);
1496
+		if (cmd_desc_is(desc, MI_BATCH_BUFFER_START)) {
1497
+			ret = check_bbstart(cmd, offset, length, batch_length,
1498
+					    batch_addr, shadow_addr,
1499
+					    jump_whitelist);
1500
+			break;
1501
+		}
1502
+
1503
+		if (!IS_ERR_OR_NULL(jump_whitelist))
1504
+			__set_bit(offset, jump_whitelist);
15021505 
15031506 		cmd += length;
15041507 		offset += length;
15051508 		if  (cmd >= batch_end) {
1506
-			DRM_DEBUG_DRIVER("CMD: Got to the end of the buffer w/o a BBE cmd!\n");
1509
+			DRM_DEBUG("CMD: Got to the end of the buffer w/o a BBE cmd!\n");
15071510 			ret = -EINVAL;
1508
-			goto err;
1511
+			break;
15091512 		}
15101513 	} while (1);
15111514 
1512
-	if (needs_clflush_after) {
1513
-		void *ptr = page_mask_bits(shadow_batch_obj->mm.mapping);
1515
+	if (trampoline) {
1516
+		/*
1517
+		 * With the trampoline, the shadow is executed twice.
1518
+		 *
1519
+		 *   1 - starting at offset 0, in privileged mode
1520
+		 *   2 - starting at offset batch_len, as non-privileged
1521
+		 *
1522
+		 * Only if the batch is valid and safe to execute, do we
1523
+		 * allow the first privileged execution to proceed. If not,
1524
+		 * we terminate the first batch and use the second batchbuffer
1525
+		 * entry to chain to the original unsafe non-privileged batch,
1526
+		 * leaving it to the HW to validate.
1527
+		 */
1528
+		*batch_end = MI_BATCH_BUFFER_END;
15141529 
1515
-		drm_clflush_virt_range(ptr, (void *)(cmd + 1) - ptr);
1530
+		if (ret) {
1531
+			/* Batch unsafe to execute with privileges, cancel! */
1532
+			cmd = page_mask_bits(shadow->obj->mm.mapping);
1533
+			*cmd = MI_BATCH_BUFFER_END;
1534
+
1535
+			/* If batch is unsafe but valid, jump to the original */
1536
+			if (ret == -EACCES) {
1537
+				unsigned int flags;
1538
+
1539
+				flags = MI_BATCH_NON_SECURE_I965;
1540
+				if (IS_HASWELL(engine->i915))
1541
+					flags = MI_BATCH_NON_SECURE_HSW;
1542
+
1543
+				GEM_BUG_ON(!IS_GEN_RANGE(engine->i915, 6, 7));
1544
+				__gen6_emit_bb_start(batch_end,
1545
+						     batch_addr,
1546
+						     flags);
1547
+
1548
+				ret = 0; /* allow execution */
1549
+			}
1550
+		}
15161551 	}
15171552 
1518
-err:
1519
-	i915_gem_object_unpin_map(shadow_batch_obj);
1553
+	i915_gem_object_flush_map(shadow->obj);
1554
+
1555
+	if (!IS_ERR_OR_NULL(jump_whitelist))
1556
+		kfree(jump_whitelist);
1557
+	i915_gem_object_unpin_map(shadow->obj);
15201558 	return ret;
15211559 }
15221560 
....@@ -1532,11 +1570,10 @@
15321570 int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv)
15331571 {
15341572 	struct intel_engine_cs *engine;
1535
-	enum intel_engine_id id;
15361573 	bool active = false;
15371574 
15381575 	/* If the command parser is not enabled, report 0 - unsupported */
1539
-	for_each_engine(engine, dev_priv, id) {
1576
+	for_each_uabi_engine(engine, dev_priv) {
15401577 		if (intel_engine_using_cmd_parser(engine)) {
15411578 			active = true;
15421579 			break;