개념 설명 전체
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c · v6.18.37
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * IOMMU API for ARM architected SMMUv3 implementations.
4 *
5 * Copyright (C) 2015 ARM Limited
6 *
7 * Author: Will Deacon <[email protected]>
8 *
9 * This driver is powered by bad coffee and bombay mix.
10 */
11
12 #include <linux/acpi.h>
13 #include <linux/acpi_iort.h>
14 #include <linux/bitops.h>
15 #include <linux/crash_dump.h>
16 #include <linux/delay.h>
17 #include <linux/err.h>
18 #include <linux/interrupt.h>
19 #include <linux/io-pgtable.h>
20 #include <linux/iopoll.h>
21 #include <linux/module.h>
22 #include <linux/msi.h>
23 #include <linux/of.h>
24 #include <linux/of_address.h>
25 #include <linux/of_platform.h>
26 #include <linux/pci.h>
27 #include <linux/pci-ats.h>
28 #include <linux/platform_device.h>
29 #include <linux/string_choices.h>
30 #include <kunit/visibility.h>
31 #include <uapi/linux/iommufd.h>
32
33 #include "arm-smmu-v3.h"
34 #include "../../dma-iommu.h"
35
36 static bool disable_msipolling;
37 module_param(disable_msipolling, bool, 0444);
38 MODULE_PARM_DESC(disable_msipolling,
39 "Disable MSI-based polling for CMD_SYNC completion.");
40
41 static const struct iommu_ops arm_smmu_ops;
42 static struct iommu_dirty_ops arm_smmu_dirty_ops;
43
44 enum arm_smmu_msi_index {
45 EVTQ_MSI_INDEX,
46 GERROR_MSI_INDEX,
47 PRIQ_MSI_INDEX,
48 ARM_SMMU_MAX_MSIS,
49 };
50
51 #define NUM_ENTRY_QWORDS 8
52 static_assert(sizeof(struct arm_smmu_ste) == NUM_ENTRY_QWORDS * sizeof(u64));
53 static_assert(sizeof(struct arm_smmu_cd) == NUM_ENTRY_QWORDS * sizeof(u64));
54
55 static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
56 [EVTQ_MSI_INDEX] = {
57 ARM_SMMU_EVTQ_IRQ_CFG0,
58 ARM_SMMU_EVTQ_IRQ_CFG1,
59 ARM_SMMU_EVTQ_IRQ_CFG2,
60 },
61 [GERROR_MSI_INDEX] = {
62 ARM_SMMU_GERROR_IRQ_CFG0,
63 ARM_SMMU_GERROR_IRQ_CFG1,
64 ARM_SMMU_GERROR_IRQ_CFG2,
65 },
66 [PRIQ_MSI_INDEX] = {
67 ARM_SMMU_PRIQ_IRQ_CFG0,
68 ARM_SMMU_PRIQ_IRQ_CFG1,
69 ARM_SMMU_PRIQ_IRQ_CFG2,
70 },
71 };
72
73 struct arm_smmu_option_prop {
74 u32 opt;
75 const char *prop;
76 };
77
78 DEFINE_XARRAY_ALLOC1(arm_smmu_asid_xa);
79 DEFINE_MUTEX(arm_smmu_asid_lock);
80
81 static struct arm_smmu_option_prop arm_smmu_options[] = {
82 { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
83 { ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
84 { 0, NULL},
85 };
86
87 static const char * const event_str[] = {
88 [EVT_ID_BAD_STREAMID_CONFIG] = "C_BAD_STREAMID",
89 [EVT_ID_STE_FETCH_FAULT] = "F_STE_FETCH",
90 [EVT_ID_BAD_STE_CONFIG] = "C_BAD_STE",
91 [EVT_ID_STREAM_DISABLED_FAULT] = "F_STREAM_DISABLED",
92 [EVT_ID_BAD_SUBSTREAMID_CONFIG] = "C_BAD_SUBSTREAMID",
93 [EVT_ID_CD_FETCH_FAULT] = "F_CD_FETCH",
94 [EVT_ID_BAD_CD_CONFIG] = "C_BAD_CD",
95 [EVT_ID_TRANSLATION_FAULT] = "F_TRANSLATION",
96 [EVT_ID_ADDR_SIZE_FAULT] = "F_ADDR_SIZE",
97 [EVT_ID_ACCESS_FAULT] = "F_ACCESS",
98 [EVT_ID_PERMISSION_FAULT] = "F_PERMISSION",
99 [EVT_ID_VMS_FETCH_FAULT] = "F_VMS_FETCH",
100 };
101
102 static const char * const event_class_str[] = {
103 [0] = "CD fetch",
104 [1] = "Stage 1 translation table fetch",
105 [2] = "Input address caused fault",
106 [3] = "Reserved",
107 };
108
109 static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master);
110
111 static void parse_driver_options(struct arm_smmu_device *smmu)
112 {
113 int i = 0;
114
115 do {
116 if (of_property_read_bool(smmu->dev->of_node,
117 arm_smmu_options[i].prop)) {
118 smmu->options |= arm_smmu_options[i].opt;
119 dev_notice(smmu->dev, "option %s\n",
120 arm_smmu_options[i].prop);
121 }
122 } while (arm_smmu_options[++i].opt);
123 }
124
125 /* Low-level queue manipulation functions */
126 static bool queue_has_space(struct arm_smmu_ll_queue *q, u32 n)
127 {
128 u32 space, prod, cons;
129
130 prod = Q_IDX(q, q->prod);
131 cons = Q_IDX(q, q->cons);
132
133 if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))
134 space = (1 << q->max_n_shift) - (prod - cons);
135 else
136 space = cons - prod;
137
138 return space >= n;
139 }
140
141 static bool queue_full(struct arm_smmu_ll_queue *q)
142 {
143 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
144 Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
145 }
146
147 static bool queue_empty(struct arm_smmu_ll_queue *q)
148 {
149 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
150 Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
151 }
152
153 static bool queue_consumed(struct arm_smmu_ll_queue *q, u32 prod)
154 {
155 return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&
156 (Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||
157 ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&
158 (Q_IDX(q, q->cons) <= Q_IDX(q, prod)));
159 }
160
161 static void queue_sync_cons_out(struct arm_smmu_queue *q)
162 {
163 /*
164 * Ensure that all CPU accesses (reads and writes) to the queue
165 * are complete before we update the cons pointer.
166 */
167 __iomb();
168 writel_relaxed(q->llq.cons, q->cons_reg);
169 }
170
171 static void queue_inc_cons(struct arm_smmu_ll_queue *q)
172 {
173 u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
174 q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
175 }
176
177 static void queue_sync_cons_ovf(struct arm_smmu_queue *q)
178 {
179 struct arm_smmu_ll_queue *llq = &q->llq;
180
181 if (likely(Q_OVF(llq->prod) == Q_OVF(llq->cons)))
182 return;
183
184 llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |
185 Q_IDX(llq, llq->cons);
186 queue_sync_cons_out(q);
187 }
188
189 static int queue_sync_prod_in(struct arm_smmu_queue *q)
190 {
191 u32 prod;
192 int ret = 0;
193
194 /*
195 * We can't use the _relaxed() variant here, as we must prevent
196 * speculative reads of the queue before we have determined that
197 * prod has indeed moved.
198 */
199 prod = readl(q->prod_reg);
200
201 if (Q_OVF(prod) != Q_OVF(q->llq.prod))
202 ret = -EOVERFLOW;
203
204 q->llq.prod = prod;
205 return ret;
206 }
207
208 static u32 queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n)
209 {
210 u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;
211 return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
212 }
213
214 static void queue_poll_init(struct arm_smmu_device *smmu,
215 struct arm_smmu_queue_poll *qp)
216 {
217 qp->delay = 1;
218 qp->spin_cnt = 0;
219 qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
220 qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
221 }
222
223 static int queue_poll(struct arm_smmu_queue_poll *qp)
224 {
225 if (ktime_compare(ktime_get(), qp->timeout) > 0)
226 return -ETIMEDOUT;
227
228 if (qp->wfe) {
229 wfe();
230 } else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) {
231 cpu_relax();
232 } else {
233 udelay(qp->delay);
234 qp->delay *= 2;
235 qp->spin_cnt = 0;
236 }
237
238 return 0;
239 }
240
241 static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
242 {
243 int i;
244
245 for (i = 0; i < n_dwords; ++i)
246 *dst++ = cpu_to_le64(*src++);
247 }
248
249 static void queue_read(u64 *dst, __le64 *src, size_t n_dwords)
250 {
251 int i;
252
253 for (i = 0; i < n_dwords; ++i)
254 *dst++ = le64_to_cpu(*src++);
255 }
256
257 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
258 {
259 if (queue_empty(&q->llq))
260 return -EAGAIN;
261
262 queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords);
263 queue_inc_cons(&q->llq);
264 queue_sync_cons_out(q);
265 return 0;
266 }
267
268 /* High-level queue accessors */
269 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
270 {
271 memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT);
272 cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
273
274 switch (ent->opcode) {
275 case CMDQ_OP_TLBI_EL2_ALL:
276 case CMDQ_OP_TLBI_NSNH_ALL:
277 break;
278 case CMDQ_OP_PREFETCH_CFG:
279 cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid);
280 break;
281 case CMDQ_OP_CFGI_CD:
282 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid);
283 fallthrough;
284 case CMDQ_OP_CFGI_STE:
285 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
286 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
287 break;
288 case CMDQ_OP_CFGI_CD_ALL:
289 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
290 break;
291 case CMDQ_OP_CFGI_ALL:
292 /* Cover the entire SID range */
293 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
294 break;
295 case CMDQ_OP_TLBI_NH_VA:
296 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
297 fallthrough;
298 case CMDQ_OP_TLBI_EL2_VA:
299 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
300 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
301 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
302 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
303 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
304 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
305 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
306 break;
307 case CMDQ_OP_TLBI_S2_IPA:
308 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
309 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
310 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
311 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
312 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
313 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
314 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
315 break;
316 case CMDQ_OP_TLBI_NH_ASID:
317 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
318 fallthrough;
319 case CMDQ_OP_TLBI_NH_ALL:
320 case CMDQ_OP_TLBI_S12_VMALL:
321 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
322 break;
323 case CMDQ_OP_TLBI_EL2_ASID:
324 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
325 break;
326 case CMDQ_OP_ATC_INV:
327 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
328 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
329 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
330 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
331 cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
332 cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
333 break;
334 case CMDQ_OP_PRI_RESP:
335 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
336 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);
337 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid);
338 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid);
339 switch (ent->pri.resp) {
340 case PRI_RESP_DENY:
341 case PRI_RESP_FAIL:
342 case PRI_RESP_SUCC:
343 break;
344 default:
345 return -EINVAL;
346 }
347 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
348 break;
349 case CMDQ_OP_RESUME:
350 cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_SID, ent->resume.sid);
351 cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_RESP, ent->resume.resp);
352 cmd[1] |= FIELD_PREP(CMDQ_RESUME_1_STAG, ent->resume.stag);
353 break;
354 case CMDQ_OP_CMD_SYNC:
355 if (ent->sync.msiaddr) {
356 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
357 cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
358 } else {
359 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV);
360 }
361 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH);
362 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
363 break;
364 default:
365 return -ENOENT;
366 }
367
368 return 0;
369 }
370
371 static struct arm_smmu_cmdq *arm_smmu_get_cmdq(struct arm_smmu_device *smmu,
372 struct arm_smmu_cmdq_ent *ent)
373 {
374 struct arm_smmu_cmdq *cmdq = NULL;
375
376 if (smmu->impl_ops && smmu->impl_ops->get_secondary_cmdq)
377 cmdq = smmu->impl_ops->get_secondary_cmdq(smmu, ent);
378
379 return cmdq ?: &smmu->cmdq;
380 }
381
382 static bool arm_smmu_cmdq_needs_busy_polling(struct arm_smmu_device *smmu,
383 struct arm_smmu_cmdq *cmdq)
384 {
385 if (cmdq == &smmu->cmdq)
386 return false;
387
388 return smmu->options & ARM_SMMU_OPT_TEGRA241_CMDQV;
389 }
390
391 static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu,
392 struct arm_smmu_cmdq *cmdq, u32 prod)
393 {
394 struct arm_smmu_queue *q = &cmdq->q;
395 struct arm_smmu_cmdq_ent ent = {
396 .opcode = CMDQ_OP_CMD_SYNC,
397 };
398
399 /*
400 * Beware that Hi16xx adds an extra 32 bits of goodness to its MSI
401 * payload, so the write will zero the entire command on that platform.
402 */
403 if (smmu->options & ARM_SMMU_OPT_MSIPOLL) {
404 ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *
405 q->ent_dwords * 8;
406 }
407
408 arm_smmu_cmdq_build_cmd(cmd, &ent);
409 if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
410 u64p_replace_bits(cmd, CMDQ_SYNC_0_CS_NONE, CMDQ_SYNC_0_CS);
411 }
412
413 void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
414 struct arm_smmu_cmdq *cmdq)
415 {
416 static const char * const cerror_str[] = {
417 [CMDQ_ERR_CERROR_NONE_IDX] = "No error",
418 [CMDQ_ERR_CERROR_ILL_IDX] = "Illegal command",
419 [CMDQ_ERR_CERROR_ABT_IDX] = "Abort on command fetch",
420 [CMDQ_ERR_CERROR_ATC_INV_IDX] = "ATC invalidate timeout",
421 };
422 struct arm_smmu_queue *q = &cmdq->q;
423
424 int i;
425 u64 cmd[CMDQ_ENT_DWORDS];
426 u32 cons = readl_relaxed(q->cons_reg);
427 u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
428 struct arm_smmu_cmdq_ent cmd_sync = {
429 .opcode = CMDQ_OP_CMD_SYNC,
430 };
431
432 dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
433 idx < ARRAY_SIZE(cerror_str) ? cerror_str[idx] : "Unknown");
434
435 switch (idx) {
436 case CMDQ_ERR_CERROR_ABT_IDX:
437 dev_err(smmu->dev, "retrying command fetch\n");
438 return;
439 case CMDQ_ERR_CERROR_NONE_IDX:
440 return;
441 case CMDQ_ERR_CERROR_ATC_INV_IDX:
442 /*
443 * ATC Invalidation Completion timeout. CONS is still pointing
444 * at the CMD_SYNC. Attempt to complete other pending commands
445 * by repeating the CMD_SYNC, though we might well end up back
446 * here since the ATC invalidation may still be pending.
447 */
448 return;
449 case CMDQ_ERR_CERROR_ILL_IDX:
450 default:
451 break;
452 }
453
454 /*
455 * We may have concurrent producers, so we need to be careful
456 * not to touch any of the shadow cmdq state.
457 */
458 queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
459 dev_err(smmu->dev, "skipping command in error state:\n");
460 for (i = 0; i < ARRAY_SIZE(cmd); ++i)
461 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
462
463 /* Convert the erroneous command into a CMD_SYNC */
464 arm_smmu_cmdq_build_cmd(cmd, &cmd_sync);
465 if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
466 u64p_replace_bits(cmd, CMDQ_SYNC_0_CS_NONE, CMDQ_SYNC_0_CS);
467
468 queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
469 }
470
471 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
472 {
473 __arm_smmu_cmdq_skip_err(smmu, &smmu->cmdq);
474 }
475
476 /*
477 * Command queue locking.
478 * This is a form of bastardised rwlock with the following major changes:
479 *
480 * - The only LOCK routines are exclusive_trylock() and shared_lock().
481 * Neither have barrier semantics, and instead provide only a control
482 * dependency.
483 *
484 * - The UNLOCK routines are supplemented with shared_tryunlock(), which
485 * fails if the caller appears to be the last lock holder (yes, this is
486 * racy). All successful UNLOCK routines have RELEASE semantics.
487 */
488 static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq)
489 {
490 /*
491 * When held in exclusive state, the lock counter is set to INT_MIN
492 * so these increments won't hurt as the value will remain negative.
493 * The increment will also signal the exclusive locker that there are
494 * shared waiters.
495 */
496 if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0)
497 return;
498
499 /*
500 * Someone else is holding the lock in exclusive state, so wait
501 * for them to finish. Since we already incremented the lock counter,
502 * no exclusive lock can be acquired until we finish. We don't need
503 * the return value since we only care that the exclusive lock is
504 * released (i.e. the lock counter is non-negative).
505 * Once the exclusive locker releases the lock, the sign bit will
506 * be cleared and our increment will make the lock counter positive,
507 * allowing us to proceed.
508 */
509 atomic_cond_read_relaxed(&cmdq->lock, VAL > 0);
510 }
511
512 static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq)
513 {
514 (void)atomic_dec_return_release(&cmdq->lock);
515 }
516
517 static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq)
518 {
519 if (atomic_read(&cmdq->lock) == 1)
520 return false;
521
522 arm_smmu_cmdq_shared_unlock(cmdq);
523 return true;
524 }
525
526 #define arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags) \
527 ({ \
528 bool __ret; \
529 local_irq_save(flags); \
530 __ret = !atomic_cmpxchg_relaxed(&cmdq->lock, 0, INT_MIN); \
531 if (!__ret) \
532 local_irq_restore(flags); \
533 __ret; \
534 })
535
536 /*
537 * Only clear the sign bit when releasing the exclusive lock this will
538 * allow any shared_lock() waiters to proceed without the possibility
539 * of entering the exclusive lock in a tight loop.
540 */
541 #define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags) \
542 ({ \
543 atomic_fetch_andnot_release(INT_MIN, &cmdq->lock); \
544 local_irq_restore(flags); \
545 })
546
547
548 /*
549 * Command queue insertion.
550 * This is made fiddly by our attempts to achieve some sort of scalability
551 * since there is one queue shared amongst all of the CPUs in the system. If
552 * you like mixed-size concurrency, dependency ordering and relaxed atomics,
553 * then you'll *love* this monstrosity.
554 *
555 * The basic idea is to split the queue up into ranges of commands that are
556 * owned by a given CPU; the owner may not have written all of the commands
557 * itself, but is responsible for advancing the hardware prod pointer when
558 * the time comes. The algorithm is roughly:
559 *
560 * 1. Allocate some space in the queue. At this point we also discover
561 * whether the head of the queue is currently owned by another CPU,
562 * or whether we are the owner.
563 *
564 * 2. Write our commands into our allocated slots in the queue.
565 *
566 * 3. Mark our slots as valid in arm_smmu_cmdq.valid_map.
567 *
568 * 4. If we are an owner:
569 * a. Wait for the previous owner to finish.
570 * b. Mark the queue head as unowned, which tells us the range
571 * that we are responsible for publishing.
572 * c. Wait for all commands in our owned range to become valid.
573 * d. Advance the hardware prod pointer.
574 * e. Tell the next owner we've finished.
575 *
576 * 5. If we are inserting a CMD_SYNC (we may or may not have been an
577 * owner), then we need to stick around until it has completed:
578 * a. If we have MSIs, the SMMU can write back into the CMD_SYNC
579 * to clear the first 4 bytes.
580 * b. Otherwise, we spin waiting for the hardware cons pointer to
581 * advance past our command.
582 *
583 * The devil is in the details, particularly the use of locking for handling
584 * SYNC completion and freeing up space in the queue before we think that it is
585 * full.
586 */
587 static void __arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq *cmdq,
588 u32 sprod, u32 eprod, bool set)
589 {
590 u32 swidx, sbidx, ewidx, ebidx;
591 struct arm_smmu_ll_queue llq = {
592 .max_n_shift = cmdq->q.llq.max_n_shift,
593 .prod = sprod,
594 };
595
596 ewidx = BIT_WORD(Q_IDX(&llq, eprod));
597 ebidx = Q_IDX(&llq, eprod) % BITS_PER_LONG;
598
599 while (llq.prod != eprod) {
600 unsigned long mask;
601 atomic_long_t *ptr;
602 u32 limit = BITS_PER_LONG;
603
604 swidx = BIT_WORD(Q_IDX(&llq, llq.prod));
605 sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG;
606
607 ptr = &cmdq->valid_map[swidx];
608
609 if ((swidx == ewidx) && (sbidx < ebidx))
610 limit = ebidx;
611
612 mask = GENMASK(limit - 1, sbidx);
613
614 /*
615 * The valid bit is the inverse of the wrap bit. This means
616 * that a zero-initialised queue is invalid and, after marking
617 * all entries as valid, they become invalid again when we
618 * wrap.
619 */
620 if (set) {
621 atomic_long_xor(mask, ptr);
622 } else { /* Poll */
623 unsigned long valid;
624
625 valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask;
626 atomic_long_cond_read_relaxed(ptr, (VAL & mask) == valid);
627 }
628
629 llq.prod = queue_inc_prod_n(&llq, limit - sbidx);
630 }
631 }
632
633 /* Mark all entries in the range [sprod, eprod) as valid */
634 static void arm_smmu_cmdq_set_valid_map(struct arm_smmu_cmdq *cmdq,
635 u32 sprod, u32 eprod)
636 {
637 __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, true);
638 }
639
640 /* Wait for all entries in the range [sprod, eprod) to become valid */
641 static void arm_smmu_cmdq_poll_valid_map(struct arm_smmu_cmdq *cmdq,
642 u32 sprod, u32 eprod)
643 {
644 __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, false);
645 }
646
647 /* Wait for the command queue to become non-full */
648 static int arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device *smmu,
649 struct arm_smmu_cmdq *cmdq,
650 struct arm_smmu_ll_queue *llq)
651 {
652 unsigned long flags;
653 struct arm_smmu_queue_poll qp;
654 int ret = 0;
655
656 /*
657 * Try to update our copy of cons by grabbing exclusive cmdq access. If
658 * that fails, spin until somebody else updates it for us.
659 */
660 if (arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)) {
661 WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg));
662 arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags);
663 llq->val = READ_ONCE(cmdq->q.llq.val);
664 return 0;
665 }
666
667 queue_poll_init(smmu, &qp);
668 do {
669 llq->val = READ_ONCE(cmdq->q.llq.val);
670 if (!queue_full(llq))
671 break;
672
673 ret = queue_poll(&qp);
674 } while (!ret);
675
676 return ret;
677 }
678
679 /*
680 * Wait until the SMMU signals a CMD_SYNC completion MSI.
681 * Must be called with the cmdq lock held in some capacity.
682 */
683 static int __arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device *smmu,
684 struct arm_smmu_cmdq *cmdq,
685 struct arm_smmu_ll_queue *llq)
686 {
687 int ret = 0;
688 struct arm_smmu_queue_poll qp;
689 u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));
690
691 queue_poll_init(smmu, &qp);
692
693 /*
694 * The MSI won't generate an event, since it's being written back
695 * into the command queue.
696 */
697 qp.wfe = false;
698 smp_cond_load_relaxed(cmd, !VAL || (ret = queue_poll(&qp)));
699 llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1);
700 return ret;
701 }
702
703 /*
704 * Wait until the SMMU cons index passes llq->prod.
705 * Must be called with the cmdq lock held in some capacity.
706 */
707 static int __arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device *smmu,
708 struct arm_smmu_cmdq *cmdq,
709 struct arm_smmu_ll_queue *llq)
710 {
711 struct arm_smmu_queue_poll qp;
712 u32 prod = llq->prod;
713 int ret = 0;
714
715 queue_poll_init(smmu, &qp);
716 llq->val = READ_ONCE(cmdq->q.llq.val);
717 do {
718 if (queue_consumed(llq, prod))
719 break;
720
721 ret = queue_poll(&qp);
722
723 /*
724 * This needs to be a readl() so that our subsequent call
725 * to arm_smmu_cmdq_shared_tryunlock() can fail accurately.
726 *
727 * Specifically, we need to ensure that we observe all
728 * shared_lock()s by other CMD_SYNCs that share our owner,
729 * so that a failing call to tryunlock() means that we're
730 * the last one out and therefore we can safely advance
731 * cmdq->q.llq.cons. Roughly speaking:
732 *
733 * CPU 0 CPU1 CPU2 (us)
734 *
735 * if (sync)
736 * shared_lock();
737 *
738 * dma_wmb();
739 * set_valid_map();
740 *
741 * if (owner) {
742 * poll_valid_map();
743 * <control dependency>
744 * writel(prod_reg);
745 *
746 * readl(cons_reg);
747 * tryunlock();
748 *
749 * Requires us to see CPU 0's shared_lock() acquisition.
750 */
751 llq->cons = readl(cmdq->q.cons_reg);
752 } while (!ret);
753
754 return ret;
755 }
756
757 static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,
758 struct arm_smmu_cmdq *cmdq,
759 struct arm_smmu_ll_queue *llq)
760 {
761 if (smmu->options & ARM_SMMU_OPT_MSIPOLL &&
762 !arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
763 return __arm_smmu_cmdq_poll_until_msi(smmu, cmdq, llq);
764
765 return __arm_smmu_cmdq_poll_until_consumed(smmu, cmdq, llq);
766 }
767
768 static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds,
769 u32 prod, int n)
770 {
771 int i;
772 struct arm_smmu_ll_queue llq = {
773 .max_n_shift = cmdq->q.llq.max_n_shift,
774 .prod = prod,
775 };
776
777 for (i = 0; i < n; ++i) {
778 u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS];
779
780 prod = queue_inc_prod_n(&llq, i);
781 queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);
782 }
783 }
784
785 /*
786 * This is the actual insertion function, and provides the following
787 * ordering guarantees to callers:
788 *
789 * - There is a dma_wmb() before publishing any commands to the queue.
790 * This can be relied upon to order prior writes to data structures
791 * in memory (such as a CD or an STE) before the command.
792 *
793 * - On completion of a CMD_SYNC, there is a control dependency.
794 * This can be relied upon to order subsequent writes to memory (e.g.
795 * freeing an IOVA) after completion of the CMD_SYNC.
796 *
797 * - Command insertion is totally ordered, so if two CPUs each race to
798 * insert their own list of commands then all of the commands from one
799 * CPU will appear before any of the commands from the other CPU.
800 */
801 int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
802 struct arm_smmu_cmdq *cmdq, u64 *cmds, int n,
803 bool sync)
804 {
805 u64 cmd_sync[CMDQ_ENT_DWORDS];
806 u32 prod;
807 unsigned long flags;
808 bool owner;
809 struct arm_smmu_ll_queue llq, head;
810 int ret = 0;
811
812 llq.max_n_shift = cmdq->q.llq.max_n_shift;
813
814 /* 1. Allocate some space in the queue */
815 local_irq_save(flags);
816 llq.val = READ_ONCE(cmdq->q.llq.val);
817 do {
818 u64 old;
819
820 while (!queue_has_space(&llq, n + sync)) {
821 local_irq_restore(flags);
822 if (arm_smmu_cmdq_poll_until_not_full(smmu, cmdq, &llq))
823 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
824 local_irq_save(flags);
825 }
826
827 head.cons = llq.cons;
828 head.prod = queue_inc_prod_n(&llq, n + sync) |
829 CMDQ_PROD_OWNED_FLAG;
830
831 old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val);
832 if (old == llq.val)
833 break;
834
835 llq.val = old;
836 } while (1);
837 owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG);
838 head.prod &= ~CMDQ_PROD_OWNED_FLAG;
839 llq.prod &= ~CMDQ_PROD_OWNED_FLAG;
840
841 /*
842 * 2. Write our commands into the queue
843 * Dependency ordering from the cmpxchg() loop above.
844 */
845 arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);
846 if (sync) {
847 prod = queue_inc_prod_n(&llq, n);
848 arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, cmdq, prod);
849 queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);
850
851 /*
852 * In order to determine completion of our CMD_SYNC, we must
853 * ensure that the queue can't wrap twice without us noticing.
854 * We achieve that by taking the cmdq lock as shared before
855 * marking our slot as valid.
856 */
857 arm_smmu_cmdq_shared_lock(cmdq);
858 }
859
860 /* 3. Mark our slots as valid, ensuring commands are visible first */
861 dma_wmb();
862 arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod);
863
864 /* 4. If we are the owner, take control of the SMMU hardware */
865 if (owner) {
866 /* a. Wait for previous owner to finish */
867 atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod);
868
869 /* b. Stop gathering work by clearing the owned flag */
870 prod = atomic_fetch_andnot_relaxed(CMDQ_PROD_OWNED_FLAG,
871 &cmdq->q.llq.atomic.prod);
872 prod &= ~CMDQ_PROD_OWNED_FLAG;
873
874 /*
875 * c. Wait for any gathered work to be written to the queue.
876 * Note that we read our own entries so that we have the control
877 * dependency required by (d).
878 */
879 arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod);
880
881 /*
882 * d. Advance the hardware prod pointer
883 * Control dependency ordering from the entries becoming valid.
884 */
885 writel_relaxed(prod, cmdq->q.prod_reg);
886
887 /*
888 * e. Tell the next owner we're done
889 * Make sure we've updated the hardware first, so that we don't
890 * race to update prod and potentially move it backwards.
891 */
892 atomic_set_release(&cmdq->owner_prod, prod);
893 }
894
895 /* 5. If we are inserting a CMD_SYNC, we must wait for it to complete */
896 if (sync) {
897 llq.prod = queue_inc_prod_n(&llq, n);
898 ret = arm_smmu_cmdq_poll_until_sync(smmu, cmdq, &llq);
899 if (ret) {
900 dev_err_ratelimited(smmu->dev,
901 "CMD_SYNC timeout at 0x%08x [hwprod 0x%08x, hwcons 0x%08x]\n",
902 llq.prod,
903 readl_relaxed(cmdq->q.prod_reg),
904 readl_relaxed(cmdq->q.cons_reg));
905 }
906
907 /*
908 * Try to unlock the cmdq lock. This will fail if we're the last
909 * reader, in which case we can safely update cmdq->q.llq.cons
910 */
911 if (!arm_smmu_cmdq_shared_tryunlock(cmdq)) {
912 WRITE_ONCE(cmdq->q.llq.cons, llq.cons);
913 arm_smmu_cmdq_shared_unlock(cmdq);
914 }
915 }
916
917 local_irq_restore(flags);
918 return ret;
919 }
920
921 static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
922 struct arm_smmu_cmdq_ent *ent,
923 bool sync)
924 {
925 u64 cmd[CMDQ_ENT_DWORDS];
926
927 if (unlikely(arm_smmu_cmdq_build_cmd(cmd, ent))) {
928 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
929 ent->opcode);
930 return -EINVAL;
931 }
932
933 return arm_smmu_cmdq_issue_cmdlist(
934 smmu, arm_smmu_get_cmdq(smmu, ent), cmd, 1, sync);
935 }
936
937 static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
938 struct arm_smmu_cmdq_ent *ent)
939 {
940 return __arm_smmu_cmdq_issue_cmd(smmu, ent, false);
941 }
942
943 static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu,
944 struct arm_smmu_cmdq_ent *ent)
945 {
946 return __arm_smmu_cmdq_issue_cmd(smmu, ent, true);
947 }
948
949 static void arm_smmu_cmdq_batch_init(struct arm_smmu_device *smmu,
950 struct arm_smmu_cmdq_batch *cmds,
951 struct arm_smmu_cmdq_ent *ent)
952 {
953 cmds->num = 0;
954 cmds->cmdq = arm_smmu_get_cmdq(smmu, ent);
955 }
956
957 static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu,
958 struct arm_smmu_cmdq_batch *cmds,
959 struct arm_smmu_cmdq_ent *cmd)
960 {
961 bool unsupported_cmd = !arm_smmu_cmdq_supports_cmd(cmds->cmdq, cmd);
962 bool force_sync = (cmds->num == CMDQ_BATCH_ENTRIES - 1) &&
963 (smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC);
964 int index;
965
966 if (force_sync || unsupported_cmd) {
967 arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
968 cmds->num, true);
969 arm_smmu_cmdq_batch_init(smmu, cmds, cmd);
970 }
971
972 if (cmds->num == CMDQ_BATCH_ENTRIES) {
973 arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
974 cmds->num, false);
975 arm_smmu_cmdq_batch_init(smmu, cmds, cmd);
976 }
977
978 index = cmds->num * CMDQ_ENT_DWORDS;
979 if (unlikely(arm_smmu_cmdq_build_cmd(&cmds->cmds[index], cmd))) {
980 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
981 cmd->opcode);
982 return;
983 }
984
985 cmds->num++;
986 }
987
988 static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu,
989 struct arm_smmu_cmdq_batch *cmds)
990 {
991 return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
992 cmds->num, true);
993 }
994
995 static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused,
996 struct iommu_page_response *resp)
997 {
998 struct arm_smmu_cmdq_ent cmd = {0};
999 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
1000 int sid = master->streams[0].id;
1001
1002 if (WARN_ON(!master->stall_enabled))
1003 return;
1004
1005 cmd.opcode = CMDQ_OP_RESUME;
1006 cmd.resume.sid = sid;
1007 cmd.resume.stag = resp->grpid;
1008 switch (resp->code) {
1009 case IOMMU_PAGE_RESP_INVALID:
1010 case IOMMU_PAGE_RESP_FAILURE:
1011 cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT;
1012 break;
1013 case IOMMU_PAGE_RESP_SUCCESS:
1014 cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY;
1015 break;
1016 default:
1017 break;
1018 }
1019
1020 arm_smmu_cmdq_issue_cmd(master->smmu, &cmd);
1021 /*
1022 * Don't send a SYNC, it doesn't do anything for RESUME or PRI_RESP.
1023 * RESUME consumption guarantees that the stalled transaction will be
1024 * terminated... at some point in the future. PRI_RESP is fire and
1025 * forget.
1026 */
1027 }
1028
1029 /* Context descriptor manipulation functions */
1030 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid)
1031 {
1032 struct arm_smmu_cmdq_ent cmd = {
1033 .opcode = smmu->features & ARM_SMMU_FEAT_E2H ?
1034 CMDQ_OP_TLBI_EL2_ASID : CMDQ_OP_TLBI_NH_ASID,
1035 .tlbi.asid = asid,
1036 };
1037
1038 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
1039 }
1040
1041 /*
1042 * Based on the value of ent report which bits of the STE the HW will access. It
1043 * would be nice if this was complete according to the spec, but minimally it
1044 * has to capture the bits this driver uses.
1045 */
1046 VISIBLE_IF_KUNIT
1047 void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits)
1048 {
1049 unsigned int cfg = FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(ent[0]));
1050
1051 used_bits[0] = cpu_to_le64(STRTAB_STE_0_V);
1052 if (!(ent[0] & cpu_to_le64(STRTAB_STE_0_V)))
1053 return;
1054
1055 used_bits[0] |= cpu_to_le64(STRTAB_STE_0_CFG);
1056
1057 /* S1 translates */
1058 if (cfg & BIT(0)) {
1059 used_bits[0] |= cpu_to_le64(STRTAB_STE_0_S1FMT |
1060 STRTAB_STE_0_S1CTXPTR_MASK |
1061 STRTAB_STE_0_S1CDMAX);
1062 used_bits[1] |=
1063 cpu_to_le64(STRTAB_STE_1_S1DSS | STRTAB_STE_1_S1CIR |
1064 STRTAB_STE_1_S1COR | STRTAB_STE_1_S1CSH |
1065 STRTAB_STE_1_S1STALLD | STRTAB_STE_1_STRW |
1066 STRTAB_STE_1_EATS | STRTAB_STE_1_MEV);
1067 used_bits[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID);
1068
1069 /*
1070 * See 13.5 Summary of attribute/permission configuration fields
1071 * for the SHCFG behavior.
1072 */
1073 if (FIELD_GET(STRTAB_STE_1_S1DSS, le64_to_cpu(ent[1])) ==
1074 STRTAB_STE_1_S1DSS_BYPASS)
1075 used_bits[1] |= cpu_to_le64(STRTAB_STE_1_SHCFG);
1076 }
1077
1078 /* S2 translates */
1079 if (cfg & BIT(1)) {
1080 used_bits[1] |=
1081 cpu_to_le64(STRTAB_STE_1_S2FWB | STRTAB_STE_1_EATS |
1082 STRTAB_STE_1_SHCFG | STRTAB_STE_1_MEV);
1083 used_bits[2] |=
1084 cpu_to_le64(STRTAB_STE_2_S2VMID | STRTAB_STE_2_VTCR |
1085 STRTAB_STE_2_S2AA64 | STRTAB_STE_2_S2ENDI |
1086 STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2S |
1087 STRTAB_STE_2_S2R);
1088 used_bits[3] |= cpu_to_le64(STRTAB_STE_3_S2TTB_MASK);
1089 }
1090
1091 if (cfg == STRTAB_STE_0_CFG_BYPASS)
1092 used_bits[1] |= cpu_to_le64(STRTAB_STE_1_SHCFG);
1093 }
1094 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_used);
1095
1096 VISIBLE_IF_KUNIT
1097 void arm_smmu_get_ste_update_safe(const __le64 *cur, const __le64 *target,
1098 __le64 *safe_bits)
1099 {
1100 const __le64 eats_s1chk =
1101 FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_S1CHK);
1102 const __le64 eats_trans =
1103 FIELD_PREP(STRTAB_STE_1_EATS, STRTAB_STE_1_EATS_TRANS);
1104
1105 /*
1106 * When an STE changes EATS_TRANS, the sequencing code in the attach
1107 * logic already will have the PCI cap for ATS disabled. Thus at this
1108 * moment we can expect that the device will not generate ATS queries
1109 * and so we don't care about the sequencing of EATS. The purpose of
1110 * EATS_TRANS is to protect the system from hostile untrusted devices
1111 * that issue ATS when the PCI config space is disabled. However, if
1112 * EATS_TRANS is being changed, then we must have already trusted the
1113 * device as the EATS_TRANS security block is being disabled.
1114 *
1115 * Note: now the EATS_TRANS update is moved to the first entry_set().
1116 * Changing S2S and EATS might transiently result in S2S=1 and EATS=1
1117 * which is a bad STE (see "5.2 Stream Table Entry"). In such a case,
1118 * we can't do a hitless update. Also, it should not be added to the
1119 * safe bits with STRTAB_STE_1_EATS_S1CHK, because EATS=0b11 would be
1120 * effectively an errant 0b00 configuration.
1121 */
1122 if (!((cur[1] | target[1]) & cpu_to_le64(eats_s1chk)) &&
1123 !((cur[2] | target[2]) & cpu_to_le64(STRTAB_STE_2_S2S)))
1124 safe_bits[1] |= cpu_to_le64(eats_trans);
1125
1126 /*
1127 * MEV does not meaningfully impact the operation of the HW, it only
1128 * changes how many fault events are generated, thus we can relax it
1129 * when computing the ordering. The spec notes the device can act like
1130 * MEV=1 anyhow:
1131 *
1132 * Note: Software must expect, and be able to deal with, coalesced
1133 * fault records even when MEV == 0.
1134 */
1135 safe_bits[1] |= cpu_to_le64(STRTAB_STE_1_MEV);
1136 }
1137 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_update_safe);
1138
1139 /*
1140 * Figure out if we can do a hitless update of entry to become target. Returns a
1141 * bit mask where 1 indicates that qword needs to be set disruptively.
1142 * unused_update is an intermediate value of entry that has unused bits set to
1143 * their new values.
1144 */
1145 static u8 arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer *writer,
1146 const __le64 *entry, const __le64 *target,
1147 __le64 *unused_update)
1148 {
1149 __le64 target_used[NUM_ENTRY_QWORDS] = {};
1150 __le64 cur_used[NUM_ENTRY_QWORDS] = {};
1151 __le64 safe[NUM_ENTRY_QWORDS] = {};
1152 u8 used_qword_diff = 0;
1153 unsigned int i;
1154
1155 writer->ops->get_used(entry, cur_used);
1156 writer->ops->get_used(target, target_used);
1157 if (writer->ops->get_update_safe)
1158 writer->ops->get_update_safe(entry, target, safe);
1159
1160 for (i = 0; i != NUM_ENTRY_QWORDS; i++) {
1161 /*
1162 * Safe is only used for bits that are used by both entries,
1163 * otherwise it is sequenced according to the unused entry.
1164 */
1165 safe[i] &= target_used[i] & cur_used[i];
1166
1167 /*
1168 * Check that masks are up to date, the make functions are not
1169 * allowed to set a bit to 1 if the used function doesn't say it
1170 * is used.
1171 */
1172 WARN_ON_ONCE(target[i] & ~target_used[i]);
1173
1174 /* Bits can change because they are not currently being used */
1175 cur_used[i] &= ~safe[i];
1176 unused_update[i] = (entry[i] & cur_used[i]) |
1177 (target[i] & ~cur_used[i]);
1178 /*
1179 * Each bit indicates that a used bit in a qword needs to be
1180 * changed after unused_update is applied.
1181 */
1182 if ((unused_update[i] & target_used[i]) != target[i])
1183 used_qword_diff |= 1 << i;
1184 }
1185 return used_qword_diff;
1186 }
1187
1188 static void entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry,
1189 const __le64 *target, unsigned int start,
1190 unsigned int len)
1191 {
1192 bool changed = false;
1193 unsigned int i;
1194
1195 for (i = start; len != 0; len--, i++) {
1196 if (entry[i] != target[i]) {
1197 WRITE_ONCE(entry[i], target[i]);
1198 changed = true;
1199 }
1200 }
1201
1202 if (changed)
1203 writer->ops->sync(writer);
1204 }
1205
1206 /*
1207 * Update the STE/CD to the target configuration. The transition from the
1208 * current entry to the target entry takes place over multiple steps that
1209 * attempts to make the transition hitless if possible. This function takes care
1210 * not to create a situation where the HW can perceive a corrupted entry. HW is
1211 * only required to have a 64 bit atomicity with stores from the CPU, while
1212 * entries are many 64 bit values big.
1213 *
1214 * The difference between the current value and the target value is analyzed to
1215 * determine which of three updates are required - disruptive, hitless or no
1216 * change.
1217 *
1218 * In the most general disruptive case we can make any update in three steps:
1219 * - Disrupting the entry (V=0)
1220 * - Fill now unused qwords, execpt qword 0 which contains V
1221 * - Make qword 0 have the final value and valid (V=1) with a single 64
1222 * bit store
1223 *
1224 * However this disrupts the HW while it is happening. There are several
1225 * interesting cases where a STE/CD can be updated without disturbing the HW
1226 * because only a small number of bits are changing (S1DSS, CONFIG, etc) or
1227 * because the used bits don't intersect. We can detect this by calculating how
1228 * many 64 bit values need update after adjusting the unused bits and skip the
1229 * V=0 process. This relies on the IGNORED behavior described in the
1230 * specification.
1231 */
1232 VISIBLE_IF_KUNIT
1233 void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *entry,
1234 const __le64 *target)
1235 {
1236 __le64 unused_update[NUM_ENTRY_QWORDS];
1237 u8 used_qword_diff;
1238
1239 /*
1240 * Many of the entry structures have pointers to other structures that
1241 * need to have their updates be visible before any writes of the entry
1242 * happen.
1243 */
1244 dma_wmb();
1245
1246 used_qword_diff =
1247 arm_smmu_entry_qword_diff(writer, entry, target, unused_update);
1248 if (hweight8(used_qword_diff) == 1) {
1249 /*
1250 * Only one qword needs its used bits to be changed. This is a
1251 * hitless update, update all bits the current STE/CD is
1252 * ignoring to their new values, then update a single "critical
1253 * qword" to change the STE/CD and finally 0 out any bits that
1254 * are now unused in the target configuration.
1255 */
1256 unsigned int critical_qword_index = ffs(used_qword_diff) - 1;
1257
1258 /*
1259 * Skip writing unused bits in the critical qword since we'll be
1260 * writing it in the next step anyways. This can save a sync
1261 * when the only change is in that qword.
1262 */
1263 unused_update[critical_qword_index] =
1264 entry[critical_qword_index];
1265 entry_set(writer, entry, unused_update, 0, NUM_ENTRY_QWORDS);
1266 entry_set(writer, entry, target, critical_qword_index, 1);
1267 entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS);
1268 } else if (used_qword_diff) {
1269 /*
1270 * At least two qwords need their inuse bits to be changed. This
1271 * requires a breaking update, zero the V bit, write all qwords
1272 * but 0, then set qword 0
1273 */
1274 unused_update[0] = 0;
1275 entry_set(writer, entry, unused_update, 0, 1);
1276 entry_set(writer, entry, target, 1, NUM_ENTRY_QWORDS - 1);
1277 entry_set(writer, entry, target, 0, 1);
1278 } else {
1279 /*
1280 * No inuse bit changed, though safe bits may have changed.
1281 */
1282 entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS);
1283 }
1284 }
1285 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_write_entry);
1286
1287 static void arm_smmu_sync_cd(struct arm_smmu_master *master,
1288 int ssid, bool leaf)
1289 {
1290 size_t i;
1291 struct arm_smmu_cmdq_batch cmds;
1292 struct arm_smmu_device *smmu = master->smmu;
1293 struct arm_smmu_cmdq_ent cmd = {
1294 .opcode = CMDQ_OP_CFGI_CD,
1295 .cfgi = {
1296 .ssid = ssid,
1297 .leaf = leaf,
1298 },
1299 };
1300
1301 arm_smmu_cmdq_batch_init(smmu, &cmds, &cmd);
1302 for (i = 0; i < master->num_streams; i++) {
1303 cmd.cfgi.sid = master->streams[i].id;
1304 arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
1305 }
1306
1307 arm_smmu_cmdq_batch_submit(smmu, &cmds);
1308 }
1309
1310 static void arm_smmu_write_cd_l1_desc(struct arm_smmu_cdtab_l1 *dst,
1311 dma_addr_t l2ptr_dma)
1312 {
1313 u64 val = (l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) | CTXDESC_L1_DESC_V;
1314
1315 /* The HW has 64 bit atomicity with stores to the L2 CD table */
1316 WRITE_ONCE(dst->l2ptr, cpu_to_le64(val));
1317 }
1318
1319 static dma_addr_t arm_smmu_cd_l1_get_desc(const struct arm_smmu_cdtab_l1 *src)
1320 {
1321 return le64_to_cpu(src->l2ptr) & CTXDESC_L1_DESC_L2PTR_MASK;
1322 }
1323
1324 struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master,
1325 u32 ssid)
1326 {
1327 struct arm_smmu_cdtab_l2 *l2;
1328 struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1329
1330 if (!arm_smmu_cdtab_allocated(cd_table))
1331 return NULL;
1332
1333 if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_LINEAR)
1334 return &cd_table->linear.table[ssid];
1335
1336 l2 = cd_table->l2.l2ptrs[arm_smmu_cdtab_l1_idx(ssid)];
1337 if (!l2)
1338 return NULL;
1339 return &l2->cds[arm_smmu_cdtab_l2_idx(ssid)];
1340 }
1341
1342 static struct arm_smmu_cd *arm_smmu_alloc_cd_ptr(struct arm_smmu_master *master,
1343 u32 ssid)
1344 {
1345 struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1346 struct arm_smmu_device *smmu = master->smmu;
1347
1348 might_sleep();
1349 iommu_group_mutex_assert(master->dev);
1350
1351 if (!arm_smmu_cdtab_allocated(cd_table)) {
1352 if (arm_smmu_alloc_cd_tables(master))
1353 return NULL;
1354 }
1355
1356 if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_64K_L2) {
1357 unsigned int idx = arm_smmu_cdtab_l1_idx(ssid);
1358 struct arm_smmu_cdtab_l2 **l2ptr = &cd_table->l2.l2ptrs[idx];
1359
1360 if (!*l2ptr) {
1361 dma_addr_t l2ptr_dma;
1362
1363 *l2ptr = dma_alloc_coherent(smmu->dev, sizeof(**l2ptr),
1364 &l2ptr_dma, GFP_KERNEL);
1365 if (!*l2ptr)
1366 return NULL;
1367
1368 arm_smmu_write_cd_l1_desc(&cd_table->l2.l1tab[idx],
1369 l2ptr_dma);
1370 /* An invalid L1CD can be cached */
1371 arm_smmu_sync_cd(master, ssid, false);
1372 }
1373 }
1374 return arm_smmu_get_cd_ptr(master, ssid);
1375 }
1376
1377 struct arm_smmu_cd_writer {
1378 struct arm_smmu_entry_writer writer;
1379 unsigned int ssid;
1380 };
1381
1382 VISIBLE_IF_KUNIT
1383 void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits)
1384 {
1385 used_bits[0] = cpu_to_le64(CTXDESC_CD_0_V);
1386 if (!(ent[0] & cpu_to_le64(CTXDESC_CD_0_V)))
1387 return;
1388 memset(used_bits, 0xFF, sizeof(struct arm_smmu_cd));
1389
1390 /*
1391 * If EPD0 is set by the make function it means
1392 * T0SZ/TG0/IR0/OR0/SH0/TTB0 are IGNORED
1393 */
1394 if (ent[0] & cpu_to_le64(CTXDESC_CD_0_TCR_EPD0)) {
1395 used_bits[0] &= ~cpu_to_le64(
1396 CTXDESC_CD_0_TCR_T0SZ | CTXDESC_CD_0_TCR_TG0 |
1397 CTXDESC_CD_0_TCR_IRGN0 | CTXDESC_CD_0_TCR_ORGN0 |
1398 CTXDESC_CD_0_TCR_SH0);
1399 used_bits[1] &= ~cpu_to_le64(CTXDESC_CD_1_TTB0_MASK);
1400 }
1401 }
1402 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_cd_used);
1403
1404 static void arm_smmu_cd_writer_sync_entry(struct arm_smmu_entry_writer *writer)
1405 {
1406 struct arm_smmu_cd_writer *cd_writer =
1407 container_of(writer, struct arm_smmu_cd_writer, writer);
1408
1409 arm_smmu_sync_cd(writer->master, cd_writer->ssid, true);
1410 }
1411
1412 static const struct arm_smmu_entry_writer_ops arm_smmu_cd_writer_ops = {
1413 .sync = arm_smmu_cd_writer_sync_entry,
1414 .get_used = arm_smmu_get_cd_used,
1415 };
1416
1417 void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid,
1418 struct arm_smmu_cd *cdptr,
1419 const struct arm_smmu_cd *target)
1420 {
1421 bool target_valid = target->data[0] & cpu_to_le64(CTXDESC_CD_0_V);
1422 bool cur_valid = cdptr->data[0] & cpu_to_le64(CTXDESC_CD_0_V);
1423 struct arm_smmu_cd_writer cd_writer = {
1424 .writer = {
1425 .ops = &arm_smmu_cd_writer_ops,
1426 .master = master,
1427 },
1428 .ssid = ssid,
1429 };
1430
1431 if (ssid != IOMMU_NO_PASID && cur_valid != target_valid) {
1432 if (cur_valid)
1433 master->cd_table.used_ssids--;
1434 else
1435 master->cd_table.used_ssids++;
1436 }
1437
1438 arm_smmu_write_entry(&cd_writer.writer, cdptr->data, target->data);
1439 }
1440
1441 void arm_smmu_make_s1_cd(struct arm_smmu_cd *target,
1442 struct arm_smmu_master *master,
1443 struct arm_smmu_domain *smmu_domain)
1444 {
1445 struct arm_smmu_ctx_desc *cd = &smmu_domain->cd;
1446 const struct io_pgtable_cfg *pgtbl_cfg =
1447 &io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops)->cfg;
1448 typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr =
1449 &pgtbl_cfg->arm_lpae_s1_cfg.tcr;
1450
1451 memset(target, 0, sizeof(*target));
1452
1453 target->data[0] = cpu_to_le64(
1454 FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) |
1455 FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) |
1456 FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) |
1457 FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) |
1458 FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) |
1459 #ifdef __BIG_ENDIAN
1460 CTXDESC_CD_0_ENDI |
1461 #endif
1462 CTXDESC_CD_0_TCR_EPD1 |
1463 CTXDESC_CD_0_V |
1464 FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) |
1465 CTXDESC_CD_0_AA64 |
1466 (master->stall_enabled ? CTXDESC_CD_0_S : 0) |
1467 CTXDESC_CD_0_R |
1468 CTXDESC_CD_0_A |
1469 CTXDESC_CD_0_ASET |
1470 FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid)
1471 );
1472
1473 /* To enable dirty flag update, set both Access flag and dirty state update */
1474 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_HD)
1475 target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HA |
1476 CTXDESC_CD_0_TCR_HD);
1477
1478 target->data[1] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.ttbr &
1479 CTXDESC_CD_1_TTB0_MASK);
1480 target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.mair);
1481 }
1482 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_s1_cd);
1483
1484 void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid)
1485 {
1486 struct arm_smmu_cd target = {};
1487 struct arm_smmu_cd *cdptr;
1488
1489 if (!arm_smmu_cdtab_allocated(&master->cd_table))
1490 return;
1491 cdptr = arm_smmu_get_cd_ptr(master, ssid);
1492 if (WARN_ON(!cdptr))
1493 return;
1494 arm_smmu_write_cd_entry(master, ssid, cdptr, &target);
1495 }
1496
1497 static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master)
1498 {
1499 int ret;
1500 size_t l1size;
1501 size_t max_contexts;
1502 struct arm_smmu_device *smmu = master->smmu;
1503 struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1504
1505 cd_table->s1cdmax = master->ssid_bits;
1506 max_contexts = 1 << cd_table->s1cdmax;
1507
1508 if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) ||
1509 max_contexts <= CTXDESC_L2_ENTRIES) {
1510 cd_table->s1fmt = STRTAB_STE_0_S1FMT_LINEAR;
1511 cd_table->linear.num_ents = max_contexts;
1512
1513 l1size = max_contexts * sizeof(struct arm_smmu_cd);
1514 cd_table->linear.table = dma_alloc_coherent(smmu->dev, l1size,
1515 &cd_table->cdtab_dma,
1516 GFP_KERNEL);
1517 if (!cd_table->linear.table)
1518 return -ENOMEM;
1519 } else {
1520 cd_table->s1fmt = STRTAB_STE_0_S1FMT_64K_L2;
1521 cd_table->l2.num_l1_ents =
1522 DIV_ROUND_UP(max_contexts, CTXDESC_L2_ENTRIES);
1523
1524 cd_table->l2.l2ptrs = kcalloc(cd_table->l2.num_l1_ents,
1525 sizeof(*cd_table->l2.l2ptrs),
1526 GFP_KERNEL);
1527 if (!cd_table->l2.l2ptrs)
1528 return -ENOMEM;
1529
1530 l1size = cd_table->l2.num_l1_ents * sizeof(struct arm_smmu_cdtab_l1);
1531 cd_table->l2.l1tab = dma_alloc_coherent(smmu->dev, l1size,
1532 &cd_table->cdtab_dma,
1533 GFP_KERNEL);
1534 if (!cd_table->l2.l1tab) {
1535 ret = -ENOMEM;
1536 goto err_free_l2ptrs;
1537 }
1538 }
1539 return 0;
1540
1541 err_free_l2ptrs:
1542 kfree(cd_table->l2.l2ptrs);
1543 cd_table->l2.l2ptrs = NULL;
1544 return ret;
1545 }
1546
1547 static void arm_smmu_free_cd_tables(struct arm_smmu_master *master)
1548 {
1549 int i;
1550 struct arm_smmu_device *smmu = master->smmu;
1551 struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1552
1553 if (cd_table->s1fmt != STRTAB_STE_0_S1FMT_LINEAR) {
1554 for (i = 0; i < cd_table->l2.num_l1_ents; i++) {
1555 if (!cd_table->l2.l2ptrs[i])
1556 continue;
1557
1558 dma_free_coherent(smmu->dev,
1559 sizeof(*cd_table->l2.l2ptrs[i]),
1560 cd_table->l2.l2ptrs[i],
1561 arm_smmu_cd_l1_get_desc(&cd_table->l2.l1tab[i]));
1562 }
1563 kfree(cd_table->l2.l2ptrs);
1564
1565 dma_free_coherent(smmu->dev,
1566 cd_table->l2.num_l1_ents *
1567 sizeof(struct arm_smmu_cdtab_l1),
1568 cd_table->l2.l1tab, cd_table->cdtab_dma);
1569 } else {
1570 dma_free_coherent(smmu->dev,
1571 cd_table->linear.num_ents *
1572 sizeof(struct arm_smmu_cd),
1573 cd_table->linear.table, cd_table->cdtab_dma);
1574 }
1575 }
1576
1577 /* Stream table manipulation functions */
1578 static void arm_smmu_write_strtab_l1_desc(struct arm_smmu_strtab_l1 *dst,
1579 dma_addr_t l2ptr_dma)
1580 {
1581 u64 val = 0;
1582
1583 val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, STRTAB_SPLIT + 1);
1584 val |= l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK;
1585
1586 /* The HW has 64 bit atomicity with stores to the L2 STE table */
1587 WRITE_ONCE(dst->l2ptr, cpu_to_le64(val));
1588 }
1589
1590 struct arm_smmu_ste_writer {
1591 struct arm_smmu_entry_writer writer;
1592 u32 sid;
1593 };
1594
1595 static void arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer *writer)
1596 {
1597 struct arm_smmu_ste_writer *ste_writer =
1598 container_of(writer, struct arm_smmu_ste_writer, writer);
1599 struct arm_smmu_cmdq_ent cmd = {
1600 .opcode = CMDQ_OP_CFGI_STE,
1601 .cfgi = {
1602 .sid = ste_writer->sid,
1603 .leaf = true,
1604 },
1605 };
1606
1607 arm_smmu_cmdq_issue_cmd_with_sync(writer->master->smmu, &cmd);
1608 }
1609
1610 static const struct arm_smmu_entry_writer_ops arm_smmu_ste_writer_ops = {
1611 .sync = arm_smmu_ste_writer_sync_entry,
1612 .get_used = arm_smmu_get_ste_used,
1613 .get_update_safe = arm_smmu_get_ste_update_safe,
1614 };
1615
1616 static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid,
1617 struct arm_smmu_ste *ste,
1618 const struct arm_smmu_ste *target)
1619 {
1620 struct arm_smmu_device *smmu = master->smmu;
1621 struct arm_smmu_ste_writer ste_writer = {
1622 .writer = {
1623 .ops = &arm_smmu_ste_writer_ops,
1624 .master = master,
1625 },
1626 .sid = sid,
1627 };
1628
1629 arm_smmu_write_entry(&ste_writer.writer, ste->data, target->data);
1630
1631 /* It's likely that we'll want to use the new STE soon */
1632 if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) {
1633 struct arm_smmu_cmdq_ent
1634 prefetch_cmd = { .opcode = CMDQ_OP_PREFETCH_CFG,
1635 .prefetch = {
1636 .sid = sid,
1637 } };
1638
1639 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1640 }
1641 }
1642
1643 void arm_smmu_make_abort_ste(struct arm_smmu_ste *target)
1644 {
1645 memset(target, 0, sizeof(*target));
1646 target->data[0] = cpu_to_le64(
1647 STRTAB_STE_0_V |
1648 FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT));
1649 }
1650 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_abort_ste);
1651
1652 VISIBLE_IF_KUNIT
1653 void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu,
1654 struct arm_smmu_ste *target)
1655 {
1656 memset(target, 0, sizeof(*target));
1657 target->data[0] = cpu_to_le64(
1658 STRTAB_STE_0_V |
1659 FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS));
1660
1661 if (smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR)
1662 target->data[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1663 STRTAB_STE_1_SHCFG_INCOMING));
1664 }
1665 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_bypass_ste);
1666
1667 VISIBLE_IF_KUNIT
1668 void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target,
1669 struct arm_smmu_master *master, bool ats_enabled,
1670 unsigned int s1dss)
1671 {
1672 struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1673 struct arm_smmu_device *smmu = master->smmu;
1674
1675 memset(target, 0, sizeof(*target));
1676 target->data[0] = cpu_to_le64(
1677 STRTAB_STE_0_V |
1678 FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) |
1679 FIELD_PREP(STRTAB_STE_0_S1FMT, cd_table->s1fmt) |
1680 (cd_table->cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
1681 FIELD_PREP(STRTAB_STE_0_S1CDMAX, cd_table->s1cdmax));
1682
1683 target->data[1] = cpu_to_le64(
1684 FIELD_PREP(STRTAB_STE_1_S1DSS, s1dss) |
1685 FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1686 FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1687 FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) |
1688 ((smmu->features & ARM_SMMU_FEAT_STALLS &&
1689 !master->stall_enabled) ?
1690 STRTAB_STE_1_S1STALLD :
1691 0) |
1692 FIELD_PREP(STRTAB_STE_1_EATS,
1693 ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0));
1694
1695 if ((smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR) &&
1696 s1dss == STRTAB_STE_1_S1DSS_BYPASS)
1697 target->data[1] |= cpu_to_le64(FIELD_PREP(
1698 STRTAB_STE_1_SHCFG, STRTAB_STE_1_SHCFG_INCOMING));
1699
1700 if (smmu->features & ARM_SMMU_FEAT_E2H) {
1701 /*
1702 * To support BTM the streamworld needs to match the
1703 * configuration of the CPU so that the ASID broadcasts are
1704 * properly matched. This means either S/NS-EL2-E2H (hypervisor)
1705 * or NS-EL1 (guest). Since an SVA domain can be installed in a
1706 * PASID this should always use a BTM compatible configuration
1707 * if the HW supports it.
1708 */
1709 target->data[1] |= cpu_to_le64(
1710 FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_EL2));
1711 } else {
1712 target->data[1] |= cpu_to_le64(
1713 FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1));
1714
1715 /*
1716 * VMID 0 is reserved for stage-2 bypass EL1 STEs, see
1717 * arm_smmu_domain_alloc_id()
1718 */
1719 target->data[2] =
1720 cpu_to_le64(FIELD_PREP(STRTAB_STE_2_S2VMID, 0));
1721 }
1722 }
1723 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_cdtable_ste);
1724
1725 void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target,
1726 struct arm_smmu_master *master,
1727 struct arm_smmu_domain *smmu_domain,
1728 bool ats_enabled)
1729 {
1730 struct arm_smmu_s2_cfg *s2_cfg = &smmu_domain->s2_cfg;
1731 const struct io_pgtable_cfg *pgtbl_cfg =
1732 &io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops)->cfg;
1733 typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr =
1734 &pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
1735 u64 vtcr_val;
1736 struct arm_smmu_device *smmu = master->smmu;
1737
1738 memset(target, 0, sizeof(*target));
1739 target->data[0] = cpu_to_le64(
1740 STRTAB_STE_0_V |
1741 FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS));
1742
1743 target->data[1] = cpu_to_le64(
1744 FIELD_PREP(STRTAB_STE_1_EATS,
1745 ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0));
1746
1747 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_S2FWB)
1748 target->data[1] |= cpu_to_le64(STRTAB_STE_1_S2FWB);
1749 if (smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR)
1750 target->data[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1751 STRTAB_STE_1_SHCFG_INCOMING));
1752
1753 vtcr_val = FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) |
1754 FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) |
1755 FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) |
1756 FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) |
1757 FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) |
1758 FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) |
1759 FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps);
1760 target->data[2] = cpu_to_le64(
1761 FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |
1762 FIELD_PREP(STRTAB_STE_2_VTCR, vtcr_val) |
1763 STRTAB_STE_2_S2AA64 |
1764 #ifdef __BIG_ENDIAN
1765 STRTAB_STE_2_S2ENDI |
1766 #endif
1767 STRTAB_STE_2_S2PTW |
1768 (master->stall_enabled ? STRTAB_STE_2_S2S : 0) |
1769 STRTAB_STE_2_S2R);
1770
1771 target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s2_cfg.vttbr &
1772 STRTAB_STE_3_S2TTB_MASK);
1773 }
1774 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_s2_domain_ste);
1775
1776 /*
1777 * This can safely directly manipulate the STE memory without a sync sequence
1778 * because the STE table has not been installed in the SMMU yet.
1779 */
1780 static void arm_smmu_init_initial_stes(struct arm_smmu_ste *strtab,
1781 unsigned int nent)
1782 {
1783 unsigned int i;
1784
1785 for (i = 0; i < nent; ++i) {
1786 arm_smmu_make_abort_ste(strtab);
1787 strtab++;
1788 }
1789 }
1790
1791 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1792 {
1793 dma_addr_t l2ptr_dma;
1794 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1795 struct arm_smmu_strtab_l2 **l2table;
1796
1797 l2table = &cfg->l2.l2ptrs[arm_smmu_strtab_l1_idx(sid)];
1798 if (*l2table)
1799 return 0;
1800
1801 *l2table = dmam_alloc_coherent(smmu->dev, sizeof(**l2table),
1802 &l2ptr_dma, GFP_KERNEL);
1803 if (!*l2table) {
1804 dev_err(smmu->dev,
1805 "failed to allocate l2 stream table for SID %u\n",
1806 sid);
1807 return -ENOMEM;
1808 }
1809
1810 arm_smmu_init_initial_stes((*l2table)->stes,
1811 ARRAY_SIZE((*l2table)->stes));
1812 arm_smmu_write_strtab_l1_desc(&cfg->l2.l1tab[arm_smmu_strtab_l1_idx(sid)],
1813 l2ptr_dma);
1814 return 0;
1815 }
1816
1817 static int arm_smmu_streams_cmp_key(const void *lhs, const struct rb_node *rhs)
1818 {
1819 struct arm_smmu_stream *stream_rhs =
1820 rb_entry(rhs, struct arm_smmu_stream, node);
1821 const u32 *sid_lhs = lhs;
1822
1823 if (*sid_lhs < stream_rhs->id)
1824 return -1;
1825 if (*sid_lhs > stream_rhs->id)
1826 return 1;
1827 return 0;
1828 }
1829
1830 static int arm_smmu_streams_cmp_node(struct rb_node *lhs,
1831 const struct rb_node *rhs)
1832 {
1833 return arm_smmu_streams_cmp_key(
1834 &rb_entry(lhs, struct arm_smmu_stream, node)->id, rhs);
1835 }
1836
1837 static struct arm_smmu_master *
1838 arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid)
1839 {
1840 struct rb_node *node;
1841
1842 lockdep_assert_held(&smmu->streams_mutex);
1843
1844 node = rb_find(&sid, &smmu->streams, arm_smmu_streams_cmp_key);
1845 if (!node)
1846 return NULL;
1847 return rb_entry(node, struct arm_smmu_stream, node)->master;
1848 }
1849
1850 /* IRQ and event handlers */
1851 static void arm_smmu_decode_event(struct arm_smmu_device *smmu, u64 *raw,
1852 struct arm_smmu_event *event)
1853 {
1854 struct arm_smmu_master *master;
1855
1856 event->id = FIELD_GET(EVTQ_0_ID, raw[0]);
1857 event->sid = FIELD_GET(EVTQ_0_SID, raw[0]);
1858 event->ssv = FIELD_GET(EVTQ_0_SSV, raw[0]);
1859 event->ssid = event->ssv ? FIELD_GET(EVTQ_0_SSID, raw[0]) : IOMMU_NO_PASID;
1860 event->privileged = FIELD_GET(EVTQ_1_PnU, raw[1]);
1861 event->instruction = FIELD_GET(EVTQ_1_InD, raw[1]);
1862 event->s2 = FIELD_GET(EVTQ_1_S2, raw[1]);
1863 event->read = FIELD_GET(EVTQ_1_RnW, raw[1]);
1864 event->stag = FIELD_GET(EVTQ_1_STAG, raw[1]);
1865 event->stall = FIELD_GET(EVTQ_1_STALL, raw[1]);
1866 event->class = FIELD_GET(EVTQ_1_CLASS, raw[1]);
1867 event->iova = FIELD_GET(EVTQ_2_ADDR, raw[2]);
1868 event->ipa = raw[3] & EVTQ_3_IPA;
1869 event->fetch_addr = raw[3] & EVTQ_3_FETCH_ADDR;
1870 event->ttrnw = FIELD_GET(EVTQ_1_TT_READ, raw[1]);
1871 event->class_tt = false;
1872 event->dev = NULL;
1873
1874 if (event->id == EVT_ID_PERMISSION_FAULT)
1875 event->class_tt = (event->class == EVTQ_1_CLASS_TT);
1876
1877 mutex_lock(&smmu->streams_mutex);
1878 master = arm_smmu_find_master(smmu, event->sid);
1879 if (master)
1880 event->dev = get_device(master->dev);
1881 mutex_unlock(&smmu->streams_mutex);
1882 }
1883
1884 static int arm_smmu_handle_event(struct arm_smmu_device *smmu, u64 *evt,
1885 struct arm_smmu_event *event)
1886 {
1887 int ret = 0;
1888 u32 perm = 0;
1889 struct arm_smmu_master *master;
1890 struct iopf_fault fault_evt = { };
1891 struct iommu_fault *flt = &fault_evt.fault;
1892
1893 switch (event->id) {
1894 case EVT_ID_BAD_STE_CONFIG:
1895 case EVT_ID_STREAM_DISABLED_FAULT:
1896 case EVT_ID_BAD_SUBSTREAMID_CONFIG:
1897 case EVT_ID_BAD_CD_CONFIG:
1898 case EVT_ID_TRANSLATION_FAULT:
1899 case EVT_ID_ADDR_SIZE_FAULT:
1900 case EVT_ID_ACCESS_FAULT:
1901 case EVT_ID_PERMISSION_FAULT:
1902 break;
1903 default:
1904 return -EOPNOTSUPP;
1905 }
1906
1907 if (event->stall) {
1908 if (event->read)
1909 perm |= IOMMU_FAULT_PERM_READ;
1910 else
1911 perm |= IOMMU_FAULT_PERM_WRITE;
1912
1913 if (event->instruction)
1914 perm |= IOMMU_FAULT_PERM_EXEC;
1915
1916 if (event->privileged)
1917 perm |= IOMMU_FAULT_PERM_PRIV;
1918
1919 flt->type = IOMMU_FAULT_PAGE_REQ;
1920 flt->prm = (struct iommu_fault_page_request){
1921 .flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE,
1922 .grpid = event->stag,
1923 .perm = perm,
1924 .addr = event->iova,
1925 };
1926
1927 if (event->ssv) {
1928 flt->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
1929 flt->prm.pasid = event->ssid;
1930 }
1931 }
1932
1933 mutex_lock(&smmu->streams_mutex);
1934 master = arm_smmu_find_master(smmu, event->sid);
1935 if (!master) {
1936 ret = -EINVAL;
1937 goto out_unlock;
1938 }
1939
1940 if (event->stall)
1941 ret = iommu_report_device_fault(master->dev, &fault_evt);
1942 else if (master->vmaster && !event->s2)
1943 ret = arm_vmaster_report_event(master->vmaster, evt);
1944 else
1945 ret = -EOPNOTSUPP; /* Unhandled events should be pinned */
1946 out_unlock:
1947 mutex_unlock(&smmu->streams_mutex);
1948 return ret;
1949 }
1950
1951 static void arm_smmu_dump_raw_event(struct arm_smmu_device *smmu, u64 *raw,
1952 struct arm_smmu_event *event)
1953 {
1954 int i;
1955
1956 dev_err(smmu->dev, "event 0x%02x received:\n", event->id);
1957
1958 for (i = 0; i < EVTQ_ENT_DWORDS; ++i)
1959 dev_err(smmu->dev, "\t0x%016llx\n", raw[i]);
1960 }
1961
1962 #define ARM_SMMU_EVT_KNOWN(e) ((e)->id < ARRAY_SIZE(event_str) && event_str[(e)->id])
1963 #define ARM_SMMU_LOG_EVT_STR(e) ARM_SMMU_EVT_KNOWN(e) ? event_str[(e)->id] : "UNKNOWN"
1964 #define ARM_SMMU_LOG_CLIENT(e) (e)->dev ? dev_name((e)->dev) : "(unassigned sid)"
1965
1966 static void arm_smmu_dump_event(struct arm_smmu_device *smmu, u64 *raw,
1967 struct arm_smmu_event *evt,
1968 struct ratelimit_state *rs)
1969 {
1970 if (!__ratelimit(rs))
1971 return;
1972
1973 arm_smmu_dump_raw_event(smmu, raw, evt);
1974
1975 switch (evt->id) {
1976 case EVT_ID_TRANSLATION_FAULT:
1977 case EVT_ID_ADDR_SIZE_FAULT:
1978 case EVT_ID_ACCESS_FAULT:
1979 case EVT_ID_PERMISSION_FAULT:
1980 dev_err(smmu->dev, "event: %s client: %s sid: %#x ssid: %#x iova: %#llx ipa: %#llx",
1981 ARM_SMMU_LOG_EVT_STR(evt), ARM_SMMU_LOG_CLIENT(evt),
1982 evt->sid, evt->ssid, evt->iova, evt->ipa);
1983
1984 dev_err(smmu->dev, "%s %s %s %s \"%s\"%s%s stag: %#x",
1985 evt->privileged ? "priv" : "unpriv",
1986 evt->instruction ? "inst" : "data",
1987 str_read_write(evt->read),
1988 evt->s2 ? "s2" : "s1", event_class_str[evt->class],
1989 evt->class_tt ? (evt->ttrnw ? " ttd_read" : " ttd_write") : "",
1990 evt->stall ? " stall" : "", evt->stag);
1991
1992 break;
1993
1994 case EVT_ID_STE_FETCH_FAULT:
1995 case EVT_ID_CD_FETCH_FAULT:
1996 case EVT_ID_VMS_FETCH_FAULT:
1997 dev_err(smmu->dev, "event: %s client: %s sid: %#x ssid: %#x fetch_addr: %#llx",
1998 ARM_SMMU_LOG_EVT_STR(evt), ARM_SMMU_LOG_CLIENT(evt),
1999 evt->sid, evt->ssid, evt->fetch_addr);
2000
2001 break;
2002
2003 default:
2004 dev_err(smmu->dev, "event: %s client: %s sid: %#x ssid: %#x",
2005 ARM_SMMU_LOG_EVT_STR(evt), ARM_SMMU_LOG_CLIENT(evt),
2006 evt->sid, evt->ssid);
2007 }
2008 }
2009
2010 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
2011 {
2012 u64 evt[EVTQ_ENT_DWORDS];
2013 struct arm_smmu_event event = {0};
2014 struct arm_smmu_device *smmu = dev;
2015 struct arm_smmu_queue *q = &smmu->evtq.q;
2016 struct arm_smmu_ll_queue *llq = &q->llq;
2017 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
2018 DEFAULT_RATELIMIT_BURST);
2019
2020 do {
2021 while (!queue_remove_raw(q, evt)) {
2022 arm_smmu_decode_event(smmu, evt, &event);
2023 if (arm_smmu_handle_event(smmu, evt, &event))
2024 arm_smmu_dump_event(smmu, evt, &event, &rs);
2025
2026 put_device(event.dev);
2027 cond_resched();
2028 }
2029
2030 /*
2031 * Not much we can do on overflow, so scream and pretend we're
2032 * trying harder.
2033 */
2034 if (queue_sync_prod_in(q) == -EOVERFLOW)
2035 dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
2036 } while (!queue_empty(llq));
2037
2038 /* Sync our overflow flag, as we believe we're up to speed */
2039 queue_sync_cons_ovf(q);
2040 return IRQ_HANDLED;
2041 }
2042
2043 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
2044 {
2045 u32 sid, ssid;
2046 u16 grpid;
2047 bool ssv, last;
2048
2049 sid = FIELD_GET(PRIQ_0_SID, evt[0]);
2050 ssv = FIELD_GET(PRIQ_0_SSID_V, evt[0]);
2051 ssid = ssv ? FIELD_GET(PRIQ_0_SSID, evt[0]) : IOMMU_NO_PASID;
2052 last = FIELD_GET(PRIQ_0_PRG_LAST, evt[0]);
2053 grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]);
2054
2055 dev_info(smmu->dev, "unexpected PRI request received:\n");
2056 dev_info(smmu->dev,
2057 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
2058 sid, ssid, grpid, last ? "L" : "",
2059 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
2060 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
2061 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
2062 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
2063 evt[1] & PRIQ_1_ADDR_MASK);
2064
2065 if (last) {
2066 struct arm_smmu_cmdq_ent cmd = {
2067 .opcode = CMDQ_OP_PRI_RESP,
2068 .substream_valid = ssv,
2069 .pri = {
2070 .sid = sid,
2071 .ssid = ssid,
2072 .grpid = grpid,
2073 .resp = PRI_RESP_DENY,
2074 },
2075 };
2076
2077 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
2078 }
2079 }
2080
2081 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
2082 {
2083 struct arm_smmu_device *smmu = dev;
2084 struct arm_smmu_queue *q = &smmu->priq.q;
2085 struct arm_smmu_ll_queue *llq = &q->llq;
2086 u64 evt[PRIQ_ENT_DWORDS];
2087
2088 do {
2089 while (!queue_remove_raw(q, evt))
2090 arm_smmu_handle_ppr(smmu, evt);
2091
2092 if (queue_sync_prod_in(q) == -EOVERFLOW)
2093 dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
2094 } while (!queue_empty(llq));
2095
2096 /* Sync our overflow flag, as we believe we're up to speed */
2097 queue_sync_cons_ovf(q);
2098 return IRQ_HANDLED;
2099 }
2100
2101 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
2102
2103 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
2104 {
2105 u32 gerror, gerrorn, active;
2106 struct arm_smmu_device *smmu = dev;
2107
2108 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
2109 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
2110
2111 active = gerror ^ gerrorn;
2112 if (!(active & GERROR_ERR_MASK))
2113 return IRQ_NONE; /* No errors pending */
2114
2115 dev_warn(smmu->dev,
2116 "unexpected global error reported (0x%08x), this could be serious\n",
2117 active);
2118
2119 if (active & GERROR_SFM_ERR) {
2120 dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
2121 arm_smmu_device_disable(smmu);
2122 }
2123
2124 if (active & GERROR_MSI_GERROR_ABT_ERR)
2125 dev_warn(smmu->dev, "GERROR MSI write aborted\n");
2126
2127 if (active & GERROR_MSI_PRIQ_ABT_ERR)
2128 dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
2129
2130 if (active & GERROR_MSI_EVTQ_ABT_ERR)
2131 dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
2132
2133 if (active & GERROR_MSI_CMDQ_ABT_ERR)
2134 dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
2135
2136 if (active & GERROR_PRIQ_ABT_ERR)
2137 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
2138
2139 if (active & GERROR_EVTQ_ABT_ERR)
2140 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
2141
2142 if (active & GERROR_CMDQ_ERR)
2143 arm_smmu_cmdq_skip_err(smmu);
2144
2145 writel(gerror, smmu->base + ARM_SMMU_GERRORN);
2146 return IRQ_HANDLED;
2147 }
2148
2149 static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev)
2150 {
2151 struct arm_smmu_device *smmu = dev;
2152
2153 arm_smmu_evtq_thread(irq, dev);
2154 if (smmu->features & ARM_SMMU_FEAT_PRI)
2155 arm_smmu_priq_thread(irq, dev);
2156
2157 return IRQ_HANDLED;
2158 }
2159
2160 static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
2161 {
2162 arm_smmu_gerror_handler(irq, dev);
2163 return IRQ_WAKE_THREAD;
2164 }
2165
2166 static void
2167 arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
2168 struct arm_smmu_cmdq_ent *cmd)
2169 {
2170 size_t log2_span;
2171 size_t span_mask;
2172 /* ATC invalidates are always on 4096-bytes pages */
2173 size_t inval_grain_shift = 12;
2174 unsigned long page_start, page_end;
2175
2176 /*
2177 * ATS and PASID:
2178 *
2179 * If substream_valid is clear, the PCIe TLP is sent without a PASID
2180 * prefix. In that case all ATC entries within the address range are
2181 * invalidated, including those that were requested with a PASID! There
2182 * is no way to invalidate only entries without PASID.
2183 *
2184 * When using STRTAB_STE_1_S1DSS_SSID0 (reserving CD 0 for non-PASID
2185 * traffic), translation requests without PASID create ATC entries
2186 * without PASID, which must be invalidated with substream_valid clear.
2187 * This has the unpleasant side-effect of invalidating all PASID-tagged
2188 * ATC entries within the address range.
2189 */
2190 *cmd = (struct arm_smmu_cmdq_ent) {
2191 .opcode = CMDQ_OP_ATC_INV,
2192 .substream_valid = (ssid != IOMMU_NO_PASID),
2193 .atc.ssid = ssid,
2194 };
2195
2196 if (!size) {
2197 cmd->atc.size = ATC_INV_SIZE_ALL;
2198 return;
2199 }
2200
2201 page_start = iova >> inval_grain_shift;
2202 page_end = (iova + size - 1) >> inval_grain_shift;
2203
2204 /*
2205 * In an ATS Invalidate Request, the address must be aligned on the
2206 * range size, which must be a power of two number of page sizes. We
2207 * thus have to choose between grossly over-invalidating the region, or
2208 * splitting the invalidation into multiple commands. For simplicity
2209 * we'll go with the first solution, but should refine it in the future
2210 * if multiple commands are shown to be more efficient.
2211 *
2212 * Find the smallest power of two that covers the range. The most
2213 * significant differing bit between the start and end addresses,
2214 * fls(start ^ end), indicates the required span. For example:
2215 *
2216 * We want to invalidate pages [8; 11]. This is already the ideal range:
2217 * x = 0b1000 ^ 0b1011 = 0b11
2218 * span = 1 << fls(x) = 4
2219 *
2220 * To invalidate pages [7; 10], we need to invalidate [0; 15]:
2221 * x = 0b0111 ^ 0b1010 = 0b1101
2222 * span = 1 << fls(x) = 16
2223 */
2224 log2_span = fls_long(page_start ^ page_end);
2225 span_mask = (1ULL << log2_span) - 1;
2226
2227 page_start &= ~span_mask;
2228
2229 cmd->atc.addr = page_start << inval_grain_shift;
2230 cmd->atc.size = log2_span;
2231 }
2232
2233 static int arm_smmu_atc_inv_master(struct arm_smmu_master *master,
2234 ioasid_t ssid)
2235 {
2236 int i;
2237 struct arm_smmu_cmdq_ent cmd;
2238 struct arm_smmu_cmdq_batch cmds;
2239
2240 arm_smmu_atc_inv_to_cmd(ssid, 0, 0, &cmd);
2241
2242 arm_smmu_cmdq_batch_init(master->smmu, &cmds, &cmd);
2243 for (i = 0; i < master->num_streams; i++) {
2244 cmd.atc.sid = master->streams[i].id;
2245 arm_smmu_cmdq_batch_add(master->smmu, &cmds, &cmd);
2246 }
2247
2248 return arm_smmu_cmdq_batch_submit(master->smmu, &cmds);
2249 }
2250
2251 int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
2252 unsigned long iova, size_t size)
2253 {
2254 struct arm_smmu_master_domain *master_domain;
2255 int i;
2256 unsigned long flags;
2257 struct arm_smmu_cmdq_ent cmd = {
2258 .opcode = CMDQ_OP_ATC_INV,
2259 };
2260 struct arm_smmu_cmdq_batch cmds;
2261
2262 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
2263 return 0;
2264
2265 /*
2266 * Ensure that we've completed prior invalidation of the main TLBs
2267 * before we read 'nr_ats_masters' in case of a concurrent call to
2268 * arm_smmu_enable_ats():
2269 *
2270 * // unmap() // arm_smmu_enable_ats()
2271 * TLBI+SYNC atomic_inc(&nr_ats_masters);
2272 * smp_mb(); [...]
2273 * atomic_read(&nr_ats_masters); pci_enable_ats() // writel()
2274 *
2275 * Ensures that we always see the incremented 'nr_ats_masters' count if
2276 * ATS was enabled at the PCI device before completion of the TLBI.
2277 */
2278 smp_mb();
2279 if (!atomic_read(&smmu_domain->nr_ats_masters))
2280 return 0;
2281
2282 arm_smmu_cmdq_batch_init(smmu_domain->smmu, &cmds, &cmd);
2283
2284 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2285 list_for_each_entry(master_domain, &smmu_domain->devices,
2286 devices_elm) {
2287 struct arm_smmu_master *master = master_domain->master;
2288
2289 if (!master->ats_enabled)
2290 continue;
2291
2292 if (master_domain->nested_ats_flush) {
2293 /*
2294 * If a S2 used as a nesting parent is changed we have
2295 * no option but to completely flush the ATC.
2296 */
2297 arm_smmu_atc_inv_to_cmd(IOMMU_NO_PASID, 0, 0, &cmd);
2298 } else {
2299 arm_smmu_atc_inv_to_cmd(master_domain->ssid, iova, size,
2300 &cmd);
2301 }
2302
2303 for (i = 0; i < master->num_streams; i++) {
2304 cmd.atc.sid = master->streams[i].id;
2305 arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd);
2306 }
2307 }
2308 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2309
2310 return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds);
2311 }
2312
2313 /* IO_PGTABLE API */
2314 static void arm_smmu_tlb_inv_context(void *cookie)
2315 {
2316 struct arm_smmu_domain *smmu_domain = cookie;
2317 struct arm_smmu_device *smmu = smmu_domain->smmu;
2318 struct arm_smmu_cmdq_ent cmd;
2319
2320 /*
2321 * NOTE: when io-pgtable is in non-strict mode, we may get here with
2322 * PTEs previously cleared by unmaps on the current CPU not yet visible
2323 * to the SMMU. We are relying on the dma_wmb() implicit during cmd
2324 * insertion to guarantee those are observed before the TLBI. Do be
2325 * careful, 007.
2326 */
2327 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2328 arm_smmu_tlb_inv_asid(smmu, smmu_domain->cd.asid);
2329 } else {
2330 cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
2331 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
2332 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
2333 }
2334 arm_smmu_atc_inv_domain(smmu_domain, 0, 0);
2335 }
2336
2337 static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd,
2338 unsigned long iova, size_t size,
2339 size_t granule,
2340 struct arm_smmu_domain *smmu_domain)
2341 {
2342 struct arm_smmu_device *smmu = smmu_domain->smmu;
2343 unsigned long end = iova + size, num_pages = 0, tg = 0;
2344 size_t inv_range = granule;
2345 struct arm_smmu_cmdq_batch cmds;
2346
2347 if (!size)
2348 return;
2349
2350 if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
2351 /* Get the leaf page size */
2352 tg = __ffs(smmu_domain->domain.pgsize_bitmap);
2353
2354 num_pages = size >> tg;
2355
2356 /* Convert page size of 12,14,16 (log2) to 1,2,3 */
2357 cmd->tlbi.tg = (tg - 10) / 2;
2358
2359 /*
2360 * Determine what level the granule is at. For non-leaf, both
2361 * io-pgtable and SVA pass a nominal last-level granule because
2362 * they don't know what level(s) actually apply, so ignore that
2363 * and leave TTL=0. However for various errata reasons we still
2364 * want to use a range command, so avoid the SVA corner case
2365 * where both scale and num could be 0 as well.
2366 */
2367 if (cmd->tlbi.leaf)
2368 cmd->tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
2369 else if ((num_pages & CMDQ_TLBI_RANGE_NUM_MAX) == 1)
2370 num_pages++;
2371 }
2372
2373 arm_smmu_cmdq_batch_init(smmu, &cmds, cmd);
2374
2375 while (iova < end) {
2376 if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
2377 /*
2378 * On each iteration of the loop, the range is 5 bits
2379 * worth of the aligned size remaining.
2380 * The range in pages is:
2381 *
2382 * range = (num_pages & (0x1f << __ffs(num_pages)))
2383 */
2384 unsigned long scale, num;
2385
2386 /* Determine the power of 2 multiple number of pages */
2387 scale = __ffs(num_pages);
2388 cmd->tlbi.scale = scale;
2389
2390 /* Determine how many chunks of 2^scale size we have */
2391 num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX;
2392 cmd->tlbi.num = num - 1;
2393
2394 /* range is num * 2^scale * pgsize */
2395 inv_range = num << (scale + tg);
2396
2397 /* Clear out the lower order bits for the next iteration */
2398 num_pages -= num << scale;
2399 }
2400
2401 cmd->tlbi.addr = iova;
2402 arm_smmu_cmdq_batch_add(smmu, &cmds, cmd);
2403 iova += inv_range;
2404 }
2405 arm_smmu_cmdq_batch_submit(smmu, &cmds);
2406 }
2407
2408 static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size,
2409 size_t granule, bool leaf,
2410 struct arm_smmu_domain *smmu_domain)
2411 {
2412 struct arm_smmu_cmdq_ent cmd = {
2413 .tlbi = {
2414 .leaf = leaf,
2415 },
2416 };
2417
2418 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2419 cmd.opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
2420 CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA;
2421 cmd.tlbi.asid = smmu_domain->cd.asid;
2422 } else {
2423 cmd.opcode = CMDQ_OP_TLBI_S2_IPA;
2424 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
2425 }
2426 __arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
2427
2428 if (smmu_domain->nest_parent) {
2429 /*
2430 * When the S2 domain changes all the nested S1 ASIDs have to be
2431 * flushed too.
2432 */
2433 cmd.opcode = CMDQ_OP_TLBI_NH_ALL;
2434 arm_smmu_cmdq_issue_cmd_with_sync(smmu_domain->smmu, &cmd);
2435 }
2436
2437 /*
2438 * Unfortunately, this can't be leaf-only since we may have
2439 * zapped an entire table.
2440 */
2441 arm_smmu_atc_inv_domain(smmu_domain, iova, size);
2442 }
2443
2444 void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
2445 size_t granule, bool leaf,
2446 struct arm_smmu_domain *smmu_domain)
2447 {
2448 struct arm_smmu_cmdq_ent cmd = {
2449 .opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
2450 CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA,
2451 .tlbi = {
2452 .asid = asid,
2453 .leaf = leaf,
2454 },
2455 };
2456
2457 __arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
2458 }
2459
2460 static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
2461 unsigned long iova, size_t granule,
2462 void *cookie)
2463 {
2464 struct arm_smmu_domain *smmu_domain = cookie;
2465 struct iommu_domain *domain = &smmu_domain->domain;
2466
2467 iommu_iotlb_gather_add_page(domain, gather, iova, granule);
2468 }
2469
2470 static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
2471 size_t granule, void *cookie)
2472 {
2473 arm_smmu_tlb_inv_range_domain(iova, size, granule, false, cookie);
2474 }
2475
2476 static const struct iommu_flush_ops arm_smmu_flush_ops = {
2477 .tlb_flush_all = arm_smmu_tlb_inv_context,
2478 .tlb_flush_walk = arm_smmu_tlb_inv_walk,
2479 .tlb_add_page = arm_smmu_tlb_inv_page_nosync,
2480 };
2481
2482 static bool arm_smmu_dbm_capable(struct arm_smmu_device *smmu)
2483 {
2484 u32 features = (ARM_SMMU_FEAT_HD | ARM_SMMU_FEAT_COHERENCY);
2485
2486 return (smmu->features & features) == features;
2487 }
2488
2489 /* IOMMU API */
2490 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
2491 {
2492 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
2493
2494 switch (cap) {
2495 case IOMMU_CAP_CACHE_COHERENCY:
2496 /* Assume that a coherent TCU implies coherent TBUs */
2497 return master->smmu->features & ARM_SMMU_FEAT_COHERENCY;
2498 case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:
2499 return arm_smmu_master_canwbs(master);
2500 case IOMMU_CAP_NOEXEC:
2501 case IOMMU_CAP_DEFERRED_FLUSH:
2502 return true;
2503 case IOMMU_CAP_DIRTY_TRACKING:
2504 return arm_smmu_dbm_capable(master->smmu);
2505 default:
2506 return false;
2507 }
2508 }
2509
2510 static bool arm_smmu_enforce_cache_coherency(struct iommu_domain *domain)
2511 {
2512 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2513 struct arm_smmu_master_domain *master_domain;
2514 unsigned long flags;
2515 bool ret = true;
2516
2517 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2518 list_for_each_entry(master_domain, &smmu_domain->devices,
2519 devices_elm) {
2520 if (!arm_smmu_master_canwbs(master_domain->master)) {
2521 ret = false;
2522 break;
2523 }
2524 }
2525 smmu_domain->enforce_cache_coherency = ret;
2526 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2527 return ret;
2528 }
2529
2530 struct arm_smmu_domain *arm_smmu_domain_alloc(void)
2531 {
2532 struct arm_smmu_domain *smmu_domain;
2533
2534 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
2535 if (!smmu_domain)
2536 return ERR_PTR(-ENOMEM);
2537
2538 INIT_LIST_HEAD(&smmu_domain->devices);
2539 spin_lock_init(&smmu_domain->devices_lock);
2540
2541 return smmu_domain;
2542 }
2543
2544 static void arm_smmu_domain_free_paging(struct iommu_domain *domain)
2545 {
2546 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2547 struct arm_smmu_device *smmu = smmu_domain->smmu;
2548
2549 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
2550
2551 /* Free the ASID or VMID */
2552 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2553 /* Prevent SVA from touching the CD while we're freeing it */
2554 mutex_lock(&arm_smmu_asid_lock);
2555 xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid);
2556 mutex_unlock(&arm_smmu_asid_lock);
2557 } else {
2558 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2559 if (cfg->vmid)
2560 ida_free(&smmu->vmid_map, cfg->vmid);
2561 }
2562
2563 kfree(smmu_domain);
2564 }
2565
2566 static int arm_smmu_domain_finalise_s1(struct arm_smmu_device *smmu,
2567 struct arm_smmu_domain *smmu_domain)
2568 {
2569 int ret;
2570 u32 asid = 0;
2571 struct arm_smmu_ctx_desc *cd = &smmu_domain->cd;
2572
2573 /* Prevent SVA from modifying the ASID until it is written to the CD */
2574 mutex_lock(&arm_smmu_asid_lock);
2575 ret = xa_alloc(&arm_smmu_asid_xa, &asid, smmu_domain,
2576 XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
2577 cd->asid = (u16)asid;
2578 mutex_unlock(&arm_smmu_asid_lock);
2579 return ret;
2580 }
2581
2582 static int arm_smmu_domain_finalise_s2(struct arm_smmu_device *smmu,
2583 struct arm_smmu_domain *smmu_domain)
2584 {
2585 int vmid;
2586 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2587
2588 /* Reserve VMID 0 for stage-2 bypass STEs */
2589 vmid = ida_alloc_range(&smmu->vmid_map, 1, (1 << smmu->vmid_bits) - 1,
2590 GFP_KERNEL);
2591 if (vmid < 0)
2592 return vmid;
2593
2594 cfg->vmid = (u16)vmid;
2595 return 0;
2596 }
2597
2598 static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain,
2599 struct arm_smmu_device *smmu, u32 flags)
2600 {
2601 int ret;
2602 enum io_pgtable_fmt fmt;
2603 struct io_pgtable_cfg pgtbl_cfg;
2604 struct io_pgtable_ops *pgtbl_ops;
2605 int (*finalise_stage_fn)(struct arm_smmu_device *smmu,
2606 struct arm_smmu_domain *smmu_domain);
2607 bool enable_dirty = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
2608
2609 pgtbl_cfg = (struct io_pgtable_cfg) {
2610 .pgsize_bitmap = smmu->pgsize_bitmap,
2611 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENCY,
2612 .tlb = &arm_smmu_flush_ops,
2613 .iommu_dev = smmu->dev,
2614 };
2615
2616 switch (smmu_domain->stage) {
2617 case ARM_SMMU_DOMAIN_S1: {
2618 unsigned long ias = (smmu->features &
2619 ARM_SMMU_FEAT_VAX) ? 52 : 48;
2620
2621 pgtbl_cfg.ias = min_t(unsigned long, ias, VA_BITS);
2622 pgtbl_cfg.oas = smmu->ias;
2623 if (enable_dirty)
2624 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_ARM_HD;
2625 fmt = ARM_64_LPAE_S1;
2626 finalise_stage_fn = arm_smmu_domain_finalise_s1;
2627 break;
2628 }
2629 case ARM_SMMU_DOMAIN_S2:
2630 if (enable_dirty)
2631 return -EOPNOTSUPP;
2632 pgtbl_cfg.ias = smmu->ias;
2633 pgtbl_cfg.oas = smmu->oas;
2634 fmt = ARM_64_LPAE_S2;
2635 finalise_stage_fn = arm_smmu_domain_finalise_s2;
2636 if ((smmu->features & ARM_SMMU_FEAT_S2FWB) &&
2637 (flags & IOMMU_HWPT_ALLOC_NEST_PARENT))
2638 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_ARM_S2FWB;
2639 break;
2640 default:
2641 return -EINVAL;
2642 }
2643
2644 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
2645 if (!pgtbl_ops)
2646 return -ENOMEM;
2647
2648 smmu_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
2649 smmu_domain->domain.geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
2650 smmu_domain->domain.geometry.force_aperture = true;
2651 if (enable_dirty && smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
2652 smmu_domain->domain.dirty_ops = &arm_smmu_dirty_ops;
2653
2654 ret = finalise_stage_fn(smmu, smmu_domain);
2655 if (ret < 0) {
2656 free_io_pgtable_ops(pgtbl_ops);
2657 return ret;
2658 }
2659
2660 smmu_domain->pgtbl_ops = pgtbl_ops;
2661 smmu_domain->smmu = smmu;
2662 return 0;
2663 }
2664
2665 static struct arm_smmu_ste *
2666 arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
2667 {
2668 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2669
2670 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2671 /* Two-level walk */
2672 return &cfg->l2.l2ptrs[arm_smmu_strtab_l1_idx(sid)]
2673 ->stes[arm_smmu_strtab_l2_idx(sid)];
2674 } else {
2675 /* Simple linear lookup */
2676 return &cfg->linear.table[sid];
2677 }
2678 }
2679
2680 void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master,
2681 const struct arm_smmu_ste *target)
2682 {
2683 int i, j;
2684 struct arm_smmu_device *smmu = master->smmu;
2685
2686 master->cd_table.in_ste =
2687 FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(target->data[0])) ==
2688 STRTAB_STE_0_CFG_S1_TRANS;
2689 master->ste_ats_enabled =
2690 FIELD_GET(STRTAB_STE_1_EATS, le64_to_cpu(target->data[1])) ==
2691 STRTAB_STE_1_EATS_TRANS;
2692
2693 for (i = 0; i < master->num_streams; ++i) {
2694 u32 sid = master->streams[i].id;
2695 struct arm_smmu_ste *step =
2696 arm_smmu_get_step_for_sid(smmu, sid);
2697
2698 /* Bridged PCI devices may end up with duplicated IDs */
2699 for (j = 0; j < i; j++)
2700 if (master->streams[j].id == sid)
2701 break;
2702 if (j < i)
2703 continue;
2704
2705 arm_smmu_write_ste(master, sid, step, target);
2706 }
2707 }
2708
2709 static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
2710 {
2711 struct device *dev = master->dev;
2712 struct arm_smmu_device *smmu = master->smmu;
2713 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2714
2715 if (!(smmu->features & ARM_SMMU_FEAT_ATS))
2716 return false;
2717
2718 if (!(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS))
2719 return false;
2720
2721 return dev_is_pci(dev) && pci_ats_supported(to_pci_dev(dev));
2722 }
2723
2724 static void arm_smmu_enable_ats(struct arm_smmu_master *master)
2725 {
2726 size_t stu;
2727 struct pci_dev *pdev;
2728 struct arm_smmu_device *smmu = master->smmu;
2729
2730 /* Smallest Translation Unit: log2 of the smallest supported granule */
2731 stu = __ffs(smmu->pgsize_bitmap);
2732 pdev = to_pci_dev(master->dev);
2733
2734 /*
2735 * ATC invalidation of PASID 0 causes the entire ATC to be flushed.
2736 */
2737 arm_smmu_atc_inv_master(master, IOMMU_NO_PASID);
2738 if (pci_enable_ats(pdev, stu))
2739 dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu);
2740 }
2741
2742 static int arm_smmu_enable_pasid(struct arm_smmu_master *master)
2743 {
2744 int ret;
2745 int features;
2746 int num_pasids;
2747 struct pci_dev *pdev;
2748
2749 if (!dev_is_pci(master->dev))
2750 return -ENODEV;
2751
2752 pdev = to_pci_dev(master->dev);
2753
2754 features = pci_pasid_features(pdev);
2755 if (features < 0)
2756 return features;
2757
2758 num_pasids = pci_max_pasids(pdev);
2759 if (num_pasids <= 0)
2760 return num_pasids;
2761
2762 ret = pci_enable_pasid(pdev, features);
2763 if (ret) {
2764 dev_err(&pdev->dev, "Failed to enable PASID\n");
2765 return ret;
2766 }
2767
2768 master->ssid_bits = min_t(u8, ilog2(num_pasids),
2769 master->smmu->ssid_bits);
2770 return 0;
2771 }
2772
2773 static void arm_smmu_disable_pasid(struct arm_smmu_master *master)
2774 {
2775 struct pci_dev *pdev;
2776
2777 if (!dev_is_pci(master->dev))
2778 return;
2779
2780 pdev = to_pci_dev(master->dev);
2781
2782 if (!pdev->pasid_enabled)
2783 return;
2784
2785 master->ssid_bits = 0;
2786 pci_disable_pasid(pdev);
2787 }
2788
2789 static struct arm_smmu_master_domain *
2790 arm_smmu_find_master_domain(struct arm_smmu_domain *smmu_domain,
2791 struct iommu_domain *domain,
2792 struct arm_smmu_master *master,
2793 ioasid_t ssid, bool nested_ats_flush)
2794 {
2795 struct arm_smmu_master_domain *master_domain;
2796
2797 lockdep_assert_held(&smmu_domain->devices_lock);
2798
2799 list_for_each_entry(master_domain, &smmu_domain->devices,
2800 devices_elm) {
2801 if (master_domain->master == master &&
2802 master_domain->domain == domain &&
2803 master_domain->ssid == ssid &&
2804 master_domain->nested_ats_flush == nested_ats_flush)
2805 return master_domain;
2806 }
2807 return NULL;
2808 }
2809
2810 /*
2811 * If the domain uses the smmu_domain->devices list return the arm_smmu_domain
2812 * structure, otherwise NULL. These domains track attached devices so they can
2813 * issue invalidations.
2814 */
2815 static struct arm_smmu_domain *
2816 to_smmu_domain_devices(struct iommu_domain *domain)
2817 {
2818 /* The domain can be NULL only when processing the first attach */
2819 if (!domain)
2820 return NULL;
2821 if ((domain->type & __IOMMU_DOMAIN_PAGING) ||
2822 domain->type == IOMMU_DOMAIN_SVA)
2823 return to_smmu_domain(domain);
2824 if (domain->type == IOMMU_DOMAIN_NESTED)
2825 return to_smmu_nested_domain(domain)->vsmmu->s2_parent;
2826 return NULL;
2827 }
2828
2829 static int arm_smmu_enable_iopf(struct arm_smmu_master *master,
2830 struct arm_smmu_master_domain *master_domain)
2831 {
2832 int ret;
2833
2834 iommu_group_mutex_assert(master->dev);
2835
2836 if (!IS_ENABLED(CONFIG_ARM_SMMU_V3_SVA))
2837 return -EOPNOTSUPP;
2838
2839 /*
2840 * Drivers for devices supporting PRI or stall require iopf others have
2841 * device-specific fault handlers and don't need IOPF, so this is not a
2842 * failure.
2843 */
2844 if (!master->stall_enabled)
2845 return 0;
2846
2847 /* We're not keeping track of SIDs in fault events */
2848 if (master->num_streams != 1)
2849 return -EOPNOTSUPP;
2850
2851 if (master->iopf_refcount) {
2852 master->iopf_refcount++;
2853 master_domain->using_iopf = true;
2854 return 0;
2855 }
2856
2857 ret = iopf_queue_add_device(master->smmu->evtq.iopf, master->dev);
2858 if (ret)
2859 return ret;
2860 master->iopf_refcount = 1;
2861 master_domain->using_iopf = true;
2862 return 0;
2863 }
2864
2865 static void arm_smmu_disable_iopf(struct arm_smmu_master *master,
2866 struct arm_smmu_master_domain *master_domain)
2867 {
2868 iommu_group_mutex_assert(master->dev);
2869
2870 if (!IS_ENABLED(CONFIG_ARM_SMMU_V3_SVA))
2871 return;
2872
2873 if (!master_domain || !master_domain->using_iopf)
2874 return;
2875
2876 master->iopf_refcount--;
2877 if (master->iopf_refcount == 0)
2878 iopf_queue_remove_device(master->smmu->evtq.iopf, master->dev);
2879 }
2880
2881 static void arm_smmu_remove_master_domain(struct arm_smmu_master *master,
2882 struct iommu_domain *domain,
2883 ioasid_t ssid)
2884 {
2885 struct arm_smmu_domain *smmu_domain = to_smmu_domain_devices(domain);
2886 struct arm_smmu_master_domain *master_domain;
2887 bool nested_ats_flush = false;
2888 unsigned long flags;
2889
2890 if (!smmu_domain)
2891 return;
2892
2893 if (domain->type == IOMMU_DOMAIN_NESTED)
2894 nested_ats_flush = to_smmu_nested_domain(domain)->enable_ats;
2895
2896 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2897 master_domain = arm_smmu_find_master_domain(smmu_domain, domain, master,
2898 ssid, nested_ats_flush);
2899 if (master_domain) {
2900 list_del(&master_domain->devices_elm);
2901 if (master->ats_enabled)
2902 atomic_dec(&smmu_domain->nr_ats_masters);
2903 }
2904 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2905
2906 arm_smmu_disable_iopf(master, master_domain);
2907 kfree(master_domain);
2908 }
2909
2910 /*
2911 * Start the sequence to attach a domain to a master. The sequence contains three
2912 * steps:
2913 * arm_smmu_attach_prepare()
2914 * arm_smmu_install_ste_for_dev()
2915 * arm_smmu_attach_commit()
2916 *
2917 * If prepare succeeds then the sequence must be completed. The STE installed
2918 * must set the STE.EATS field according to state.ats_enabled.
2919 *
2920 * If the device supports ATS then this determines if EATS should be enabled
2921 * in the STE, and starts sequencing EATS disable if required.
2922 *
2923 * The change of the EATS in the STE and the PCI ATS config space is managed by
2924 * this sequence to be in the right order so that if PCI ATS is enabled then
2925 * STE.ETAS is enabled.
2926 *
2927 * new_domain can be a non-paging domain. In this case ATS will not be enabled,
2928 * and invalidations won't be tracked.
2929 */
2930 int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state,
2931 struct iommu_domain *new_domain)
2932 {
2933 struct arm_smmu_master *master = state->master;
2934 struct arm_smmu_master_domain *master_domain;
2935 struct arm_smmu_domain *smmu_domain =
2936 to_smmu_domain_devices(new_domain);
2937 unsigned long flags;
2938 int ret;
2939
2940 /*
2941 * arm_smmu_share_asid() must not see two domains pointing to the same
2942 * arm_smmu_master_domain contents otherwise it could randomly write one
2943 * or the other to the CD.
2944 */
2945 lockdep_assert_held(&arm_smmu_asid_lock);
2946
2947 if (smmu_domain || state->cd_needs_ats) {
2948 /*
2949 * The SMMU does not support enabling ATS with bypass/abort.
2950 * When the STE is in bypass (STE.Config[2:0] == 0b100), ATS
2951 * Translation Requests and Translated transactions are denied
2952 * as though ATS is disabled for the stream (STE.EATS == 0b00),
2953 * causing F_BAD_ATS_TREQ and F_TRANSL_FORBIDDEN events
2954 * (IHI0070Ea 5.2 Stream Table Entry).
2955 *
2956 * However, if we have installed a CD table and are using S1DSS
2957 * then ATS will work in S1DSS bypass. See "13.6.4 Full ATS
2958 * skipping stage 1".
2959 *
2960 * Disable ATS if we are going to create a normal 0b100 bypass
2961 * STE.
2962 */
2963 state->ats_enabled = !state->disable_ats &&
2964 arm_smmu_ats_supported(master);
2965 }
2966
2967 if (smmu_domain) {
2968 if (new_domain->type == IOMMU_DOMAIN_NESTED) {
2969 ret = arm_smmu_attach_prepare_vmaster(
2970 state, to_smmu_nested_domain(new_domain));
2971 if (ret)
2972 return ret;
2973 }
2974
2975 master_domain = kzalloc(sizeof(*master_domain), GFP_KERNEL);
2976 if (!master_domain) {
2977 ret = -ENOMEM;
2978 goto err_free_vmaster;
2979 }
2980 master_domain->domain = new_domain;
2981 master_domain->master = master;
2982 master_domain->ssid = state->ssid;
2983 if (new_domain->type == IOMMU_DOMAIN_NESTED)
2984 master_domain->nested_ats_flush =
2985 to_smmu_nested_domain(new_domain)->enable_ats;
2986
2987 if (new_domain->iopf_handler) {
2988 ret = arm_smmu_enable_iopf(master, master_domain);
2989 if (ret)
2990 goto err_free_master_domain;
2991 }
2992
2993 /*
2994 * During prepare we want the current smmu_domain and new
2995 * smmu_domain to be in the devices list before we change any
2996 * HW. This ensures that both domains will send ATS
2997 * invalidations to the master until we are done.
2998 *
2999 * It is tempting to make this list only track masters that are
3000 * using ATS, but arm_smmu_share_asid() also uses this to change
3001 * the ASID of a domain, unrelated to ATS.
3002 *
3003 * Notice if we are re-attaching the same domain then the list
3004 * will have two identical entries and commit will remove only
3005 * one of them.
3006 */
3007 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
3008 if (smmu_domain->enforce_cache_coherency &&
3009 !arm_smmu_master_canwbs(master)) {
3010 spin_unlock_irqrestore(&smmu_domain->devices_lock,
3011 flags);
3012 ret = -EINVAL;
3013 goto err_iopf;
3014 }
3015
3016 if (state->ats_enabled)
3017 atomic_inc(&smmu_domain->nr_ats_masters);
3018 list_add(&master_domain->devices_elm, &smmu_domain->devices);
3019 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
3020 }
3021
3022 if (!state->ats_enabled && master->ats_enabled) {
3023 pci_disable_ats(to_pci_dev(master->dev));
3024 /*
3025 * This is probably overkill, but the config write for disabling
3026 * ATS should complete before the STE is configured to generate
3027 * UR to avoid AER noise.
3028 */
3029 wmb();
3030 }
3031 return 0;
3032
3033 err_iopf:
3034 arm_smmu_disable_iopf(master, master_domain);
3035 err_free_master_domain:
3036 kfree(master_domain);
3037 err_free_vmaster:
3038 kfree(state->vmaster);
3039 return ret;
3040 }
3041
3042 /*
3043 * Commit is done after the STE/CD are configured with the EATS setting. It
3044 * completes synchronizing the PCI device's ATC and finishes manipulating the
3045 * smmu_domain->devices list.
3046 */
3047 void arm_smmu_attach_commit(struct arm_smmu_attach_state *state)
3048 {
3049 struct arm_smmu_master *master = state->master;
3050
3051 lockdep_assert_held(&arm_smmu_asid_lock);
3052
3053 arm_smmu_attach_commit_vmaster(state);
3054
3055 if (state->ats_enabled && !master->ats_enabled) {
3056 arm_smmu_enable_ats(master);
3057 } else if (state->ats_enabled && master->ats_enabled) {
3058 /*
3059 * The translation has changed, flush the ATC. At this point the
3060 * SMMU is translating for the new domain and both the old&new
3061 * domain will issue invalidations.
3062 */
3063 arm_smmu_atc_inv_master(master, state->ssid);
3064 } else if (!state->ats_enabled && master->ats_enabled) {
3065 /* ATS is being switched off, invalidate the entire ATC */
3066 arm_smmu_atc_inv_master(master, IOMMU_NO_PASID);
3067 }
3068
3069 arm_smmu_remove_master_domain(master, state->old_domain, state->ssid);
3070 master->ats_enabled = state->ats_enabled;
3071 }
3072
3073 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
3074 {
3075 int ret = 0;
3076 struct arm_smmu_ste target;
3077 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
3078 struct arm_smmu_device *smmu;
3079 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3080 struct arm_smmu_attach_state state = {
3081 .old_domain = iommu_get_domain_for_dev(dev),
3082 .ssid = IOMMU_NO_PASID,
3083 };
3084 struct arm_smmu_master *master;
3085 struct arm_smmu_cd *cdptr;
3086
3087 if (!fwspec)
3088 return -ENOENT;
3089
3090 state.master = master = dev_iommu_priv_get(dev);
3091 smmu = master->smmu;
3092
3093 if (smmu_domain->smmu != smmu)
3094 return -EINVAL;
3095
3096 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
3097 cdptr = arm_smmu_alloc_cd_ptr(master, IOMMU_NO_PASID);
3098 if (!cdptr)
3099 return -ENOMEM;
3100 } else if (arm_smmu_ssids_in_use(&master->cd_table))
3101 return -EBUSY;
3102
3103 /*
3104 * Prevent arm_smmu_share_asid() from trying to change the ASID
3105 * of either the old or new domain while we are working on it.
3106 * This allows the STE and the smmu_domain->devices list to
3107 * be inconsistent during this routine.
3108 */
3109 mutex_lock(&arm_smmu_asid_lock);
3110
3111 ret = arm_smmu_attach_prepare(&state, domain);
3112 if (ret) {
3113 mutex_unlock(&arm_smmu_asid_lock);
3114 return ret;
3115 }
3116
3117 switch (smmu_domain->stage) {
3118 case ARM_SMMU_DOMAIN_S1: {
3119 struct arm_smmu_cd target_cd;
3120
3121 arm_smmu_make_s1_cd(&target_cd, master, smmu_domain);
3122 arm_smmu_write_cd_entry(master, IOMMU_NO_PASID, cdptr,
3123 &target_cd);
3124 arm_smmu_make_cdtable_ste(&target, master, state.ats_enabled,
3125 STRTAB_STE_1_S1DSS_SSID0);
3126 arm_smmu_install_ste_for_dev(master, &target);
3127 break;
3128 }
3129 case ARM_SMMU_DOMAIN_S2:
3130 arm_smmu_make_s2_domain_ste(&target, master, smmu_domain,
3131 state.ats_enabled);
3132 arm_smmu_install_ste_for_dev(master, &target);
3133 arm_smmu_clear_cd(master, IOMMU_NO_PASID);
3134 break;
3135 }
3136
3137 arm_smmu_attach_commit(&state);
3138 mutex_unlock(&arm_smmu_asid_lock);
3139 return 0;
3140 }
3141
3142 static int arm_smmu_s1_set_dev_pasid(struct iommu_domain *domain,
3143 struct device *dev, ioasid_t id,
3144 struct iommu_domain *old)
3145 {
3146 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3147 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3148 struct arm_smmu_device *smmu = master->smmu;
3149 struct arm_smmu_cd target_cd;
3150
3151 if (smmu_domain->smmu != smmu)
3152 return -EINVAL;
3153
3154 if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1)
3155 return -EINVAL;
3156
3157 /*
3158 * We can read cd.asid outside the lock because arm_smmu_set_pasid()
3159 * will fix it
3160 */
3161 arm_smmu_make_s1_cd(&target_cd, master, smmu_domain);
3162 return arm_smmu_set_pasid(master, to_smmu_domain(domain), id,
3163 &target_cd, old);
3164 }
3165
3166 static void arm_smmu_update_ste(struct arm_smmu_master *master,
3167 struct iommu_domain *sid_domain,
3168 bool ats_enabled)
3169 {
3170 unsigned int s1dss = STRTAB_STE_1_S1DSS_TERMINATE;
3171 struct arm_smmu_ste ste;
3172
3173 if (master->cd_table.in_ste && master->ste_ats_enabled == ats_enabled)
3174 return;
3175
3176 if (sid_domain->type == IOMMU_DOMAIN_IDENTITY)
3177 s1dss = STRTAB_STE_1_S1DSS_BYPASS;
3178 else
3179 WARN_ON(sid_domain->type != IOMMU_DOMAIN_BLOCKED);
3180
3181 /*
3182 * Change the STE into a cdtable one with SID IDENTITY/BLOCKED behavior
3183 * using s1dss if necessary. If the cd_table is already installed then
3184 * the S1DSS is correct and this will just update the EATS. Otherwise it
3185 * installs the entire thing. This will be hitless.
3186 */
3187 arm_smmu_make_cdtable_ste(&ste, master, ats_enabled, s1dss);
3188 arm_smmu_install_ste_for_dev(master, &ste);
3189 }
3190
3191 int arm_smmu_set_pasid(struct arm_smmu_master *master,
3192 struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
3193 struct arm_smmu_cd *cd, struct iommu_domain *old)
3194 {
3195 struct iommu_domain *sid_domain = iommu_get_domain_for_dev(master->dev);
3196 struct arm_smmu_attach_state state = {
3197 .master = master,
3198 .ssid = pasid,
3199 .old_domain = old,
3200 };
3201 struct arm_smmu_cd *cdptr;
3202 int ret;
3203
3204 /* The core code validates pasid */
3205
3206 if (smmu_domain->smmu != master->smmu)
3207 return -EINVAL;
3208
3209 if (!master->cd_table.in_ste &&
3210 sid_domain->type != IOMMU_DOMAIN_IDENTITY &&
3211 sid_domain->type != IOMMU_DOMAIN_BLOCKED)
3212 return -EINVAL;
3213
3214 cdptr = arm_smmu_alloc_cd_ptr(master, pasid);
3215 if (!cdptr)
3216 return -ENOMEM;
3217
3218 mutex_lock(&arm_smmu_asid_lock);
3219 ret = arm_smmu_attach_prepare(&state, &smmu_domain->domain);
3220 if (ret)
3221 goto out_unlock;
3222
3223 /*
3224 * We don't want to obtain to the asid_lock too early, so fix up the
3225 * caller set ASID under the lock in case it changed.
3226 */
3227 cd->data[0] &= ~cpu_to_le64(CTXDESC_CD_0_ASID);
3228 cd->data[0] |= cpu_to_le64(
3229 FIELD_PREP(CTXDESC_CD_0_ASID, smmu_domain->cd.asid));
3230
3231 arm_smmu_write_cd_entry(master, pasid, cdptr, cd);
3232 arm_smmu_update_ste(master, sid_domain, state.ats_enabled);
3233
3234 arm_smmu_attach_commit(&state);
3235
3236 out_unlock:
3237 mutex_unlock(&arm_smmu_asid_lock);
3238 return ret;
3239 }
3240
3241 static int arm_smmu_blocking_set_dev_pasid(struct iommu_domain *new_domain,
3242 struct device *dev, ioasid_t pasid,
3243 struct iommu_domain *old_domain)
3244 {
3245 struct arm_smmu_domain *smmu_domain = to_smmu_domain(old_domain);
3246 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3247
3248 mutex_lock(&arm_smmu_asid_lock);
3249 arm_smmu_clear_cd(master, pasid);
3250 if (master->ats_enabled)
3251 arm_smmu_atc_inv_master(master, pasid);
3252 arm_smmu_remove_master_domain(master, &smmu_domain->domain, pasid);
3253 mutex_unlock(&arm_smmu_asid_lock);
3254
3255 /*
3256 * When the last user of the CD table goes away downgrade the STE back
3257 * to a non-cd_table one.
3258 */
3259 if (!arm_smmu_ssids_in_use(&master->cd_table)) {
3260 struct iommu_domain *sid_domain =
3261 iommu_get_domain_for_dev(master->dev);
3262
3263 if (sid_domain->type == IOMMU_DOMAIN_IDENTITY ||
3264 sid_domain->type == IOMMU_DOMAIN_BLOCKED)
3265 sid_domain->ops->attach_dev(sid_domain, dev);
3266 }
3267 return 0;
3268 }
3269
3270 static void arm_smmu_attach_dev_ste(struct iommu_domain *domain,
3271 struct device *dev,
3272 struct arm_smmu_ste *ste,
3273 unsigned int s1dss)
3274 {
3275 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3276 struct arm_smmu_attach_state state = {
3277 .master = master,
3278 .old_domain = iommu_get_domain_for_dev(dev),
3279 .ssid = IOMMU_NO_PASID,
3280 };
3281
3282 /*
3283 * Do not allow any ASID to be changed while are working on the STE,
3284 * otherwise we could miss invalidations.
3285 */
3286 mutex_lock(&arm_smmu_asid_lock);
3287
3288 /*
3289 * If the CD table is not in use we can use the provided STE, otherwise
3290 * we use a cdtable STE with the provided S1DSS.
3291 */
3292 if (arm_smmu_ssids_in_use(&master->cd_table)) {
3293 /*
3294 * If a CD table has to be present then we need to run with ATS
3295 * on because we have to assume a PASID is using ATS. For
3296 * IDENTITY this will setup things so that S1DSS=bypass which
3297 * follows the explanation in "13.6.4 Full ATS skipping stage 1"
3298 * and allows for ATS on the RID to work.
3299 */
3300 state.cd_needs_ats = true;
3301 arm_smmu_attach_prepare(&state, domain);
3302 arm_smmu_make_cdtable_ste(ste, master, state.ats_enabled, s1dss);
3303 } else {
3304 arm_smmu_attach_prepare(&state, domain);
3305 }
3306 arm_smmu_install_ste_for_dev(master, ste);
3307 arm_smmu_attach_commit(&state);
3308 mutex_unlock(&arm_smmu_asid_lock);
3309
3310 /*
3311 * This has to be done after removing the master from the
3312 * arm_smmu_domain->devices to avoid races updating the same context
3313 * descriptor from arm_smmu_share_asid().
3314 */
3315 arm_smmu_clear_cd(master, IOMMU_NO_PASID);
3316 }
3317
3318 static int arm_smmu_attach_dev_identity(struct iommu_domain *domain,
3319 struct device *dev)
3320 {
3321 struct arm_smmu_ste ste;
3322 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3323
3324 arm_smmu_master_clear_vmaster(master);
3325 arm_smmu_make_bypass_ste(master->smmu, &ste);
3326 arm_smmu_attach_dev_ste(domain, dev, &ste, STRTAB_STE_1_S1DSS_BYPASS);
3327 return 0;
3328 }
3329
3330 static const struct iommu_domain_ops arm_smmu_identity_ops = {
3331 .attach_dev = arm_smmu_attach_dev_identity,
3332 };
3333
3334 static struct iommu_domain arm_smmu_identity_domain = {
3335 .type = IOMMU_DOMAIN_IDENTITY,
3336 .ops = &arm_smmu_identity_ops,
3337 };
3338
3339 static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain,
3340 struct device *dev)
3341 {
3342 struct arm_smmu_ste ste;
3343 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3344
3345 arm_smmu_master_clear_vmaster(master);
3346 arm_smmu_make_abort_ste(&ste);
3347 arm_smmu_attach_dev_ste(domain, dev, &ste,
3348 STRTAB_STE_1_S1DSS_TERMINATE);
3349 return 0;
3350 }
3351
3352 static const struct iommu_domain_ops arm_smmu_blocked_ops = {
3353 .attach_dev = arm_smmu_attach_dev_blocked,
3354 .set_dev_pasid = arm_smmu_blocking_set_dev_pasid,
3355 };
3356
3357 static struct iommu_domain arm_smmu_blocked_domain = {
3358 .type = IOMMU_DOMAIN_BLOCKED,
3359 .ops = &arm_smmu_blocked_ops,
3360 };
3361
3362 static struct iommu_domain *
3363 arm_smmu_domain_alloc_paging_flags(struct device *dev, u32 flags,
3364 const struct iommu_user_data *user_data)
3365 {
3366 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3367 struct arm_smmu_device *smmu = master->smmu;
3368 const u32 PAGING_FLAGS = IOMMU_HWPT_ALLOC_DIRTY_TRACKING |
3369 IOMMU_HWPT_ALLOC_PASID |
3370 IOMMU_HWPT_ALLOC_NEST_PARENT;
3371 struct arm_smmu_domain *smmu_domain;
3372 int ret;
3373
3374 if (flags & ~PAGING_FLAGS)
3375 return ERR_PTR(-EOPNOTSUPP);
3376 if (user_data)
3377 return ERR_PTR(-EOPNOTSUPP);
3378
3379 smmu_domain = arm_smmu_domain_alloc();
3380 if (IS_ERR(smmu_domain))
3381 return ERR_CAST(smmu_domain);
3382
3383 switch (flags) {
3384 case 0:
3385 /* Prefer S1 if available */
3386 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
3387 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
3388 else
3389 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
3390 break;
3391 case IOMMU_HWPT_ALLOC_NEST_PARENT:
3392 if (!(smmu->features & ARM_SMMU_FEAT_NESTING)) {
3393 ret = -EOPNOTSUPP;
3394 goto err_free;
3395 }
3396 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
3397 smmu_domain->nest_parent = true;
3398 break;
3399 case IOMMU_HWPT_ALLOC_DIRTY_TRACKING:
3400 case IOMMU_HWPT_ALLOC_DIRTY_TRACKING | IOMMU_HWPT_ALLOC_PASID:
3401 case IOMMU_HWPT_ALLOC_PASID:
3402 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) {
3403 ret = -EOPNOTSUPP;
3404 goto err_free;
3405 }
3406 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
3407 break;
3408 default:
3409 ret = -EOPNOTSUPP;
3410 goto err_free;
3411 }
3412
3413 smmu_domain->domain.type = IOMMU_DOMAIN_UNMANAGED;
3414 smmu_domain->domain.ops = arm_smmu_ops.default_domain_ops;
3415 ret = arm_smmu_domain_finalise(smmu_domain, smmu, flags);
3416 if (ret)
3417 goto err_free;
3418 return &smmu_domain->domain;
3419
3420 err_free:
3421 kfree(smmu_domain);
3422 return ERR_PTR(ret);
3423 }
3424
3425 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
3426 phys_addr_t paddr, size_t pgsize, size_t pgcount,
3427 int prot, gfp_t gfp, size_t *mapped)
3428 {
3429 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
3430
3431 if (!ops)
3432 return -ENODEV;
3433
3434 return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
3435 }
3436
3437 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
3438 size_t pgsize, size_t pgcount,
3439 struct iommu_iotlb_gather *gather)
3440 {
3441 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3442 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
3443
3444 if (!ops)
3445 return 0;
3446
3447 return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
3448 }
3449
3450 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
3451 {
3452 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3453
3454 if (smmu_domain->smmu)
3455 arm_smmu_tlb_inv_context(smmu_domain);
3456 }
3457
3458 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
3459 struct iommu_iotlb_gather *gather)
3460 {
3461 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3462
3463 if (!gather->pgsize)
3464 return;
3465
3466 arm_smmu_tlb_inv_range_domain(gather->start,
3467 gather->end - gather->start + 1,
3468 gather->pgsize, true, smmu_domain);
3469 }
3470
3471 static phys_addr_t
3472 arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
3473 {
3474 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
3475
3476 if (!ops)
3477 return 0;
3478
3479 return ops->iova_to_phys(ops, iova);
3480 }
3481
3482 static struct platform_driver arm_smmu_driver;
3483
3484 static
3485 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
3486 {
3487 struct device *dev = bus_find_device_by_fwnode(&platform_bus_type, fwnode);
3488
3489 put_device(dev);
3490 return dev ? dev_get_drvdata(dev) : NULL;
3491 }
3492
3493 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
3494 {
3495 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3496 return arm_smmu_strtab_l1_idx(sid) < smmu->strtab_cfg.l2.num_l1_ents;
3497 return sid < smmu->strtab_cfg.linear.num_ents;
3498 }
3499
3500 static int arm_smmu_init_sid_strtab(struct arm_smmu_device *smmu, u32 sid)
3501 {
3502 /* Check the SIDs are in range of the SMMU and our stream table */
3503 if (!arm_smmu_sid_in_range(smmu, sid))
3504 return -ERANGE;
3505
3506 /* Ensure l2 strtab is initialised */
3507 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3508 return arm_smmu_init_l2_strtab(smmu, sid);
3509
3510 return 0;
3511 }
3512
3513 static int arm_smmu_insert_master(struct arm_smmu_device *smmu,
3514 struct arm_smmu_master *master)
3515 {
3516 int i;
3517 int ret = 0;
3518 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
3519
3520 master->streams = kcalloc(fwspec->num_ids, sizeof(*master->streams),
3521 GFP_KERNEL);
3522 if (!master->streams)
3523 return -ENOMEM;
3524 master->num_streams = fwspec->num_ids;
3525
3526 mutex_lock(&smmu->streams_mutex);
3527 for (i = 0; i < fwspec->num_ids; i++) {
3528 struct arm_smmu_stream *new_stream = &master->streams[i];
3529 struct rb_node *existing;
3530 u32 sid = fwspec->ids[i];
3531
3532 new_stream->id = sid;
3533 new_stream->master = master;
3534
3535 ret = arm_smmu_init_sid_strtab(smmu, sid);
3536 if (ret)
3537 break;
3538
3539 /* Insert into SID tree */
3540 existing = rb_find_add(&new_stream->node, &smmu->streams,
3541 arm_smmu_streams_cmp_node);
3542 if (existing) {
3543 struct arm_smmu_master *existing_master =
3544 rb_entry(existing, struct arm_smmu_stream, node)
3545 ->master;
3546
3547 /* Bridged PCI devices may end up with duplicated IDs */
3548 if (existing_master == master)
3549 continue;
3550
3551 dev_warn(master->dev,
3552 "Aliasing StreamID 0x%x (from %s) unsupported, expect DMA to be broken\n",
3553 sid, dev_name(existing_master->dev));
3554 ret = -ENODEV;
3555 break;
3556 }
3557 }
3558
3559 if (ret) {
3560 for (i--; i >= 0; i--)
3561 rb_erase(&master->streams[i].node, &smmu->streams);
3562 kfree(master->streams);
3563 }
3564 mutex_unlock(&smmu->streams_mutex);
3565
3566 return ret;
3567 }
3568
3569 static void arm_smmu_remove_master(struct arm_smmu_master *master)
3570 {
3571 int i;
3572 struct arm_smmu_device *smmu = master->smmu;
3573 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
3574
3575 if (!smmu || !master->streams)
3576 return;
3577
3578 mutex_lock(&smmu->streams_mutex);
3579 for (i = 0; i < fwspec->num_ids; i++)
3580 rb_erase(&master->streams[i].node, &smmu->streams);
3581 mutex_unlock(&smmu->streams_mutex);
3582
3583 kfree(master->streams);
3584 }
3585
3586 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
3587 {
3588 int ret;
3589 struct arm_smmu_device *smmu;
3590 struct arm_smmu_master *master;
3591 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
3592
3593 if (WARN_ON_ONCE(dev_iommu_priv_get(dev)))
3594 return ERR_PTR(-EBUSY);
3595
3596 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
3597 if (!smmu)
3598 return ERR_PTR(-ENODEV);
3599
3600 master = kzalloc(sizeof(*master), GFP_KERNEL);
3601 if (!master)
3602 return ERR_PTR(-ENOMEM);
3603
3604 master->dev = dev;
3605 master->smmu = smmu;
3606 dev_iommu_priv_set(dev, master);
3607
3608 ret = arm_smmu_insert_master(smmu, master);
3609 if (ret)
3610 goto err_free_master;
3611
3612 device_property_read_u32(dev, "pasid-num-bits", &master->ssid_bits);
3613 master->ssid_bits = min(smmu->ssid_bits, master->ssid_bits);
3614
3615 /*
3616 * Note that PASID must be enabled before, and disabled after ATS:
3617 * PCI Express Base 4.0r1.0 - 10.5.1.3 ATS Control Register
3618 *
3619 * Behavior is undefined if this bit is Set and the value of the PASID
3620 * Enable, Execute Requested Enable, or Privileged Mode Requested bits
3621 * are changed.
3622 */
3623 arm_smmu_enable_pasid(master);
3624
3625 if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB))
3626 master->ssid_bits = min_t(u8, master->ssid_bits,
3627 CTXDESC_LINEAR_CDMAX);
3628
3629 if ((smmu->features & ARM_SMMU_FEAT_STALLS &&
3630 device_property_read_bool(dev, "dma-can-stall")) ||
3631 smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
3632 master->stall_enabled = true;
3633
3634 if (dev_is_pci(dev)) {
3635 unsigned int stu = __ffs(smmu->pgsize_bitmap);
3636
3637 pci_prepare_ats(to_pci_dev(dev), stu);
3638 }
3639
3640 return &smmu->iommu;
3641
3642 err_free_master:
3643 kfree(master);
3644 return ERR_PTR(ret);
3645 }
3646
3647 static void arm_smmu_release_device(struct device *dev)
3648 {
3649 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3650
3651 WARN_ON(master->iopf_refcount);
3652
3653 /* Put the STE back to what arm_smmu_init_strtab() sets */
3654 if (dev->iommu->require_direct)
3655 arm_smmu_attach_dev_identity(&arm_smmu_identity_domain, dev);
3656 else
3657 arm_smmu_attach_dev_blocked(&arm_smmu_blocked_domain, dev);
3658
3659 arm_smmu_disable_pasid(master);
3660 arm_smmu_remove_master(master);
3661 if (arm_smmu_cdtab_allocated(&master->cd_table))
3662 arm_smmu_free_cd_tables(master);
3663 kfree(master);
3664 }
3665
3666 static int arm_smmu_read_and_clear_dirty(struct iommu_domain *domain,
3667 unsigned long iova, size_t size,
3668 unsigned long flags,
3669 struct iommu_dirty_bitmap *dirty)
3670 {
3671 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3672 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
3673
3674 return ops->read_and_clear_dirty(ops, iova, size, flags, dirty);
3675 }
3676
3677 static int arm_smmu_set_dirty_tracking(struct iommu_domain *domain,
3678 bool enabled)
3679 {
3680 /*
3681 * Always enabled and the dirty bitmap is cleared prior to
3682 * set_dirty_tracking().
3683 */
3684 return 0;
3685 }
3686
3687 static struct iommu_group *arm_smmu_device_group(struct device *dev)
3688 {
3689 struct iommu_group *group;
3690
3691 /*
3692 * We don't support devices sharing stream IDs other than PCI RID
3693 * aliases, since the necessary ID-to-device lookup becomes rather
3694 * impractical given a potential sparse 32-bit stream ID space.
3695 */
3696 if (dev_is_pci(dev))
3697 group = pci_device_group(dev);
3698 else
3699 group = generic_device_group(dev);
3700
3701 return group;
3702 }
3703
3704 static int arm_smmu_of_xlate(struct device *dev,
3705 const struct of_phandle_args *args)
3706 {
3707 return iommu_fwspec_add_ids(dev, args->args, 1);
3708 }
3709
3710 static void arm_smmu_get_resv_regions(struct device *dev,
3711 struct list_head *head)
3712 {
3713 struct iommu_resv_region *region;
3714 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
3715
3716 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
3717 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
3718 if (!region)
3719 return;
3720
3721 list_add_tail(®ion->list, head);
3722
3723 iommu_dma_get_resv_regions(dev, head);
3724 }
3725
3726 /*
3727 * HiSilicon PCIe tune and trace device can be used to trace TLP headers on the
3728 * PCIe link and save the data to memory by DMA. The hardware is restricted to
3729 * use identity mapping only.
3730 */
3731 #define IS_HISI_PTT_DEVICE(pdev) ((pdev)->vendor == PCI_VENDOR_ID_HUAWEI && \
3732 (pdev)->device == 0xa12e)
3733
3734 static int arm_smmu_def_domain_type(struct device *dev)
3735 {
3736 if (dev_is_pci(dev)) {
3737 struct pci_dev *pdev = to_pci_dev(dev);
3738
3739 if (IS_HISI_PTT_DEVICE(pdev))
3740 return IOMMU_DOMAIN_IDENTITY;
3741 }
3742
3743 return 0;
3744 }
3745
3746 static const struct iommu_ops arm_smmu_ops = {
3747 .identity_domain = &arm_smmu_identity_domain,
3748 .blocked_domain = &arm_smmu_blocked_domain,
3749 .capable = arm_smmu_capable,
3750 .hw_info = arm_smmu_hw_info,
3751 .domain_alloc_sva = arm_smmu_sva_domain_alloc,
3752 .domain_alloc_paging_flags = arm_smmu_domain_alloc_paging_flags,
3753 .probe_device = arm_smmu_probe_device,
3754 .release_device = arm_smmu_release_device,
3755 .device_group = arm_smmu_device_group,
3756 .of_xlate = arm_smmu_of_xlate,
3757 .get_resv_regions = arm_smmu_get_resv_regions,
3758 .page_response = arm_smmu_page_response,
3759 .def_domain_type = arm_smmu_def_domain_type,
3760 .get_viommu_size = arm_smmu_get_viommu_size,
3761 .viommu_init = arm_vsmmu_init,
3762 .user_pasid_table = 1,
3763 .owner = THIS_MODULE,
3764 .default_domain_ops = &(const struct iommu_domain_ops) {
3765 .attach_dev = arm_smmu_attach_dev,
3766 .enforce_cache_coherency = arm_smmu_enforce_cache_coherency,
3767 .set_dev_pasid = arm_smmu_s1_set_dev_pasid,
3768 .map_pages = arm_smmu_map_pages,
3769 .unmap_pages = arm_smmu_unmap_pages,
3770 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
3771 .iotlb_sync = arm_smmu_iotlb_sync,
3772 .iova_to_phys = arm_smmu_iova_to_phys,
3773 .free = arm_smmu_domain_free_paging,
3774 }
3775 };
3776
3777 static struct iommu_dirty_ops arm_smmu_dirty_ops = {
3778 .read_and_clear_dirty = arm_smmu_read_and_clear_dirty,
3779 .set_dirty_tracking = arm_smmu_set_dirty_tracking,
3780 };
3781
3782 /* Probing and initialisation functions */
3783 int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
3784 struct arm_smmu_queue *q, void __iomem *page,
3785 unsigned long prod_off, unsigned long cons_off,
3786 size_t dwords, const char *name)
3787 {
3788 size_t qsz;
3789
3790 do {
3791 qsz = ((1 << q->llq.max_n_shift) * dwords) << 3;
3792 q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma,
3793 GFP_KERNEL);
3794 if (q->base || qsz < PAGE_SIZE)
3795 break;
3796
3797 q->llq.max_n_shift--;
3798 } while (1);
3799
3800 if (!q->base) {
3801 dev_err(smmu->dev,
3802 "failed to allocate queue (0x%zx bytes) for %s\n",
3803 qsz, name);
3804 return -ENOMEM;
3805 }
3806
3807 if (!WARN_ON(q->base_dma & (qsz - 1))) {
3808 dev_info(smmu->dev, "allocated %u entries for %s\n",
3809 1 << q->llq.max_n_shift, name);
3810 }
3811
3812 q->prod_reg = page + prod_off;
3813 q->cons_reg = page + cons_off;
3814 q->ent_dwords = dwords;
3815
3816 q->q_base = Q_BASE_RWA;
3817 q->q_base |= q->base_dma & Q_BASE_ADDR_MASK;
3818 q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift);
3819
3820 q->llq.prod = q->llq.cons = 0;
3821 return 0;
3822 }
3823
3824 int arm_smmu_cmdq_init(struct arm_smmu_device *smmu,
3825 struct arm_smmu_cmdq *cmdq)
3826 {
3827 unsigned int nents = 1 << cmdq->q.llq.max_n_shift;
3828
3829 atomic_set(&cmdq->owner_prod, 0);
3830 atomic_set(&cmdq->lock, 0);
3831
3832 cmdq->valid_map = (atomic_long_t *)devm_bitmap_zalloc(smmu->dev, nents,
3833 GFP_KERNEL);
3834 if (!cmdq->valid_map)
3835 return -ENOMEM;
3836
3837 return 0;
3838 }
3839
3840 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
3841 {
3842 int ret;
3843
3844 /* cmdq */
3845 ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, smmu->base,
3846 ARM_SMMU_CMDQ_PROD, ARM_SMMU_CMDQ_CONS,
3847 CMDQ_ENT_DWORDS, "cmdq");
3848 if (ret)
3849 return ret;
3850
3851 ret = arm_smmu_cmdq_init(smmu, &smmu->cmdq);
3852 if (ret)
3853 return ret;
3854
3855 /* evtq */
3856 ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, smmu->page1,
3857 ARM_SMMU_EVTQ_PROD, ARM_SMMU_EVTQ_CONS,
3858 EVTQ_ENT_DWORDS, "evtq");
3859 if (ret)
3860 return ret;
3861
3862 if ((smmu->features & ARM_SMMU_FEAT_SVA) &&
3863 (smmu->features & ARM_SMMU_FEAT_STALLS)) {
3864 smmu->evtq.iopf = iopf_queue_alloc(dev_name(smmu->dev));
3865 if (!smmu->evtq.iopf)
3866 return -ENOMEM;
3867 }
3868
3869 /* priq */
3870 if (!(smmu->features & ARM_SMMU_FEAT_PRI))
3871 return 0;
3872
3873 return arm_smmu_init_one_queue(smmu, &smmu->priq.q, smmu->page1,
3874 ARM_SMMU_PRIQ_PROD, ARM_SMMU_PRIQ_CONS,
3875 PRIQ_ENT_DWORDS, "priq");
3876 }
3877
3878 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
3879 {
3880 u32 l1size;
3881 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3882 unsigned int last_sid_idx =
3883 arm_smmu_strtab_l1_idx((1ULL << smmu->sid_bits) - 1);
3884
3885 /* Calculate the L1 size, capped to the SIDSIZE. */
3886 cfg->l2.num_l1_ents = min(last_sid_idx + 1, STRTAB_MAX_L1_ENTRIES);
3887 if (cfg->l2.num_l1_ents <= last_sid_idx)
3888 dev_warn(smmu->dev,
3889 "2-level strtab only covers %u/%u bits of SID\n",
3890 ilog2(cfg->l2.num_l1_ents * STRTAB_NUM_L2_STES),
3891 smmu->sid_bits);
3892
3893 l1size = cfg->l2.num_l1_ents * sizeof(struct arm_smmu_strtab_l1);
3894 cfg->l2.l1tab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->l2.l1_dma,
3895 GFP_KERNEL);
3896 if (!cfg->l2.l1tab) {
3897 dev_err(smmu->dev,
3898 "failed to allocate l1 stream table (%u bytes)\n",
3899 l1size);
3900 return -ENOMEM;
3901 }
3902
3903 cfg->l2.l2ptrs = devm_kcalloc(smmu->dev, cfg->l2.num_l1_ents,
3904 sizeof(*cfg->l2.l2ptrs), GFP_KERNEL);
3905 if (!cfg->l2.l2ptrs)
3906 return -ENOMEM;
3907
3908 return 0;
3909 }
3910
3911 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
3912 {
3913 u32 size;
3914 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3915
3916 size = (1 << smmu->sid_bits) * sizeof(struct arm_smmu_ste);
3917 cfg->linear.table = dmam_alloc_coherent(smmu->dev, size,
3918 &cfg->linear.ste_dma,
3919 GFP_KERNEL);
3920 if (!cfg->linear.table) {
3921 dev_err(smmu->dev,
3922 "failed to allocate linear stream table (%u bytes)\n",
3923 size);
3924 return -ENOMEM;
3925 }
3926 cfg->linear.num_ents = 1 << smmu->sid_bits;
3927
3928 arm_smmu_init_initial_stes(cfg->linear.table, cfg->linear.num_ents);
3929 return 0;
3930 }
3931
3932 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
3933 {
3934 int ret;
3935
3936 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3937 ret = arm_smmu_init_strtab_2lvl(smmu);
3938 else
3939 ret = arm_smmu_init_strtab_linear(smmu);
3940 if (ret)
3941 return ret;
3942
3943 ida_init(&smmu->vmid_map);
3944
3945 return 0;
3946 }
3947
3948 static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
3949 {
3950 int ret;
3951
3952 mutex_init(&smmu->streams_mutex);
3953 smmu->streams = RB_ROOT;
3954
3955 ret = arm_smmu_init_queues(smmu);
3956 if (ret)
3957 return ret;
3958
3959 ret = arm_smmu_init_strtab(smmu);
3960 if (ret)
3961 return ret;
3962
3963 if (smmu->impl_ops && smmu->impl_ops->init_structures)
3964 return smmu->impl_ops->init_structures(smmu);
3965
3966 return 0;
3967 }
3968
3969 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
3970 unsigned int reg_off, unsigned int ack_off)
3971 {
3972 u32 reg;
3973
3974 writel_relaxed(val, smmu->base + reg_off);
3975 return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
3976 1, ARM_SMMU_POLL_TIMEOUT_US);
3977 }
3978
3979 /* GBPA is "special" */
3980 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
3981 {
3982 int ret;
3983 u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
3984
3985 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
3986 1, ARM_SMMU_POLL_TIMEOUT_US);
3987 if (ret)
3988 return ret;
3989
3990 reg &= ~clr;
3991 reg |= set;
3992 writel_relaxed(reg | GBPA_UPDATE, gbpa);
3993 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
3994 1, ARM_SMMU_POLL_TIMEOUT_US);
3995
3996 if (ret)
3997 dev_err(smmu->dev, "GBPA not responding to update\n");
3998 return ret;
3999 }
4000
4001 static void arm_smmu_free_msis(void *data)
4002 {
4003 struct device *dev = data;
4004
4005 platform_device_msi_free_irqs_all(dev);
4006 }
4007
4008 static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
4009 {
4010 phys_addr_t doorbell;
4011 struct device *dev = msi_desc_to_dev(desc);
4012 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
4013 phys_addr_t *cfg = arm_smmu_msi_cfg[desc->msi_index];
4014
4015 doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
4016 doorbell &= MSI_CFG0_ADDR_MASK;
4017
4018 writeq_relaxed(doorbell, smmu->base + cfg[0]);
4019 writel_relaxed(msg->data, smmu->base + cfg[1]);
4020 writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
4021 }
4022
4023 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
4024 {
4025 int ret, nvec = ARM_SMMU_MAX_MSIS;
4026 struct device *dev = smmu->dev;
4027
4028 /* Clear the MSI address regs */
4029 writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
4030 writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
4031
4032 if (smmu->features & ARM_SMMU_FEAT_PRI)
4033 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
4034 else
4035 nvec--;
4036
4037 if (!(smmu->features & ARM_SMMU_FEAT_MSI))
4038 return;
4039
4040 if (!dev->msi.domain) {
4041 dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n");
4042 return;
4043 }
4044
4045 /* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */
4046 ret = platform_device_msi_init_and_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
4047 if (ret) {
4048 dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n");
4049 return;
4050 }
4051
4052 smmu->evtq.q.irq = msi_get_virq(dev, EVTQ_MSI_INDEX);
4053 smmu->gerr_irq = msi_get_virq(dev, GERROR_MSI_INDEX);
4054 smmu->priq.q.irq = msi_get_virq(dev, PRIQ_MSI_INDEX);
4055
4056 /* Add callback to free MSIs on teardown */
4057 devm_add_action_or_reset(dev, arm_smmu_free_msis, dev);
4058 }
4059
4060 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
4061 {
4062 int irq, ret;
4063
4064 arm_smmu_setup_msis(smmu);
4065
4066 /* Request interrupt lines */
4067 irq = smmu->evtq.q.irq;
4068 if (irq) {
4069 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
4070 arm_smmu_evtq_thread,
4071 IRQF_ONESHOT,
4072 "arm-smmu-v3-evtq", smmu);
4073 if (ret < 0)
4074 dev_warn(smmu->dev, "failed to enable evtq irq\n");
4075 } else {
4076 dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n");
4077 }
4078
4079 irq = smmu->gerr_irq;
4080 if (irq) {
4081 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
4082 0, "arm-smmu-v3-gerror", smmu);
4083 if (ret < 0)
4084 dev_warn(smmu->dev, "failed to enable gerror irq\n");
4085 } else {
4086 dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n");
4087 }
4088
4089 if (smmu->features & ARM_SMMU_FEAT_PRI) {
4090 irq = smmu->priq.q.irq;
4091 if (irq) {
4092 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
4093 arm_smmu_priq_thread,
4094 IRQF_ONESHOT,
4095 "arm-smmu-v3-priq",
4096 smmu);
4097 if (ret < 0)
4098 dev_warn(smmu->dev,
4099 "failed to enable priq irq\n");
4100 } else {
4101 dev_warn(smmu->dev, "no priq irq - PRI will be broken\n");
4102 }
4103 }
4104 }
4105
4106 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
4107 {
4108 int ret, irq;
4109 u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
4110
4111 /* Disable IRQs first */
4112 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
4113 ARM_SMMU_IRQ_CTRLACK);
4114 if (ret) {
4115 dev_err(smmu->dev, "failed to disable irqs\n");
4116 return ret;
4117 }
4118
4119 irq = smmu->combined_irq;
4120 if (irq) {
4121 /*
4122 * Cavium ThunderX2 implementation doesn't support unique irq
4123 * lines. Use a single irq line for all the SMMUv3 interrupts.
4124 */
4125 ret = devm_request_threaded_irq(smmu->dev, irq,
4126 arm_smmu_combined_irq_handler,
4127 arm_smmu_combined_irq_thread,
4128 IRQF_ONESHOT,
4129 "arm-smmu-v3-combined-irq", smmu);
4130 if (ret < 0)
4131 dev_warn(smmu->dev, "failed to enable combined irq\n");
4132 } else
4133 arm_smmu_setup_unique_irqs(smmu);
4134
4135 if (smmu->features & ARM_SMMU_FEAT_PRI)
4136 irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
4137
4138 /* Enable interrupt generation on the SMMU */
4139 ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
4140 ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
4141 if (ret)
4142 dev_warn(smmu->dev, "failed to enable irqs\n");
4143
4144 return 0;
4145 }
4146
4147 static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
4148 {
4149 int ret;
4150
4151 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
4152 if (ret)
4153 dev_err(smmu->dev, "failed to clear cr0\n");
4154
4155 return ret;
4156 }
4157
4158 static void arm_smmu_write_strtab(struct arm_smmu_device *smmu)
4159 {
4160 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
4161 dma_addr_t dma;
4162 u32 reg;
4163
4164 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
4165 reg = FIELD_PREP(STRTAB_BASE_CFG_FMT,
4166 STRTAB_BASE_CFG_FMT_2LVL) |
4167 FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE,
4168 ilog2(cfg->l2.num_l1_ents) + STRTAB_SPLIT) |
4169 FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT);
4170 dma = cfg->l2.l1_dma;
4171 } else {
4172 reg = FIELD_PREP(STRTAB_BASE_CFG_FMT,
4173 STRTAB_BASE_CFG_FMT_LINEAR) |
4174 FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits);
4175 dma = cfg->linear.ste_dma;
4176 }
4177 writeq_relaxed((dma & STRTAB_BASE_ADDR_MASK) | STRTAB_BASE_RA,
4178 smmu->base + ARM_SMMU_STRTAB_BASE);
4179 writel_relaxed(reg, smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
4180 }
4181
4182 static int arm_smmu_device_reset(struct arm_smmu_device *smmu)
4183 {
4184 int ret;
4185 u32 reg, enables;
4186 struct arm_smmu_cmdq_ent cmd;
4187
4188 /* Clear CR0 and sync (disables SMMU and queue processing) */
4189 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
4190 if (reg & CR0_SMMUEN) {
4191 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
4192 arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0);
4193 }
4194
4195 ret = arm_smmu_device_disable(smmu);
4196 if (ret)
4197 return ret;
4198
4199 /* CR1 (table and queue memory attributes) */
4200 reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) |
4201 FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) |
4202 FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) |
4203 FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) |
4204 FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) |
4205 FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB);
4206 writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
4207
4208 /* CR2 (random crap) */
4209 reg = CR2_PTM | CR2_RECINVSID;
4210
4211 if (smmu->features & ARM_SMMU_FEAT_E2H)
4212 reg |= CR2_E2H;
4213
4214 writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
4215
4216 /* Stream table */
4217 arm_smmu_write_strtab(smmu);
4218
4219 /* Command queue */
4220 writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
4221 writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
4222 writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
4223
4224 enables = CR0_CMDQEN;
4225 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4226 ARM_SMMU_CR0ACK);
4227 if (ret) {
4228 dev_err(smmu->dev, "failed to enable command queue\n");
4229 return ret;
4230 }
4231
4232 /* Invalidate any cached configuration */
4233 cmd.opcode = CMDQ_OP_CFGI_ALL;
4234 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
4235
4236 /* Invalidate any stale TLB entries */
4237 if (smmu->features & ARM_SMMU_FEAT_HYP) {
4238 cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
4239 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
4240 }
4241
4242 cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
4243 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
4244
4245 /* Event queue */
4246 writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
4247 writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD);
4248 writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS);
4249
4250 enables |= CR0_EVTQEN;
4251 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4252 ARM_SMMU_CR0ACK);
4253 if (ret) {
4254 dev_err(smmu->dev, "failed to enable event queue\n");
4255 return ret;
4256 }
4257
4258 /* PRI queue */
4259 if (smmu->features & ARM_SMMU_FEAT_PRI) {
4260 writeq_relaxed(smmu->priq.q.q_base,
4261 smmu->base + ARM_SMMU_PRIQ_BASE);
4262 writel_relaxed(smmu->priq.q.llq.prod,
4263 smmu->page1 + ARM_SMMU_PRIQ_PROD);
4264 writel_relaxed(smmu->priq.q.llq.cons,
4265 smmu->page1 + ARM_SMMU_PRIQ_CONS);
4266
4267 enables |= CR0_PRIQEN;
4268 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4269 ARM_SMMU_CR0ACK);
4270 if (ret) {
4271 dev_err(smmu->dev, "failed to enable PRI queue\n");
4272 return ret;
4273 }
4274 }
4275
4276 if (smmu->features & ARM_SMMU_FEAT_ATS) {
4277 enables |= CR0_ATSCHK;
4278 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4279 ARM_SMMU_CR0ACK);
4280 if (ret) {
4281 dev_err(smmu->dev, "failed to enable ATS check\n");
4282 return ret;
4283 }
4284 }
4285
4286 ret = arm_smmu_setup_irqs(smmu);
4287 if (ret) {
4288 dev_err(smmu->dev, "failed to setup irqs\n");
4289 return ret;
4290 }
4291
4292 if (is_kdump_kernel())
4293 enables &= ~(CR0_EVTQEN | CR0_PRIQEN);
4294
4295 /* Enable the SMMU interface */
4296 enables |= CR0_SMMUEN;
4297 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4298 ARM_SMMU_CR0ACK);
4299 if (ret) {
4300 dev_err(smmu->dev, "failed to enable SMMU interface\n");
4301 return ret;
4302 }
4303
4304 if (smmu->impl_ops && smmu->impl_ops->device_reset) {
4305 ret = smmu->impl_ops->device_reset(smmu);
4306 if (ret) {
4307 dev_err(smmu->dev, "failed to reset impl\n");
4308 return ret;
4309 }
4310 }
4311
4312 return 0;
4313 }
4314
4315 #define IIDR_IMPLEMENTER_ARM 0x43b
4316 #define IIDR_PRODUCTID_ARM_MMU_600 0x483
4317 #define IIDR_PRODUCTID_ARM_MMU_700 0x487
4318
4319 static void arm_smmu_device_iidr_probe(struct arm_smmu_device *smmu)
4320 {
4321 u32 reg;
4322 unsigned int implementer, productid, variant, revision;
4323
4324 reg = readl_relaxed(smmu->base + ARM_SMMU_IIDR);
4325 implementer = FIELD_GET(IIDR_IMPLEMENTER, reg);
4326 productid = FIELD_GET(IIDR_PRODUCTID, reg);
4327 variant = FIELD_GET(IIDR_VARIANT, reg);
4328 revision = FIELD_GET(IIDR_REVISION, reg);
4329
4330 switch (implementer) {
4331 case IIDR_IMPLEMENTER_ARM:
4332 switch (productid) {
4333 case IIDR_PRODUCTID_ARM_MMU_600:
4334 /* Arm erratum 1076982 */
4335 if (variant == 0 && revision <= 2)
4336 smmu->features &= ~ARM_SMMU_FEAT_SEV;
4337 /* Arm erratum 1209401 */
4338 if (variant < 2)
4339 smmu->features &= ~ARM_SMMU_FEAT_NESTING;
4340 break;
4341 case IIDR_PRODUCTID_ARM_MMU_700:
4342 /* Arm erratum 2812531 */
4343 smmu->features &= ~ARM_SMMU_FEAT_BTM;
4344 smmu->options |= ARM_SMMU_OPT_CMDQ_FORCE_SYNC;
4345 /* Arm errata 2268618, 2812531 */
4346 smmu->features &= ~ARM_SMMU_FEAT_NESTING;
4347 break;
4348 }
4349 break;
4350 }
4351 }
4352
4353 static void arm_smmu_get_httu(struct arm_smmu_device *smmu, u32 reg)
4354 {
4355 u32 fw_features = smmu->features & (ARM_SMMU_FEAT_HA | ARM_SMMU_FEAT_HD);
4356 u32 hw_features = 0;
4357
4358 switch (FIELD_GET(IDR0_HTTU, reg)) {
4359 case IDR0_HTTU_ACCESS_DIRTY:
4360 hw_features |= ARM_SMMU_FEAT_HD;
4361 fallthrough;
4362 case IDR0_HTTU_ACCESS:
4363 hw_features |= ARM_SMMU_FEAT_HA;
4364 }
4365
4366 if (smmu->dev->of_node)
4367 smmu->features |= hw_features;
4368 else if (hw_features != fw_features)
4369 /* ACPI IORT sets the HTTU bits */
4370 dev_warn(smmu->dev,
4371 "IDR0.HTTU features(0x%x) overridden by FW configuration (0x%x)\n",
4372 hw_features, fw_features);
4373 }
4374
4375 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
4376 {
4377 u32 reg;
4378 bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
4379
4380 /* IDR0 */
4381 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
4382
4383 /* 2-level structures */
4384 if (FIELD_GET(IDR0_ST_LVL, reg) == IDR0_ST_LVL_2LVL)
4385 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
4386
4387 if (reg & IDR0_CD2L)
4388 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
4389
4390 /*
4391 * Translation table endianness.
4392 * We currently require the same endianness as the CPU, but this
4393 * could be changed later by adding a new IO_PGTABLE_QUIRK.
4394 */
4395 switch (FIELD_GET(IDR0_TTENDIAN, reg)) {
4396 case IDR0_TTENDIAN_MIXED:
4397 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
4398 break;
4399 #ifdef __BIG_ENDIAN
4400 case IDR0_TTENDIAN_BE:
4401 smmu->features |= ARM_SMMU_FEAT_TT_BE;
4402 break;
4403 #else
4404 case IDR0_TTENDIAN_LE:
4405 smmu->features |= ARM_SMMU_FEAT_TT_LE;
4406 break;
4407 #endif
4408 default:
4409 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
4410 return -ENXIO;
4411 }
4412
4413 /* Boolean feature flags */
4414 if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
4415 smmu->features |= ARM_SMMU_FEAT_PRI;
4416
4417 if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
4418 smmu->features |= ARM_SMMU_FEAT_ATS;
4419
4420 if (reg & IDR0_SEV)
4421 smmu->features |= ARM_SMMU_FEAT_SEV;
4422
4423 if (reg & IDR0_MSI) {
4424 smmu->features |= ARM_SMMU_FEAT_MSI;
4425 if (coherent && !disable_msipolling)
4426 smmu->options |= ARM_SMMU_OPT_MSIPOLL;
4427 }
4428
4429 if (reg & IDR0_HYP) {
4430 smmu->features |= ARM_SMMU_FEAT_HYP;
4431 if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN))
4432 smmu->features |= ARM_SMMU_FEAT_E2H;
4433 }
4434
4435 arm_smmu_get_httu(smmu, reg);
4436
4437 /*
4438 * The coherency feature as set by FW is used in preference to the ID
4439 * register, but warn on mismatch.
4440 */
4441 if (!!(reg & IDR0_COHACC) != coherent)
4442 dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n",
4443 str_true_false(coherent));
4444
4445 switch (FIELD_GET(IDR0_STALL_MODEL, reg)) {
4446 case IDR0_STALL_MODEL_FORCE:
4447 smmu->features |= ARM_SMMU_FEAT_STALL_FORCE;
4448 fallthrough;
4449 case IDR0_STALL_MODEL_STALL:
4450 smmu->features |= ARM_SMMU_FEAT_STALLS;
4451 }
4452
4453 if (reg & IDR0_S1P)
4454 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
4455
4456 if (reg & IDR0_S2P)
4457 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
4458
4459 if (!(reg & (IDR0_S1P | IDR0_S2P))) {
4460 dev_err(smmu->dev, "no translation support!\n");
4461 return -ENXIO;
4462 }
4463
4464 /* We only support the AArch64 table format at present */
4465 switch (FIELD_GET(IDR0_TTF, reg)) {
4466 case IDR0_TTF_AARCH32_64:
4467 smmu->ias = 40;
4468 fallthrough;
4469 case IDR0_TTF_AARCH64:
4470 break;
4471 default:
4472 dev_err(smmu->dev, "AArch64 table format not supported!\n");
4473 return -ENXIO;
4474 }
4475
4476 /* ASID/VMID sizes */
4477 smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
4478 smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
4479
4480 /* IDR1 */
4481 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
4482 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
4483 dev_err(smmu->dev, "embedded implementation not supported\n");
4484 return -ENXIO;
4485 }
4486
4487 if (reg & IDR1_ATTR_TYPES_OVR)
4488 smmu->features |= ARM_SMMU_FEAT_ATTR_TYPES_OVR;
4489
4490 /* Queue sizes, capped to ensure natural alignment */
4491 smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,
4492 FIELD_GET(IDR1_CMDQS, reg));
4493 if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) {
4494 /*
4495 * We don't support splitting up batches, so one batch of
4496 * commands plus an extra sync needs to fit inside the command
4497 * queue. There's also no way we can handle the weird alignment
4498 * restrictions on the base pointer for a unit-length queue.
4499 */
4500 dev_err(smmu->dev, "command queue size <= %d entries not supported\n",
4501 CMDQ_BATCH_ENTRIES);
4502 return -ENXIO;
4503 }
4504
4505 smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,
4506 FIELD_GET(IDR1_EVTQS, reg));
4507 smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,
4508 FIELD_GET(IDR1_PRIQS, reg));
4509
4510 /* SID/SSID sizes */
4511 smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
4512 smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
4513 smmu->iommu.max_pasids = 1UL << smmu->ssid_bits;
4514
4515 /*
4516 * If the SMMU supports fewer bits than would fill a single L2 stream
4517 * table, use a linear table instead.
4518 */
4519 if (smmu->sid_bits <= STRTAB_SPLIT)
4520 smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
4521
4522 /* IDR3 */
4523 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR3);
4524 if (FIELD_GET(IDR3_RIL, reg))
4525 smmu->features |= ARM_SMMU_FEAT_RANGE_INV;
4526 if (FIELD_GET(IDR3_FWB, reg))
4527 smmu->features |= ARM_SMMU_FEAT_S2FWB;
4528
4529 if (FIELD_GET(IDR3_BBM, reg) == 2)
4530 smmu->features |= ARM_SMMU_FEAT_BBML2;
4531
4532 /* IDR5 */
4533 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
4534
4535 /* Maximum number of outstanding stalls */
4536 smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg);
4537
4538 /* Page sizes */
4539 if (reg & IDR5_GRAN64K)
4540 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
4541 if (reg & IDR5_GRAN16K)
4542 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
4543 if (reg & IDR5_GRAN4K)
4544 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
4545
4546 /* Input address size */
4547 if (FIELD_GET(IDR5_VAX, reg) == IDR5_VAX_52_BIT)
4548 smmu->features |= ARM_SMMU_FEAT_VAX;
4549
4550 /* Output address size */
4551 switch (FIELD_GET(IDR5_OAS, reg)) {
4552 case IDR5_OAS_32_BIT:
4553 smmu->oas = 32;
4554 break;
4555 case IDR5_OAS_36_BIT:
4556 smmu->oas = 36;
4557 break;
4558 case IDR5_OAS_40_BIT:
4559 smmu->oas = 40;
4560 break;
4561 case IDR5_OAS_42_BIT:
4562 smmu->oas = 42;
4563 break;
4564 case IDR5_OAS_44_BIT:
4565 smmu->oas = 44;
4566 break;
4567 case IDR5_OAS_52_BIT:
4568 smmu->oas = 52;
4569 smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */
4570 break;
4571 default:
4572 dev_info(smmu->dev,
4573 "unknown output address size. Truncating to 48-bit\n");
4574 fallthrough;
4575 case IDR5_OAS_48_BIT:
4576 smmu->oas = 48;
4577 }
4578
4579 /* Set the DMA mask for our table walker */
4580 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
4581 dev_warn(smmu->dev,
4582 "failed to set DMA mask for table walker\n");
4583
4584 smmu->ias = max(smmu->ias, smmu->oas);
4585
4586 if ((smmu->features & ARM_SMMU_FEAT_TRANS_S1) &&
4587 (smmu->features & ARM_SMMU_FEAT_TRANS_S2))
4588 smmu->features |= ARM_SMMU_FEAT_NESTING;
4589
4590 arm_smmu_device_iidr_probe(smmu);
4591
4592 if (arm_smmu_sva_supported(smmu))
4593 smmu->features |= ARM_SMMU_FEAT_SVA;
4594
4595 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
4596 smmu->ias, smmu->oas, smmu->features);
4597 return 0;
4598 }
4599
4600 #ifdef CONFIG_ACPI
4601 #ifdef CONFIG_TEGRA241_CMDQV
4602 static void acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node *node,
4603 struct arm_smmu_device *smmu)
4604 {
4605 const char *uid = kasprintf(GFP_KERNEL, "%u", node->identifier);
4606 struct acpi_device *adev;
4607
4608 /* Look for an NVDA200C node whose _UID matches the SMMU node ID */
4609 adev = acpi_dev_get_first_match_dev("NVDA200C", uid, -1);
4610 if (adev) {
4611 /* Tegra241 CMDQV driver is responsible for put_device() */
4612 smmu->impl_dev = &adev->dev;
4613 smmu->options |= ARM_SMMU_OPT_TEGRA241_CMDQV;
4614 dev_info(smmu->dev, "found companion CMDQV device: %s\n",
4615 dev_name(smmu->impl_dev));
4616 }
4617 kfree(uid);
4618 }
4619 #else
4620 static void acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node *node,
4621 struct arm_smmu_device *smmu)
4622 {
4623 }
4624 #endif
4625
4626 static int acpi_smmu_iort_probe_model(struct acpi_iort_node *node,
4627 struct arm_smmu_device *smmu)
4628 {
4629 struct acpi_iort_smmu_v3 *iort_smmu =
4630 (struct acpi_iort_smmu_v3 *)node->node_data;
4631
4632 switch (iort_smmu->model) {
4633 case ACPI_IORT_SMMU_V3_CAVIUM_CN99XX:
4634 smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY;
4635 break;
4636 case ACPI_IORT_SMMU_V3_HISILICON_HI161X:
4637 smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH;
4638 break;
4639 case ACPI_IORT_SMMU_V3_GENERIC:
4640 /*
4641 * Tegra241 implementation stores its SMMU options and impl_dev
4642 * in DSDT. Thus, go through the ACPI tables unconditionally.
4643 */
4644 acpi_smmu_dsdt_probe_tegra241_cmdqv(node, smmu);
4645 break;
4646 }
4647
4648 dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options);
4649 return 0;
4650 }
4651
4652 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
4653 struct arm_smmu_device *smmu)
4654 {
4655 struct acpi_iort_smmu_v3 *iort_smmu;
4656 struct device *dev = smmu->dev;
4657 struct acpi_iort_node *node;
4658
4659 node = *(struct acpi_iort_node **)dev_get_platdata(dev);
4660
4661 /* Retrieve SMMUv3 specific data */
4662 iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
4663
4664 if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
4665 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
4666
4667 switch (FIELD_GET(ACPI_IORT_SMMU_V3_HTTU_OVERRIDE, iort_smmu->flags)) {
4668 case IDR0_HTTU_ACCESS_DIRTY:
4669 smmu->features |= ARM_SMMU_FEAT_HD;
4670 fallthrough;
4671 case IDR0_HTTU_ACCESS:
4672 smmu->features |= ARM_SMMU_FEAT_HA;
4673 }
4674
4675 return acpi_smmu_iort_probe_model(node, smmu);
4676 }
4677 #else
4678 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
4679 struct arm_smmu_device *smmu)
4680 {
4681 return -ENODEV;
4682 }
4683 #endif
4684
4685 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
4686 struct arm_smmu_device *smmu)
4687 {
4688 struct device *dev = &pdev->dev;
4689 u32 cells;
4690 int ret = -EINVAL;
4691
4692 if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
4693 dev_err(dev, "missing #iommu-cells property\n");
4694 else if (cells != 1)
4695 dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
4696 else
4697 ret = 0;
4698
4699 parse_driver_options(smmu);
4700
4701 if (of_dma_is_coherent(dev->of_node))
4702 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
4703
4704 return ret;
4705 }
4706
4707 static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu)
4708 {
4709 if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY)
4710 return SZ_64K;
4711 else
4712 return SZ_128K;
4713 }
4714
4715 static void __iomem *arm_smmu_ioremap(struct device *dev, resource_size_t start,
4716 resource_size_t size)
4717 {
4718 struct resource res = DEFINE_RES_MEM(start, size);
4719
4720 return devm_ioremap_resource(dev, &res);
4721 }
4722
4723 static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu)
4724 {
4725 struct list_head rmr_list;
4726 struct iommu_resv_region *e;
4727
4728 INIT_LIST_HEAD(&rmr_list);
4729 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
4730
4731 list_for_each_entry(e, &rmr_list, list) {
4732 struct iommu_iort_rmr_data *rmr;
4733 int ret, i;
4734
4735 rmr = container_of(e, struct iommu_iort_rmr_data, rr);
4736 for (i = 0; i < rmr->num_sids; i++) {
4737 ret = arm_smmu_init_sid_strtab(smmu, rmr->sids[i]);
4738 if (ret) {
4739 dev_err(smmu->dev, "RMR SID(0x%x) bypass failed\n",
4740 rmr->sids[i]);
4741 continue;
4742 }
4743
4744 /*
4745 * STE table is not programmed to HW, see
4746 * arm_smmu_initial_bypass_stes()
4747 */
4748 arm_smmu_make_bypass_ste(smmu,
4749 arm_smmu_get_step_for_sid(smmu, rmr->sids[i]));
4750 }
4751 }
4752
4753 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
4754 }
4755
4756 static void arm_smmu_impl_remove(void *data)
4757 {
4758 struct arm_smmu_device *smmu = data;
4759
4760 if (smmu->impl_ops && smmu->impl_ops->device_remove)
4761 smmu->impl_ops->device_remove(smmu);
4762 }
4763
4764 /*
4765 * Probe all the compiled in implementations. Each one checks to see if it
4766 * matches this HW and if so returns a devm_krealloc'd arm_smmu_device which
4767 * replaces the callers. Otherwise the original is returned or ERR_PTR.
4768 */
4769 static struct arm_smmu_device *arm_smmu_impl_probe(struct arm_smmu_device *smmu)
4770 {
4771 struct arm_smmu_device *new_smmu = ERR_PTR(-ENODEV);
4772 const struct arm_smmu_impl_ops *ops;
4773 int ret;
4774
4775 if (smmu->impl_dev && (smmu->options & ARM_SMMU_OPT_TEGRA241_CMDQV))
4776 new_smmu = tegra241_cmdqv_probe(smmu);
4777
4778 if (new_smmu == ERR_PTR(-ENODEV))
4779 return smmu;
4780 if (IS_ERR(new_smmu))
4781 return new_smmu;
4782
4783 ops = new_smmu->impl_ops;
4784 if (ops) {
4785 /* get_viommu_size and vsmmu_init ops must be paired */
4786 if (WARN_ON(!ops->get_viommu_size != !ops->vsmmu_init)) {
4787 ret = -EINVAL;
4788 goto err_remove;
4789 }
4790 }
4791
4792 ret = devm_add_action_or_reset(new_smmu->dev, arm_smmu_impl_remove,
4793 new_smmu);
4794 if (ret)
4795 return ERR_PTR(ret);
4796 return new_smmu;
4797
4798 err_remove:
4799 arm_smmu_impl_remove(new_smmu);
4800 return ERR_PTR(ret);
4801 }
4802
4803 static int arm_smmu_device_probe(struct platform_device *pdev)
4804 {
4805 int irq, ret;
4806 struct resource *res;
4807 resource_size_t ioaddr;
4808 struct arm_smmu_device *smmu;
4809 struct device *dev = &pdev->dev;
4810
4811 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
4812 if (!smmu)
4813 return -ENOMEM;
4814 smmu->dev = dev;
4815
4816 if (dev->of_node) {
4817 ret = arm_smmu_device_dt_probe(pdev, smmu);
4818 } else {
4819 ret = arm_smmu_device_acpi_probe(pdev, smmu);
4820 }
4821 if (ret)
4822 return ret;
4823
4824 smmu = arm_smmu_impl_probe(smmu);
4825 if (IS_ERR(smmu))
4826 return PTR_ERR(smmu);
4827
4828 /* Base address */
4829 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
4830 if (!res)
4831 return -EINVAL;
4832 if (resource_size(res) < arm_smmu_resource_size(smmu)) {
4833 dev_err(dev, "MMIO region too small (%pr)\n", res);
4834 return -EINVAL;
4835 }
4836 ioaddr = res->start;
4837
4838 /*
4839 * Don't map the IMPLEMENTATION DEFINED regions, since they may contain
4840 * the PMCG registers which are reserved by the PMU driver.
4841 */
4842 smmu->base = arm_smmu_ioremap(dev, ioaddr, ARM_SMMU_REG_SZ);
4843 if (IS_ERR(smmu->base))
4844 return PTR_ERR(smmu->base);
4845
4846 if (arm_smmu_resource_size(smmu) > SZ_64K) {
4847 smmu->page1 = arm_smmu_ioremap(dev, ioaddr + SZ_64K,
4848 ARM_SMMU_REG_SZ);
4849 if (IS_ERR(smmu->page1))
4850 return PTR_ERR(smmu->page1);
4851 } else {
4852 smmu->page1 = smmu->base;
4853 }
4854
4855 /* Interrupt lines */
4856
4857 irq = platform_get_irq_byname_optional(pdev, "combined");
4858 if (irq > 0)
4859 smmu->combined_irq = irq;
4860 else {
4861 irq = platform_get_irq_byname_optional(pdev, "eventq");
4862 if (irq > 0)
4863 smmu->evtq.q.irq = irq;
4864
4865 irq = platform_get_irq_byname_optional(pdev, "priq");
4866 if (irq > 0)
4867 smmu->priq.q.irq = irq;
4868
4869 irq = platform_get_irq_byname_optional(pdev, "gerror");
4870 if (irq > 0)
4871 smmu->gerr_irq = irq;
4872 }
4873 /* Probe the h/w */
4874 ret = arm_smmu_device_hw_probe(smmu);
4875 if (ret)
4876 return ret;
4877
4878 /* Initialise in-memory data structures */
4879 ret = arm_smmu_init_structures(smmu);
4880 if (ret)
4881 goto err_free_iopf;
4882
4883 /* Record our private device structure */
4884 platform_set_drvdata(pdev, smmu);
4885
4886 /* Check for RMRs and install bypass STEs if any */
4887 arm_smmu_rmr_install_bypass_ste(smmu);
4888
4889 /* Reset the device */
4890 ret = arm_smmu_device_reset(smmu);
4891 if (ret)
4892 goto err_disable;
4893
4894 /* And we're up. Go go go! */
4895 ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
4896 "smmu3.%pa", &ioaddr);
4897 if (ret)
4898 goto err_disable;
4899
4900 ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);
4901 if (ret) {
4902 dev_err(dev, "Failed to register iommu\n");
4903 goto err_free_sysfs;
4904 }
4905
4906 return 0;
4907
4908 err_free_sysfs:
4909 iommu_device_sysfs_remove(&smmu->iommu);
4910 err_disable:
4911 arm_smmu_device_disable(smmu);
4912 err_free_iopf:
4913 iopf_queue_free(smmu->evtq.iopf);
4914 return ret;
4915 }
4916
4917 static void arm_smmu_device_remove(struct platform_device *pdev)
4918 {
4919 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
4920
4921 iommu_device_unregister(&smmu->iommu);
4922 iommu_device_sysfs_remove(&smmu->iommu);
4923 arm_smmu_device_disable(smmu);
4924 iopf_queue_free(smmu->evtq.iopf);
4925 ida_destroy(&smmu->vmid_map);
4926 }
4927
4928 static void arm_smmu_device_shutdown(struct platform_device *pdev)
4929 {
4930 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
4931
4932 arm_smmu_device_disable(smmu);
4933 }
4934
4935 static const struct of_device_id arm_smmu_of_match[] = {
4936 { .compatible = "arm,smmu-v3", },
4937 { },
4938 };
4939 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
4940
4941 static void arm_smmu_driver_unregister(struct platform_driver *drv)
4942 {
4943 arm_smmu_sva_notifier_synchronize();
4944 platform_driver_unregister(drv);
4945 }
4946
4947 static struct platform_driver arm_smmu_driver = {
4948 .driver = {
4949 .name = "arm-smmu-v3",
4950 .of_match_table = arm_smmu_of_match,
4951 .suppress_bind_attrs = true,
4952 },
4953 .probe = arm_smmu_device_probe,
4954 .remove = arm_smmu_device_remove,
4955 .shutdown = arm_smmu_device_shutdown,
4956 };
4957 module_driver(arm_smmu_driver, platform_driver_register,
4958 arm_smmu_driver_unregister);
4959
4960 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
4961 MODULE_AUTHOR("Will Deacon <[email protected]>");
4962 MODULE_ALIAS("platform:arm-smmu-v3");
4963 MODULE_LICENSE("GPL v2");