개념 설명 전체 · v6.6 / kernel/irq/irqdesc.c

    1 // SPDX-License-Identifier: GPL-2.0
    2 /*
    3  * Copyright (C) 1992, 1998-2006 Linus Torvalds, Ingo Molnar
    4  * Copyright (C) 2005-2006, Thomas Gleixner, Russell King
    5  *
    6  * This file contains the interrupt descriptor management code. Detailed
    7  * information is available in Documentation/core-api/genericirq.rst
    8  *
    9  */
   10 #include <linux/irq.h>
   11 #include <linux/slab.h>
   12 #include <linux/export.h>
   13 #include <linux/interrupt.h>
   14 #include <linux/kernel_stat.h>
   15 #include <linux/maple_tree.h>
   16 #include <linux/irqdomain.h>
   17 #include <linux/sysfs.h>
   18 
   19 #include "internals.h"
   20 
   21 /*
   22  * lockdep: we want to handle all irq_desc locks as a single lock-class:
   23  */
   24 static struct lock_class_key irq_desc_lock_class;
   25 
   26 #if defined(CONFIG_SMP)
   27 static int __init irq_affinity_setup(char *str)
   28 {
   29 	alloc_bootmem_cpumask_var(&irq_default_affinity);
   30 	cpulist_parse(str, irq_default_affinity);
   31 	/*
   32 	 * Set at least the boot cpu. We don't want to end up with
   33 	 * bugreports caused by random commandline masks
   34 	 */
   35 	cpumask_set_cpu(smp_processor_id(), irq_default_affinity);
   36 	return 1;
   37 }
   38 __setup("irqaffinity=", irq_affinity_setup);
   39 
   40 static void __init init_irq_default_affinity(void)
   41 {
   42 	if (!cpumask_available(irq_default_affinity))
   43 		zalloc_cpumask_var(&irq_default_affinity, GFP_NOWAIT);
   44 	if (cpumask_empty(irq_default_affinity))
   45 		cpumask_setall(irq_default_affinity);
   46 }
   47 #else
   48 static void __init init_irq_default_affinity(void)
   49 {
   50 }
   51 #endif
   52 
   53 #ifdef CONFIG_SMP
   54 static int alloc_masks(struct irq_desc *desc, int node)
   55 {
   56 	if (!zalloc_cpumask_var_node(&desc->irq_common_data.affinity,
   57 				     GFP_KERNEL, node))
   58 		return -ENOMEM;
   59 
   60 #ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
   61 	if (!zalloc_cpumask_var_node(&desc->irq_common_data.effective_affinity,
   62 				     GFP_KERNEL, node)) {
   63 		free_cpumask_var(desc->irq_common_data.affinity);
   64 		return -ENOMEM;
   65 	}
   66 #endif
   67 
   68 #ifdef CONFIG_GENERIC_PENDING_IRQ
   69 	if (!zalloc_cpumask_var_node(&desc->pending_mask, GFP_KERNEL, node)) {
   70 #ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
   71 		free_cpumask_var(desc->irq_common_data.effective_affinity);
   72 #endif
   73 		free_cpumask_var(desc->irq_common_data.affinity);
   74 		return -ENOMEM;
   75 	}
   76 #endif
   77 	return 0;
   78 }
   79 
   80 static void desc_smp_init(struct irq_desc *desc, int node,
   81 			  const struct cpumask *affinity)
   82 {
   83 	if (!affinity)
   84 		affinity = irq_default_affinity;
   85 	cpumask_copy(desc->irq_common_data.affinity, affinity);
   86 
   87 #ifdef CONFIG_GENERIC_PENDING_IRQ
   88 	cpumask_clear(desc->pending_mask);
   89 #endif
   90 #ifdef CONFIG_NUMA
   91 	desc->irq_common_data.node = node;
   92 #endif
   93 }
   94 
   95 #else
   96 static inline int
   97 alloc_masks(struct irq_desc *desc, int node) { return 0; }
   98 static inline void
   99 desc_smp_init(struct irq_desc *desc, int node, const struct cpumask *affinity) { }
  100 #endif
  101 
  102 static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node,
  103 			      const struct cpumask *affinity, struct module *owner)
  104 {
  105 	int cpu;
  106 
  107 	desc->irq_common_data.handler_data = NULL;
  108 	desc->irq_common_data.msi_desc = NULL;
  109 
  110 	desc->irq_data.common = &desc->irq_common_data;
  111 	desc->irq_data.irq = irq;
  112 	desc->irq_data.chip = &no_irq_chip;
  113 	desc->irq_data.chip_data = NULL;
  114 	irq_settings_clr_and_set(desc, ~0, _IRQ_DEFAULT_INIT_FLAGS);
  115 	irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);
  116 	irqd_set(&desc->irq_data, IRQD_IRQ_MASKED);
  117 	desc->handle_irq = handle_bad_irq;
  118 	desc->depth = 1;
  119 	desc->irq_count = 0;
  120 	desc->irqs_unhandled = 0;
  121 	desc->tot_count = 0;
  122 	desc->name = NULL;
  123 	desc->owner = owner;
  124 	for_each_possible_cpu(cpu)
  125 		*per_cpu_ptr(desc->kstat_irqs, cpu) = 0;
  126 	desc_smp_init(desc, node, affinity);
  127 }
  128 
  129 int nr_irqs = NR_IRQS;
  130 EXPORT_SYMBOL_GPL(nr_irqs);
  131 
  132 static DEFINE_MUTEX(sparse_irq_lock);
  133 static struct maple_tree sparse_irqs = MTREE_INIT_EXT(sparse_irqs,
  134 					MT_FLAGS_ALLOC_RANGE |
  135 					MT_FLAGS_LOCK_EXTERN |
  136 					MT_FLAGS_USE_RCU,
  137 					sparse_irq_lock);
  138 
  139 static int irq_find_free_area(unsigned int from, unsigned int cnt)
  140 {
  141 	MA_STATE(mas, &sparse_irqs, 0, 0);
  142 
  143 	if (mas_empty_area(&mas, from, MAX_SPARSE_IRQS, cnt))
  144 		return -ENOSPC;
  145 	return mas.index;
  146 }
  147 
  148 static unsigned int irq_find_at_or_after(unsigned int offset)
  149 {
  150 	unsigned long index = offset;
  151 	struct irq_desc *desc = mt_find(&sparse_irqs, &index, nr_irqs);
  152 
  153 	return desc ? irq_desc_get_irq(desc) : nr_irqs;
  154 }
  155 
  156 static void irq_insert_desc(unsigned int irq, struct irq_desc *desc)
  157 {
  158 	MA_STATE(mas, &sparse_irqs, irq, irq);
  159 	WARN_ON(mas_store_gfp(&mas, desc, GFP_KERNEL) != 0);
  160 }
  161 
  162 static void delete_irq_desc(unsigned int irq)
  163 {
  164 	MA_STATE(mas, &sparse_irqs, irq, irq);
  165 	mas_erase(&mas);
  166 }
  167 
  168 #ifdef CONFIG_SPARSE_IRQ
  169 
  170 static void irq_kobj_release(struct kobject *kobj);
  171 
  172 #ifdef CONFIG_SYSFS
  173 static struct kobject *irq_kobj_base;
  174 
  175 #define IRQ_ATTR_RO(_name) \
  176 static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
  177 
  178 static ssize_t per_cpu_count_show(struct kobject *kobj,
  179 				  struct kobj_attribute *attr, char *buf)
  180 {
  181 	struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
  182 	ssize_t ret = 0;
  183 	char *p = "";
  184 	int cpu;
  185 
  186 	for_each_possible_cpu(cpu) {
  187 		unsigned int c = irq_desc_kstat_cpu(desc, cpu);
  188 
  189 		ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s%u", p, c);
  190 		p = ",";
  191 	}
  192 
  193 	ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n");
  194 	return ret;
  195 }
  196 IRQ_ATTR_RO(per_cpu_count);
  197 
  198 static ssize_t chip_name_show(struct kobject *kobj,
  199 			      struct kobj_attribute *attr, char *buf)
  200 {
  201 	struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
  202 	ssize_t ret = 0;
  203 
  204 	raw_spin_lock_irq(&desc->lock);
  205 	if (desc->irq_data.chip && desc->irq_data.chip->name) {
  206 		ret = scnprintf(buf, PAGE_SIZE, "%s\n",
  207 				desc->irq_data.chip->name);
  208 	}
  209 	raw_spin_unlock_irq(&desc->lock);
  210 
  211 	return ret;
  212 }
  213 IRQ_ATTR_RO(chip_name);
  214 
  215 static ssize_t hwirq_show(struct kobject *kobj,
  216 			  struct kobj_attribute *attr, char *buf)
  217 {
  218 	struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
  219 	ssize_t ret = 0;
  220 
  221 	raw_spin_lock_irq(&desc->lock);
  222 	if (desc->irq_data.domain)
  223 		ret = sprintf(buf, "%lu\n", desc->irq_data.hwirq);
  224 	raw_spin_unlock_irq(&desc->lock);
  225 
  226 	return ret;
  227 }
  228 IRQ_ATTR_RO(hwirq);
  229 
  230 static ssize_t type_show(struct kobject *kobj,
  231 			 struct kobj_attribute *attr, char *buf)
  232 {
  233 	struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
  234 	ssize_t ret = 0;
  235 
  236 	raw_spin_lock_irq(&desc->lock);
  237 	ret = sprintf(buf, "%s\n",
  238 		      irqd_is_level_type(&desc->irq_data) ? "level" : "edge");
  239 	raw_spin_unlock_irq(&desc->lock);
  240 
  241 	return ret;
  242 
  243 }
  244 IRQ_ATTR_RO(type);
  245 
  246 static ssize_t wakeup_show(struct kobject *kobj,
  247 			   struct kobj_attribute *attr, char *buf)
  248 {
  249 	struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
  250 	ssize_t ret = 0;
  251 
  252 	raw_spin_lock_irq(&desc->lock);
  253 	ret = sprintf(buf, "%s\n",
  254 		      irqd_is_wakeup_set(&desc->irq_data) ? "enabled" : "disabled");
  255 	raw_spin_unlock_irq(&desc->lock);
  256 
  257 	return ret;
  258 
  259 }
  260 IRQ_ATTR_RO(wakeup);
  261 
  262 static ssize_t name_show(struct kobject *kobj,
  263 			 struct kobj_attribute *attr, char *buf)
  264 {
  265 	struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
  266 	ssize_t ret = 0;
  267 
  268 	raw_spin_lock_irq(&desc->lock);
  269 	if (desc->name)
  270 		ret = scnprintf(buf, PAGE_SIZE, "%s\n", desc->name);
  271 	raw_spin_unlock_irq(&desc->lock);
  272 
  273 	return ret;
  274 }
  275 IRQ_ATTR_RO(name);
  276 
  277 static ssize_t actions_show(struct kobject *kobj,
  278 			    struct kobj_attribute *attr, char *buf)
  279 {
  280 	struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
  281 	struct irqaction *action;
  282 	ssize_t ret = 0;
  283 	char *p = "";
  284 
  285 	raw_spin_lock_irq(&desc->lock);
  286 	for_each_action_of_desc(desc, action) {
  287 		ret += scnprintf(buf + ret, PAGE_SIZE - ret, "%s%s",
  288 				 p, action->name);
  289 		p = ",";
  290 	}
  291 	raw_spin_unlock_irq(&desc->lock);
  292 
  293 	if (ret)
  294 		ret += scnprintf(buf + ret, PAGE_SIZE - ret, "\n");
  295 
  296 	return ret;
  297 }
  298 IRQ_ATTR_RO(actions);
  299 
  300 static struct attribute *irq_attrs[] = {
  301 	&per_cpu_count_attr.attr,
  302 	&chip_name_attr.attr,
  303 	&hwirq_attr.attr,
  304 	&type_attr.attr,
  305 	&wakeup_attr.attr,
  306 	&name_attr.attr,
  307 	&actions_attr.attr,
  308 	NULL
  309 };
  310 ATTRIBUTE_GROUPS(irq);
  311 
  312 static const struct kobj_type irq_kobj_type = {
  313 	.release	= irq_kobj_release,
  314 	.sysfs_ops	= &kobj_sysfs_ops,
  315 	.default_groups = irq_groups,
  316 };
  317 
  318 static void irq_sysfs_add(int irq, struct irq_desc *desc)
  319 {
  320 	if (irq_kobj_base) {
  321 		/*
  322 		 * Continue even in case of failure as this is nothing
  323 		 * crucial and failures in the late irq_sysfs_init()
  324 		 * cannot be rolled back.
  325 		 */
  326 		if (kobject_add(&desc->kobj, irq_kobj_base, "%d", irq))
  327 			pr_warn("Failed to add kobject for irq %d\n", irq);
  328 		else
  329 			desc->istate |= IRQS_SYSFS;
  330 	}
  331 }
  332 
  333 static void irq_sysfs_del(struct irq_desc *desc)
  334 {
  335 	/*
  336 	 * Only invoke kobject_del() when kobject_add() was successfully
  337 	 * invoked for the descriptor. This covers both early boot, where
  338 	 * sysfs is not initialized yet, and the case of a failed
  339 	 * kobject_add() invocation.
  340 	 */
  341 	if (desc->istate & IRQS_SYSFS)
  342 		kobject_del(&desc->kobj);
  343 }
  344 
  345 static int __init irq_sysfs_init(void)
  346 {
  347 	struct irq_desc *desc;
  348 	int irq;
  349 
  350 	/* Prevent concurrent irq alloc/free */
  351 	irq_lock_sparse();
  352 
  353 	irq_kobj_base = kobject_create_and_add("irq", kernel_kobj);
  354 	if (!irq_kobj_base) {
  355 		irq_unlock_sparse();
  356 		return -ENOMEM;
  357 	}
  358 
  359 	/* Add the already allocated interrupts */
  360 	for_each_irq_desc(irq, desc)
  361 		irq_sysfs_add(irq, desc);
  362 	irq_unlock_sparse();
  363 
  364 	return 0;
  365 }
  366 postcore_initcall(irq_sysfs_init);
  367 
  368 #else /* !CONFIG_SYSFS */
  369 
  370 static const struct kobj_type irq_kobj_type = {
  371 	.release	= irq_kobj_release,
  372 };
  373 
  374 static void irq_sysfs_add(int irq, struct irq_desc *desc) {}
  375 static void irq_sysfs_del(struct irq_desc *desc) {}
  376 
  377 #endif /* CONFIG_SYSFS */
  378 
  379 struct irq_desc *irq_to_desc(unsigned int irq)
  380 {
  381 	return mtree_load(&sparse_irqs, irq);
  382 }
  383 #ifdef CONFIG_KVM_BOOK3S_64_HV_MODULE
  384 EXPORT_SYMBOL_GPL(irq_to_desc);
  385 #endif
  386 
  387 #ifdef CONFIG_SMP
  388 static void free_masks(struct irq_desc *desc)
  389 {
  390 #ifdef CONFIG_GENERIC_PENDING_IRQ
  391 	free_cpumask_var(desc->pending_mask);
  392 #endif
  393 	free_cpumask_var(desc->irq_common_data.affinity);
  394 #ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
  395 	free_cpumask_var(desc->irq_common_data.effective_affinity);
  396 #endif
  397 }
  398 #else
  399 static inline void free_masks(struct irq_desc *desc) { }
  400 #endif
  401 
  402 void irq_lock_sparse(void)
  403 {
  404 	mutex_lock(&sparse_irq_lock);
  405 }
  406 
  407 void irq_unlock_sparse(void)
  408 {
  409 	mutex_unlock(&sparse_irq_lock);
  410 }
  411 
  412 static struct irq_desc *alloc_desc(int irq, int node, unsigned int flags,
  413 				   const struct cpumask *affinity,
  414 				   struct module *owner)
  415 {
  416 	struct irq_desc *desc;
  417 
  418 	desc = kzalloc_node(sizeof(*desc), GFP_KERNEL, node);
  419 	if (!desc)
  420 		return NULL;
  421 	/* allocate based on nr_cpu_ids */
  422 	desc->kstat_irqs = alloc_percpu(unsigned int);
  423 	if (!desc->kstat_irqs)
  424 		goto err_desc;
  425 
  426 	if (alloc_masks(desc, node))
  427 		goto err_kstat;
  428 
  429 	raw_spin_lock_init(&desc->lock);
  430 	lockdep_set_class(&desc->lock, &irq_desc_lock_class);
  431 	mutex_init(&desc->request_mutex);
  432 	init_rcu_head(&desc->rcu);
  433 	init_waitqueue_head(&desc->wait_for_threads);
  434 
  435 	desc_set_defaults(irq, desc, node, affinity, owner);
  436 	irqd_set(&desc->irq_data, flags);
  437 	kobject_init(&desc->kobj, &irq_kobj_type);
  438 	irq_resend_init(desc);
  439 
  440 	return desc;
  441 
  442 err_kstat:
  443 	free_percpu(desc->kstat_irqs);
  444 err_desc:
  445 	kfree(desc);
  446 	return NULL;
  447 }
  448 
  449 static void irq_kobj_release(struct kobject *kobj)
  450 {
  451 	struct irq_desc *desc = container_of(kobj, struct irq_desc, kobj);
  452 
  453 	free_masks(desc);
  454 	free_percpu(desc->kstat_irqs);
  455 	kfree(desc);
  456 }
  457 
  458 static void delayed_free_desc(struct rcu_head *rhp)
  459 {
  460 	struct irq_desc *desc = container_of(rhp, struct irq_desc, rcu);
  461 
  462 	kobject_put(&desc->kobj);
  463 }
  464 
  465 static void free_desc(unsigned int irq)
  466 {
  467 	struct irq_desc *desc = irq_to_desc(irq);
  468 
  469 	irq_remove_debugfs_entry(desc);
  470 	unregister_irq_proc(irq, desc);
  471 
  472 	/*
  473 	 * sparse_irq_lock protects also show_interrupts() and
  474 	 * kstat_irq_usr(). Once we deleted the descriptor from the
  475 	 * sparse tree we can free it. Access in proc will fail to
  476 	 * lookup the descriptor.
  477 	 *
  478 	 * The sysfs entry must be serialized against a concurrent
  479 	 * irq_sysfs_init() as well.
  480 	 */
  481 	irq_sysfs_del(desc);
  482 	delete_irq_desc(irq);
  483 
  484 	/*
  485 	 * We free the descriptor, masks and stat fields via RCU. That
  486 	 * allows demultiplex interrupts to do rcu based management of
  487 	 * the child interrupts.
  488 	 * This also allows us to use rcu in kstat_irqs_usr().
  489 	 */
  490 	call_rcu(&desc->rcu, delayed_free_desc);
  491 }
  492 
  493 static int alloc_descs(unsigned int start, unsigned int cnt, int node,
  494 		       const struct irq_affinity_desc *affinity,
  495 		       struct module *owner)
  496 {
  497 	struct irq_desc *desc;
  498 	int i;
  499 
  500 	/* Validate affinity mask(s) */
  501 	if (affinity) {
  502 		for (i = 0; i < cnt; i++) {
  503 			if (cpumask_empty(&affinity[i].mask))
  504 				return -EINVAL;
  505 		}
  506 	}
  507 
  508 	for (i = 0; i < cnt; i++) {
  509 		const struct cpumask *mask = NULL;
  510 		unsigned int flags = 0;
  511 
  512 		if (affinity) {
  513 			if (affinity->is_managed) {
  514 				flags = IRQD_AFFINITY_MANAGED |
  515 					IRQD_MANAGED_SHUTDOWN;
  516 			}
  517 			mask = &affinity->mask;
  518 			node = cpu_to_node(cpumask_first(mask));
  519 			affinity++;
  520 		}
  521 
  522 		desc = alloc_desc(start + i, node, flags, mask, owner);
  523 		if (!desc)
  524 			goto err;
  525 		irq_insert_desc(start + i, desc);
  526 		irq_sysfs_add(start + i, desc);
  527 		irq_add_debugfs_entry(start + i, desc);
  528 	}
  529 	return start;
  530 
  531 err:
  532 	for (i--; i >= 0; i--)
  533 		free_desc(start + i);
  534 	return -ENOMEM;
  535 }
  536 
  537 static int irq_expand_nr_irqs(unsigned int nr)
  538 {
  539 	if (nr > MAX_SPARSE_IRQS)
  540 		return -ENOMEM;
  541 	nr_irqs = nr;
  542 	return 0;
  543 }
  544 
  545 int __init early_irq_init(void)
  546 {
  547 	int i, initcnt, node = first_online_node;
  548 	struct irq_desc *desc;
  549 
  550 	init_irq_default_affinity();
  551 
  552 	/* Let arch update nr_irqs and return the nr of preallocated irqs */
  553 	initcnt = arch_probe_nr_irqs();
  554 	printk(KERN_INFO "NR_IRQS: %d, nr_irqs: %d, preallocated irqs: %d\n",
  555 	       NR_IRQS, nr_irqs, initcnt);
  556 
  557 	if (WARN_ON(nr_irqs > MAX_SPARSE_IRQS))
  558 		nr_irqs = MAX_SPARSE_IRQS;
  559 
  560 	if (WARN_ON(initcnt > MAX_SPARSE_IRQS))
  561 		initcnt = MAX_SPARSE_IRQS;
  562 
  563 	if (initcnt > nr_irqs)
  564 		nr_irqs = initcnt;
  565 
  566 	for (i = 0; i < initcnt; i++) {
  567 		desc = alloc_desc(i, node, 0, NULL, NULL);
  568 		irq_insert_desc(i, desc);
  569 	}
  570 	return arch_early_irq_init();
  571 }
  572 
  573 #else /* !CONFIG_SPARSE_IRQ */
  574 
  575 struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = {
  576 	[0 ... NR_IRQS-1] = {
  577 		.handle_irq	= handle_bad_irq,
  578 		.depth		= 1,
  579 		.lock		= __RAW_SPIN_LOCK_UNLOCKED(irq_desc->lock),
  580 	}
  581 };
  582 
  583 int __init early_irq_init(void)
  584 {
  585 	int count, i, node = first_online_node;
  586 	struct irq_desc *desc;
  587 
  588 	init_irq_default_affinity();
  589 
  590 	printk(KERN_INFO "NR_IRQS: %d\n", NR_IRQS);
  591 
  592 	desc = irq_desc;
  593 	count = ARRAY_SIZE(irq_desc);
  594 
  595 	for (i = 0; i < count; i++) {
  596 		desc[i].kstat_irqs = alloc_percpu(unsigned int);
  597 		alloc_masks(&desc[i], node);
  598 		raw_spin_lock_init(&desc[i].lock);
  599 		lockdep_set_class(&desc[i].lock, &irq_desc_lock_class);
  600 		mutex_init(&desc[i].request_mutex);
  601 		init_waitqueue_head(&desc[i].wait_for_threads);
  602 		desc_set_defaults(i, &desc[i], node, NULL, NULL);
  603 		irq_resend_init(desc);
  604 	}
  605 	return arch_early_irq_init();
  606 }
  607 
  608 struct irq_desc *irq_to_desc(unsigned int irq)
  609 {
  610 	return (irq < NR_IRQS) ? irq_desc + irq : NULL;
  611 }
  612 EXPORT_SYMBOL(irq_to_desc);
  613 
  614 static void free_desc(unsigned int irq)
  615 {
  616 	struct irq_desc *desc = irq_to_desc(irq);
  617 	unsigned long flags;
  618 
  619 	raw_spin_lock_irqsave(&desc->lock, flags);
  620 	desc_set_defaults(irq, desc, irq_desc_get_node(desc), NULL, NULL);
  621 	raw_spin_unlock_irqrestore(&desc->lock, flags);
  622 	delete_irq_desc(irq);
  623 }
  624 
  625 static inline int alloc_descs(unsigned int start, unsigned int cnt, int node,
  626 			      const struct irq_affinity_desc *affinity,
  627 			      struct module *owner)
  628 {
  629 	u32 i;
  630 
  631 	for (i = 0; i < cnt; i++) {
  632 		struct irq_desc *desc = irq_to_desc(start + i);
  633 
  634 		desc->owner = owner;
  635 		irq_insert_desc(start + i, desc);
  636 	}
  637 	return start;
  638 }
  639 
  640 static int irq_expand_nr_irqs(unsigned int nr)
  641 {
  642 	return -ENOMEM;
  643 }
  644 
  645 void irq_mark_irq(unsigned int irq)
  646 {
  647 	mutex_lock(&sparse_irq_lock);
  648 	irq_insert_desc(irq, irq_desc + irq);
  649 	mutex_unlock(&sparse_irq_lock);
  650 }
  651 
  652 #ifdef CONFIG_GENERIC_IRQ_LEGACY
  653 void irq_init_desc(unsigned int irq)
  654 {
  655 	free_desc(irq);
  656 }
  657 #endif
  658 
  659 #endif /* !CONFIG_SPARSE_IRQ */
  660 
  661 int handle_irq_desc(struct irq_desc *desc)
  662 {
  663 	struct irq_data *data;
  664 
  665 	if (!desc)
  666 		return -EINVAL;
  667 
  668 	data = irq_desc_get_irq_data(desc);
  669 	if (WARN_ON_ONCE(!in_hardirq() && handle_enforce_irqctx(data)))
  670 		return -EPERM;
  671 
  672 	generic_handle_irq_desc(desc);
  673 	return 0;
  674 }
  675 
  676 /**
  677  * generic_handle_irq - Invoke the handler for a particular irq
  678  * @irq:	The irq number to handle
  679  *
  680  * Returns:	0 on success, or -EINVAL if conversion has failed
  681  *
  682  * 		This function must be called from an IRQ context with irq regs
  683  * 		initialized.
  684   */
  685 int generic_handle_irq(unsigned int irq)
  686 {
  687 	return handle_irq_desc(irq_to_desc(irq));
  688 }
  689 EXPORT_SYMBOL_GPL(generic_handle_irq);
  690 
  691 /**
  692  * generic_handle_irq_safe - Invoke the handler for a particular irq from any
  693  *			     context.
  694  * @irq:	The irq number to handle
  695  *
  696  * Returns:	0 on success, a negative value on error.
  697  *
  698  * This function can be called from any context (IRQ or process context). It
  699  * will report an error if not invoked from IRQ context and the irq has been
  700  * marked to enforce IRQ-context only.
  701  */
  702 int generic_handle_irq_safe(unsigned int irq)
  703 {
  704 	unsigned long flags;
  705 	int ret;
  706 
  707 	local_irq_save(flags);
  708 	ret = handle_irq_desc(irq_to_desc(irq));
  709 	local_irq_restore(flags);
  710 	return ret;
  711 }
  712 EXPORT_SYMBOL_GPL(generic_handle_irq_safe);
  713 
  714 #ifdef CONFIG_IRQ_DOMAIN
  715 /**
  716  * generic_handle_domain_irq - Invoke the handler for a HW irq belonging
  717  *                             to a domain.
  718  * @domain:	The domain where to perform the lookup
  719  * @hwirq:	The HW irq number to convert to a logical one
  720  *
  721  * Returns:	0 on success, or -EINVAL if conversion has failed
  722  *
  723  * 		This function must be called from an IRQ context with irq regs
  724  * 		initialized.
  725  */
  726 int generic_handle_domain_irq(struct irq_domain *domain, unsigned int hwirq)
  727 {
  728 	return handle_irq_desc(irq_resolve_mapping(domain, hwirq));
  729 }
  730 EXPORT_SYMBOL_GPL(generic_handle_domain_irq);
  731 
  732  /**
  733  * generic_handle_irq_safe - Invoke the handler for a HW irq belonging
  734  *			     to a domain from any context.
  735  * @domain:	The domain where to perform the lookup
  736  * @hwirq:	The HW irq number to convert to a logical one
  737  *
  738  * Returns:	0 on success, a negative value on error.
  739  *
  740  * This function can be called from any context (IRQ or process
  741  * context). If the interrupt is marked as 'enforce IRQ-context only' then
  742  * the function must be invoked from hard interrupt context.
  743  */
  744 int generic_handle_domain_irq_safe(struct irq_domain *domain, unsigned int hwirq)
  745 {
  746 	unsigned long flags;
  747 	int ret;
  748 
  749 	local_irq_save(flags);
  750 	ret = handle_irq_desc(irq_resolve_mapping(domain, hwirq));
  751 	local_irq_restore(flags);
  752 	return ret;
  753 }
  754 EXPORT_SYMBOL_GPL(generic_handle_domain_irq_safe);
  755 
  756 /**
  757  * generic_handle_domain_nmi - Invoke the handler for a HW nmi belonging
  758  *                             to a domain.
  759  * @domain:	The domain where to perform the lookup
  760  * @hwirq:	The HW irq number to convert to a logical one
  761  *
  762  * Returns:	0 on success, or -EINVAL if conversion has failed
  763  *
  764  * 		This function must be called from an NMI context with irq regs
  765  * 		initialized.
  766  **/
  767 int generic_handle_domain_nmi(struct irq_domain *domain, unsigned int hwirq)
  768 {
  769 	WARN_ON_ONCE(!in_nmi());
  770 	return handle_irq_desc(irq_resolve_mapping(domain, hwirq));
  771 }
  772 #endif
  773 
  774 /* Dynamic interrupt handling */
  775 
  776 /**
  777  * irq_free_descs - free irq descriptors
  778  * @from:	Start of descriptor range
  779  * @cnt:	Number of consecutive irqs to free
  780  */
  781 void irq_free_descs(unsigned int from, unsigned int cnt)
  782 {
  783 	int i;
  784 
  785 	if (from >= nr_irqs || (from + cnt) > nr_irqs)
  786 		return;
  787 
  788 	mutex_lock(&sparse_irq_lock);
  789 	for (i = 0; i < cnt; i++)
  790 		free_desc(from + i);
  791 
  792 	mutex_unlock(&sparse_irq_lock);
  793 }
  794 EXPORT_SYMBOL_GPL(irq_free_descs);
  795 
  796 /**
  797  * __irq_alloc_descs - allocate and initialize a range of irq descriptors
  798  * @irq:	Allocate for specific irq number if irq >= 0
  799  * @from:	Start the search from this irq number
  800  * @cnt:	Number of consecutive irqs to allocate.
  801  * @node:	Preferred node on which the irq descriptor should be allocated
  802  * @owner:	Owning module (can be NULL)
  803  * @affinity:	Optional pointer to an affinity mask array of size @cnt which
  804  *		hints where the irq descriptors should be allocated and which
  805  *		default affinities to use
  806  *
  807  * Returns the first irq number or error code
  808  */
  809 int __ref
  810 __irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int node,
  811 		  struct module *owner, const struct irq_affinity_desc *affinity)
  812 {
  813 	int start, ret;
  814 
  815 	if (!cnt)
  816 		return -EINVAL;
  817 
  818 	if (irq >= 0) {
  819 		if (from > irq)
  820 			return -EINVAL;
  821 		from = irq;
  822 	} else {
  823 		/*
  824 		 * For interrupts which are freely allocated the
  825 		 * architecture can force a lower bound to the @from
  826 		 * argument. x86 uses this to exclude the GSI space.
  827 		 */
  828 		from = arch_dynirq_lower_bound(from);
  829 	}
  830 
  831 	mutex_lock(&sparse_irq_lock);
  832 
  833 	start = irq_find_free_area(from, cnt);
  834 	ret = -EEXIST;
  835 	if (irq >=0 && start != irq)
  836 		goto unlock;
  837 
  838 	if (start + cnt > nr_irqs) {
  839 		ret = irq_expand_nr_irqs(start + cnt);
  840 		if (ret)
  841 			goto unlock;
  842 	}
  843 	ret = alloc_descs(start, cnt, node, affinity, owner);
  844 unlock:
  845 	mutex_unlock(&sparse_irq_lock);
  846 	return ret;
  847 }
  848 EXPORT_SYMBOL_GPL(__irq_alloc_descs);
  849 
  850 /**
  851  * irq_get_next_irq - get next allocated irq number
  852  * @offset:	where to start the search
  853  *
  854  * Returns next irq number after offset or nr_irqs if none is found.
  855  */
  856 unsigned int irq_get_next_irq(unsigned int offset)
  857 {
  858 	return irq_find_at_or_after(offset);
  859 }
  860 
  861 struct irq_desc *
  862 __irq_get_desc_lock(unsigned int irq, unsigned long *flags, bool bus,
  863 		    unsigned int check)
  864 {
  865 	struct irq_desc *desc = irq_to_desc(irq);
  866 
  867 	if (desc) {
  868 		if (check & _IRQ_DESC_CHECK) {
  869 			if ((check & _IRQ_DESC_PERCPU) &&
  870 			    !irq_settings_is_per_cpu_devid(desc))
  871 				return NULL;
  872 
  873 			if (!(check & _IRQ_DESC_PERCPU) &&
  874 			    irq_settings_is_per_cpu_devid(desc))
  875 				return NULL;
  876 		}
  877 
  878 		if (bus)
  879 			chip_bus_lock(desc);
  880 		raw_spin_lock_irqsave(&desc->lock, *flags);
  881 	}
  882 	return desc;
  883 }
  884 
  885 void __irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags, bool bus)
  886 	__releases(&desc->lock)
  887 {
  888 	raw_spin_unlock_irqrestore(&desc->lock, flags);
  889 	if (bus)
  890 		chip_bus_sync_unlock(desc);
  891 }
  892 
  893 int irq_set_percpu_devid_partition(unsigned int irq,
  894 				   const struct cpumask *affinity)
  895 {
  896 	struct irq_desc *desc = irq_to_desc(irq);
  897 
  898 	if (!desc)
  899 		return -EINVAL;
  900 
  901 	if (desc->percpu_enabled)
  902 		return -EINVAL;
  903 
  904 	desc->percpu_enabled = kzalloc(sizeof(*desc->percpu_enabled), GFP_KERNEL);
  905 
  906 	if (!desc->percpu_enabled)
  907 		return -ENOMEM;
  908 
  909 	if (affinity)
  910 		desc->percpu_affinity = affinity;
  911 	else
  912 		desc->percpu_affinity = cpu_possible_mask;
  913 
  914 	irq_set_percpu_devid_flags(irq);
  915 	return 0;
  916 }
  917 
  918 int irq_set_percpu_devid(unsigned int irq)
  919 {
  920 	return irq_set_percpu_devid_partition(irq, NULL);
  921 }
  922 
  923 int irq_get_percpu_devid_partition(unsigned int irq, struct cpumask *affinity)
  924 {
  925 	struct irq_desc *desc = irq_to_desc(irq);
  926 
  927 	if (!desc || !desc->percpu_enabled)
  928 		return -EINVAL;
  929 
  930 	if (affinity)
  931 		cpumask_copy(affinity, desc->percpu_affinity);
  932 
  933 	return 0;
  934 }
  935 EXPORT_SYMBOL_GPL(irq_get_percpu_devid_partition);
  936 
  937 void kstat_incr_irq_this_cpu(unsigned int irq)
  938 {
  939 	kstat_incr_irqs_this_cpu(irq_to_desc(irq));
  940 }
  941 
  942 /**
  943  * kstat_irqs_cpu - Get the statistics for an interrupt on a cpu
  944  * @irq:	The interrupt number
  945  * @cpu:	The cpu number
  946  *
  947  * Returns the sum of interrupt counts on @cpu since boot for
  948  * @irq. The caller must ensure that the interrupt is not removed
  949  * concurrently.
  950  */
  951 unsigned int kstat_irqs_cpu(unsigned int irq, int cpu)
  952 {
  953 	struct irq_desc *desc = irq_to_desc(irq);
  954 
  955 	return desc && desc->kstat_irqs ?
  956 			*per_cpu_ptr(desc->kstat_irqs, cpu) : 0;
  957 }
  958 
  959 static bool irq_is_nmi(struct irq_desc *desc)
  960 {
  961 	return desc->istate & IRQS_NMI;
  962 }
  963 
  964 static unsigned int kstat_irqs(unsigned int irq)
  965 {
  966 	struct irq_desc *desc = irq_to_desc(irq);
  967 	unsigned int sum = 0;
  968 	int cpu;
  969 
  970 	if (!desc || !desc->kstat_irqs)
  971 		return 0;
  972 	if (!irq_settings_is_per_cpu_devid(desc) &&
  973 	    !irq_settings_is_per_cpu(desc) &&
  974 	    !irq_is_nmi(desc))
  975 		return data_race(desc->tot_count);
  976 
  977 	for_each_possible_cpu(cpu)
  978 		sum += data_race(*per_cpu_ptr(desc->kstat_irqs, cpu));
  979 	return sum;
  980 }
  981 
  982 /**
  983  * kstat_irqs_usr - Get the statistics for an interrupt from thread context
  984  * @irq:	The interrupt number
  985  *
  986  * Returns the sum of interrupt counts on all cpus since boot for @irq.
  987  *
  988  * It uses rcu to protect the access since a concurrent removal of an
  989  * interrupt descriptor is observing an rcu grace period before
  990  * delayed_free_desc()/irq_kobj_release().
  991  */
  992 unsigned int kstat_irqs_usr(unsigned int irq)
  993 {
  994 	unsigned int sum;
  995 
  996 	rcu_read_lock();
  997 	sum = kstat_irqs(irq);
  998 	rcu_read_unlock();
  999 	return sum;
 1000 }
 1001 
 1002 #ifdef CONFIG_LOCKDEP
 1003 void __irq_set_lockdep_class(unsigned int irq, struct lock_class_key *lock_class,
 1004 			     struct lock_class_key *request_class)
 1005 {
 1006 	struct irq_desc *desc = irq_to_desc(irq);
 1007 
 1008 	if (desc) {
 1009 		lockdep_set_class(&desc->lock, lock_class);
 1010 		lockdep_set_class(&desc->request_mutex, request_class);
 1011 	}
 1012 }
 1013 EXPORT_SYMBOL_GPL(__irq_set_lockdep_class);
 1014 #endif