PCI/VGA: Select VGA devices earlier

Select VGA devices in vga_arb_device_init() and pci_notify() instead of in
vga_arbiter_add_pci_device().

This is a trivial optimization for adding devices.  It's a bigger
optimization for the removal case because pci_notify() won't call
vga_arbiter_del_pci_device() for non-VGA devices, so it won't have to
search the vga_list for them.

https://lore.kernel.org/r/20230830111532.444535-3-sui.jingfeng@linux.dev
Signed-off-by: Sui Jingfeng <suijingfeng@loongson.cn>
[bhelgaas: commit log, split from functional change]
Signed-off-by: Bjorn Helgaas <bhelgaas@google.com>

Author: Sui Jingfeng

drivers/pci/vgaarb.c

@@ -764,10 +764,6 @@ static bool vga_arbiter_add_pci_device(struct pci_dev *pdev)
 	struct pci_dev *bridge;
 	u16 cmd;
 
-	/* Only deal with VGA class devices */
-	if (!pci_is_vga(pdev))
-		return false;
-
 	/* Allocate structure */
 	vgadev = kzalloc(sizeof(struct vga_device), GFP_KERNEL);
 	if (vgadev == NULL) {
@@ -1503,6 +1499,10 @@ static int pci_notify(struct notifier_block *nb, unsigned long action,
 
 	vgaarb_dbg(dev, "%s\n", __func__);
 
+	/* Only deal with VGA class devices */
+	if (!pci_is_vga(pdev))
+		return 0;
+
 	/*
 	 * For now, we're only interested in devices added and removed.
 	 * I didn't test this thing here, so someone needs to double check
@@ -1550,8 +1550,10 @@ static int __init vga_arb_device_init(void)
 	pdev = NULL;
 	while ((pdev =
 		pci_get_subsys(PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
-			       PCI_ANY_ID, pdev)) != NULL)
-		vga_arbiter_add_pci_device(pdev);
+			       PCI_ANY_ID, pdev)) != NULL) {
+		if (pci_is_vga(pdev))
+			vga_arbiter_add_pci_device(pdev);
+	}
 
 	pr_info("loaded\n");
 	return rc;