Index: trunk/src/org/openstreetmap/josm/gui/preferences/plugin/PluginListPanel.java
===================================================================
--- trunk/src/org/openstreetmap/josm/gui/preferences/plugin/PluginListPanel.java	(revision 17783)
+++ trunk/src/org/openstreetmap/josm/gui/preferences/plugin/PluginListPanel.java	(revision 17784)
@@ -10,6 +10,10 @@
 import java.awt.event.MouseAdapter;
 import java.awt.event.MouseEvent;
+import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
-
+import java.util.Set;
+
+import javax.swing.JComponent;
 import javax.swing.JLabel;
 import javax.swing.SwingConstants;
@@ -38,4 +42,7 @@
 
     private final transient PluginPreferencesModel model;
+
+    /** Whether the plugin list has been built up already in the UI. */
+    private boolean pluginListInitialized = false;
 
     /**
@@ -105,4 +112,5 @@
                 + "</html>"
         );
+        hint.putClientProperty("plugin", "empty");
         add(hint, gbc);
     }
@@ -142,8 +150,10 @@
             gbc.weighty = 0.0;
             gbc.weightx = 0.0;
+            cbPlugin.putClientProperty("plugin", pi);
             add(cbPlugin, gbc);
 
             gbc.gridx = 1;
             gbc.weightx = 1.0;
+            lblPlugin.putClientProperty("plugin", pi);
             add(lblPlugin, gbc);
 
@@ -157,20 +167,29 @@
             gbc.insets = new Insets(3, 25, 5, 5);
             gbc.weighty = 1.0;
+            description.putClientProperty("plugin", pi);
             add(description, gbc);
         }
+
+        pluginListInitialized = true;
     }
 
     /**
      * Refreshes the list.
+     *
+     * If the list has been changed completely (i.e. not just filtered),
+     * call {@link #resetDisplayedComponents()} prior to calling this method.
      */
     public void refreshView() {
         final Rectangle visibleRect = getVisibleRect();
         List<PluginInformation> displayedPlugins = model.getDisplayedPlugins();
-        removeAll();
 
         if (displayedPlugins.isEmpty()) {
+            hidePluginsNotInList(new ArrayList<>());
             displayEmptyPluginListInformation();
+        } else if (!pluginListInitialized) {
+            removeAll();
+            displayPluginList(displayedPlugins);
         } else {
-            displayPluginList(displayedPlugins);
+            hidePluginsNotInList(displayedPlugins);
         }
         revalidate();
@@ -179,4 +198,45 @@
     }
 
+    /**
+     * Hides components in the list for plugins that are currently filtered away.
+     *
+     * Since those components are relatively heavyweight rebuilding them every time
+     * when the filter changes is fairly slow, so we build them once and just hide
+     * those that shouldn't be visible.
+     *
+     * @param displayedPlugins A collection of plugins that are currently visible.
+     */
+    private void hidePluginsNotInList(List<PluginInformation> displayedPlugins) {
+        // Remove the empty plugin list warning if it's there
+        synchronized (getTreeLock()) {
+            for (int i = 0; i < getComponentCount(); i++) {
+                JComponent component = (JComponent) getComponent(i);
+                if ("empty".equals(component.getClientProperty("plugin"))) {
+                    remove(component);
+                }
+            }
+        }
+
+        Set<PluginInformation> displayedPluginsSet = new HashSet<>(displayedPlugins);
+        synchronized (getTreeLock()) {
+            for (int i = 0; i < getComponentCount(); i++) {
+                JComponent component = (JComponent) getComponent(i);
+                Object plugin = component.getClientProperty("plugin");
+                component.setVisible(displayedPluginsSet.contains(plugin));
+            }
+        }
+    }
+
+    /**
+     * Causes the components for the list items to be rebuilt from scratch.
+     *
+     * Should be called before calling {@link #refreshView()} whenever the
+     * underlying list changes to display a completely different set of
+     * plugins instead of merely hiding plugins by a filter.
+     */
+    public void resetDisplayedComponents() {
+        pluginListInitialized = false;
+    }
+
     @Override
     public int getScrollableUnitIncrement(Rectangle visibleRect, int orientation, int direction) {
Index: trunk/src/org/openstreetmap/josm/gui/preferences/plugin/PluginPreference.java
===================================================================
--- trunk/src/org/openstreetmap/josm/gui/preferences/plugin/PluginPreference.java	(revision 17783)
+++ trunk/src/org/openstreetmap/josm/gui/preferences/plugin/PluginPreference.java	(revision 17784)
@@ -323,4 +323,5 @@
                 SwingUtilities.invokeLater(() -> {
                     model.setAvailablePlugins(task.getAvailablePlugins());
+                    pnlPluginPreferences.resetDisplayedComponents();
                     pnlPluginPreferences.refreshView();
                 });
@@ -356,4 +357,5 @@
                     SwingUtilities.invokeLater(() -> {
                         model.updateAvailablePlugins(task.getAvailablePlugins());
+                        pnlPluginPreferences.resetDisplayedComponents();
                         pnlPluginPreferences.refreshView();
                         Config.getPref().putInt("pluginmanager.version", Version.getInstance().getVersion()); // fix #7030