Index: trunk/src/org/openstreetmap/josm/gui/MapFrame.java
===================================================================
--- trunk/src/org/openstreetmap/josm/gui/MapFrame.java	(revision 12078)
+++ trunk/src/org/openstreetmap/josm/gui/MapFrame.java	(revision 12079)
@@ -496,5 +496,4 @@
         if (Main.pref.getBoolean("sidetoolbar.scrollable", true)) {
             final ScrollViewport svp = new ScrollViewport(sideToolBar, ScrollViewport.VERTICAL_DIRECTION);
-            svp.addMouseWheelListener(e -> svp.scroll(0, e.getUnitsToScroll() * 5));
             sideToolBar = svp;
         }
Index: trunk/src/org/openstreetmap/josm/gui/ScrollViewport.java
===================================================================
--- trunk/src/org/openstreetmap/josm/gui/ScrollViewport.java	(revision 12078)
+++ trunk/src/org/openstreetmap/josm/gui/ScrollViewport.java	(revision 12079)
@@ -141,4 +141,10 @@
         showOrHideButtons();
 
+        if ((direction & VERTICAL_DIRECTION) != 0) {
+            addMouseWheelListener(e -> scroll(0, e.getUnitsToScroll() * 5));
+        } else if ((direction & HORIZONTAL_DIRECTION) != 0) {
+            addMouseWheelListener(e -> scroll(e.getUnitsToScroll() * 5, 0));
+        }
+
         timer.setRepeats(true);
         timer.setInitialDelay(400);
@@ -248,10 +254,25 @@
     @Override
     public Dimension getPreferredSize() {
-        return vp.getPreferredSize();
+        if (component == null) {
+            return vp.getPreferredSize();
+        } else {
+            return component.getPreferredSize();
+        }
     }
 
     @Override
     public Dimension getMinimumSize() {
-        return vp.getMinimumSize();
+        if (component == null) {
+            return vp.getMinimumSize();
+        } else {
+            Dimension minSize = component.getMinimumSize();
+            if ((allowedScrollDirections & HORIZONTAL_DIRECTION) != 0) {
+                minSize = new Dimension(20, minSize.height);
+            }
+            if ((allowedScrollDirections & VERTICAL_DIRECTION) != 0) {
+                minSize = new Dimension(minSize.width, 20);
+            }
+            return minSize;
+        }
     }