Index: src/org/openstreetmap/josm/actions/mapmode/SelectAction.java
===================================================================
--- src/org/openstreetmap/josm/actions/mapmode/SelectAction.java	(revision 1418)
+++ src/org/openstreetmap/josm/actions/mapmode/SelectAction.java	(working copy)
@@ -295,6 +295,11 @@
         boolean ctrl = (e.getModifiers() & ActionEvent.CTRL_MASK) != 0;
         // boolean alt = (e.getModifiers() & ActionEvent.ALT_MASK) != 0;
         boolean shift = (e.getModifiers() & ActionEvent.SHIFT_MASK) != 0;
+        
+        // We don't want to change to draw tool if the user tries to (de)select
+        // stuff but accidentally clicks in an empty area when selection is empty
+        if(shift || ctrl)
+            cancelDrawMode = true;
 
         mouseDownTime = System.currentTimeMillis();
         didMove = false;