--- a/src/kitemviews/private/kitemlistheaderwidget.cpp
+++ b/src/kitemviews/private/kitemlistheaderwidget.cpp
@@ -361,6 +361,10 @@
         // synchronized on each further mouse-move-event with the mouse-position.
         const int roleIndex = roleIndexAt(m_pressedMousePos);
         m_movingRole.index = roleIndex;
+        if (roleIndex < 0) {
+            // Trying to drag side padding
+            return;
+        }
         if (roleIndex == nameColumnIndex(this)) {
             // TODO: It should be configurable whether moving the first role is allowed.
             // In the context of Dolphin this is not required, however this should be
