diff --git a/src/shell/atuin.fish b/src/shell/atuin.fish
index f675d63..69175ad 100644
--- a/src/shell/atuin.fish
+++ b/src/shell/atuin.fish
@@ -23,10 +23,19 @@ function _atuin_search
 end
 
 function _atuin_bind_up
-    if commandline -P
+    # Fallback to fish's builtin up-or-search if we're in search or paging mode
+    if commandline --search-mode; or commandline --paging-mode
         up-or-search
-    else
-        _atuin_search
+        return
+    end
+
+    # Only invoke atuin if we're on the top line of the command
+    set -l lineno (commandline --line)
+    switch $lineno
+        case 1
+            _atuin_search
+        case '*'
+            up-or-search
     end
 end