diff --git a/src/edit.rs b/src/edit.rs
index b18de9e2e..dd856b9e4 100644
--- a/src/edit.rs
+++ b/src/edit.rs
@@ -578,7 +578,7 @@ impl<H: Helper> State<'_, '_, H> {
 
     /// Moves the cursor to the same column in the line above
     pub fn edit_move_line_up(&mut self, n: RepeatCount) -> Result<bool> {
-        if self.line.move_to_line_up(n) {
+        if self.line.move_to_line_up(n, &self.layout) {
             self.move_cursor(CmdKind::MoveCursor)?;
             Ok(true)
         } else {
@@ -588,7 +588,7 @@ impl<H: Helper> State<'_, '_, H> {
 
     /// Moves the cursor to the same column in the line above
     pub fn edit_move_line_down(&mut self, n: RepeatCount) -> Result<bool> {
-        if self.line.move_to_line_down(n) {
+        if self.line.move_to_line_down(n, &self.layout) {
             self.move_cursor(CmdKind::MoveCursor)?;
             Ok(true)
         } else {
diff --git a/src/line_buffer.rs b/src/line_buffer.rs
index 3f2dda9a5..4a10cae81 100644
--- a/src/line_buffer.rs
+++ b/src/line_buffer.rs
@@ -1,5 +1,6 @@
 //! Line buffer with current cursor position
 use crate::keymap::{At, CharSearch, Movement, RepeatCount, Word};
+use crate::layout::Layout;
 use std::cmp::min;
 use std::fmt;
 use std::iter;
@@ -582,7 +583,7 @@ impl LineBuffer {
     }
 
     /// Moves the cursor to the same column in the line above
-    pub fn move_to_line_up(&mut self, n: RepeatCount) -> bool {
+    pub fn move_to_line_up(&mut self, n: RepeatCount, layout: &Layout) -> bool {
         match self.buf[..self.pos].rfind('\n') {
             Some(off) => {
                 let column = self.buf[off + 1..self.pos].graphemes(true).count();
@@ -596,9 +597,14 @@ impl LineBuffer {
                     dest_end = dest_start - 1;
                     dest_start = self.buf[..dest_end].rfind('\n').map_or(0, |n| n + 1);
                 }
+                let offset = if dest_start == 0 {
+                    layout.prompt_size.col
+                } else {
+                    0
+                };
                 let gidx = self.buf[dest_start..dest_end]
                     .grapheme_indices(true)
-                    .nth(column);
+                    .nth(column.saturating_sub(offset));
 
                 self.pos = gidx.map_or(off, |(idx, _)| dest_start + idx); // if there's no enough columns
                 true
@@ -652,11 +658,16 @@ impl LineBuffer {
     }
 
     /// Moves the cursor to the same column in the line above
-    pub fn move_to_line_down(&mut self, n: RepeatCount) -> bool {
+    pub fn move_to_line_down(&mut self, n: RepeatCount, layout: &Layout) -> bool {
         match self.buf[self.pos..].find('\n') {
             Some(off) => {
                 let line_start = self.buf[..self.pos].rfind('\n').map_or(0, |n| n + 1);
-                let column = self.buf[line_start..self.pos].graphemes(true).count();
+                let offset = if line_start == 0 {
+                    layout.prompt_size.col
+                } else {
+                    0
+                };
+                let column = self.buf[line_start..self.pos].graphemes(true).count() + offset;
                 let mut dest_start = self.pos + off + 1;
                 let mut dest_end = self.buf[dest_start..]
                     .find('\n')
@@ -1182,7 +1193,10 @@ mod test {
     use super::{
         ChangeListener, DeleteListener, Direction, LineBuffer, NoListener, WordAction, MAX_LINE,
     };
-    use crate::keymap::{At, CharSearch, Word};
+    use crate::{
+        keymap::{At, CharSearch, Word},
+        layout::Layout,
+    };
 
     struct Listener {
         deleted_str: Option<String>,
@@ -1819,40 +1833,50 @@ mod test {
     fn move_by_line() {
         let text = "aa123\nsdf bc\nasdf";
         let mut s = LineBuffer::init(text, 14);
+        let mut layout = Layout::default();
         // move up
-        let ok = s.move_to_line_up(1);
+        let ok = s.move_to_line_up(1, &layout);
         assert_eq!(7, s.pos);
         assert!(ok);
 
-        let ok = s.move_to_line_up(1);
+        let ok = s.move_to_line_up(1, &layout);
         assert_eq!(1, s.pos);
         assert!(ok);
 
-        let ok = s.move_to_line_up(1);
+        let ok = s.move_to_line_up(1, &layout);
         assert_eq!(1, s.pos);
         assert!(!ok);
 
         // move down
-        let ok = s.move_to_line_down(1);
+        let ok = s.move_to_line_down(1, &layout);
         assert_eq!(7, s.pos);
         assert!(ok);
 
-        let ok = s.move_to_line_down(1);
+        let ok = s.move_to_line_down(1, &layout);
         assert_eq!(14, s.pos);
         assert!(ok);
 
-        let ok = s.move_to_line_down(1);
+        let ok = s.move_to_line_down(1, &layout);
         assert_eq!(14, s.pos);
         assert!(!ok);
 
         // move by multiple steps
-        let ok = s.move_to_line_up(2);
+        let ok = s.move_to_line_up(2, &layout);
         assert_eq!(1, s.pos);
         assert!(ok);
 
-        let ok = s.move_to_line_down(2);
+        let ok = s.move_to_line_down(2, &layout);
         assert_eq!(14, s.pos);
         assert!(ok);
+
+        // non-empty prompt
+        layout.prompt_size.col = 2;
+        s.move_to_line_up(1, &layout);
+        assert_eq!(7, s.pos);
+        s.move_to_line_up(1, &layout);
+        assert_eq!(0, s.pos);
+        s.move_to_line_down(1, &layout);
+        assert_eq!(8, s.pos);
     }
 
     #[test]