myriation/atuin
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Importer V3 (#395)

Browse source 
* start of importer refactor

* fish

* resh

* zsh
This commit is contained in:
Conrad Ludgate 2022-05-09 07:46:52 +01:00 committed by GitHub
parent d3a4ff959b
commit 1d030b9d32
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 476 additions and 566 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
Cargo.lock generated
View file

@ -118,6 +118,7 @@ dependencies = [
"itertools",
"lazy_static",
"log",
"memchr",
"minspan",
"parse_duration",
"regex",
@ -1189,9 +1190,9 @@ dependencies = [
[[package]]
name = "memchr"
version = "2.4.1"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
[[package]]
name = "mime"

1
atuin-client/Cargo.toml
View file

@ -49,6 +49,7 @@ regex = "1.5.4"
fs-err = "2.7"
sql-builder = "3"
lazy_static = "1"
memchr = "2.5"
# sync
urlencoding = { version = "2.1.0", optional = true }

2
atuin-client/src/database.rs
View file

@ -41,7 +41,7 @@ pub fn current_context() -> Context {
}
#[async_trait]
pub trait Database {
pub trait Database: Send + Sync {
async fn save(&mut self, h: &History) -> Result<()>;
async fn save_bulk(&mut self, h: &[History]) -> Result<()>;

162
atuin-client/src/import/bash.rs
View file

@ -1,134 +1,106 @@
use std::{
fs::File,
io::{BufRead, BufReader, Read, Seek},
path::{Path, PathBuf},
};
use std::{fs::File, io::Read, path::PathBuf};
use async_trait::async_trait;
use directories::UserDirs;
use eyre::{eyre, Result};
use super::{count_lines, Importer};
use super::{get_histpath, unix_byte_lines, Importer, Loader};
use crate::history::History;
#[derive(Debug)]
pub struct Bash<R> {
file: BufReader<R>,
strbuf: String,
loc: usize,
counter: i64,
pub struct Bash {
bytes: Vec<u8>,
}
impl<R: Read + Seek> Bash<R> {
fn new(r: R) -> Result<Self> {
let mut buf = BufReader::new(r);
let loc = count_lines(&mut buf)?;
fn default_histpath() -> Result<PathBuf> {
let user_dirs = UserDirs::new().ok_or_else(|| eyre!("could not find user directories"))?;
let home_dir = user_dirs.home_dir();
Ok(Self {
file: buf,
strbuf: String::new(),
loc,
counter: 0,
})
}
Ok(home_dir.join(".bash_history"))
}
impl Importer for Bash<File> {
#[async_trait]
impl Importer for Bash {
const NAME: &'static str = "bash";
fn histpath() -> Result<PathBuf> {
let user_dirs = UserDirs::new().unwrap();
let home_dir = user_dirs.home_dir();
Ok(home_dir.join(".bash_history"))
async fn new() -> Result<Self> {
let mut bytes = Vec::new();
let path = get_histpath(default_histpath)?;
let mut f = File::open(path)?;
f.read_to_end(&mut bytes)?;
Ok(Self { bytes })
}
fn parse(path: impl AsRef<Path>) -> Result<Self> {
Self::new(File::open(path)?)
async fn entries(&mut self) -> Result<usize> {
Ok(super::count_lines(&self.bytes))
}
}
impl<R: Read> Iterator for Bash<R> {
type Item = Result<History>;
async fn load(self, h: &mut impl Loader) -> Result<()> {
let now = chrono::Utc::now();
let mut line = String::new();
fn next(&mut self) -> Option<Self::Item> {
self.strbuf.clear();
match self.file.read_line(&mut self.strbuf) {
Ok(0) => return None,
Ok(_) => (),
Err(e) => return Some(Err(eyre!("failed to read line: {}", e))), // we can skip past things like invalid utf8
}
self.loc -= 1;
while self.strbuf.ends_with("\\\n") {
if self.file.read_line(&mut self.strbuf).is_err() {
// There's a chance that the last line of a command has invalid
// characters, the only safe thing to do is break :/
// usually just invalid utf8 or smth
// however, we really need to avoid missing history, so it's
// better to have some items that should have been part of
// something else, than to miss things. So break.
break;
for (i, b) in unix_byte_lines(&self.bytes).enumerate() {
let s = match std::str::from_utf8(b) {
Ok(s) => s,
Err(_) => continue, // we can skip past things like invalid utf8
};
self.loc -= 1;
if let Some(s) = s.strip_suffix('\\') {
line.push_str(s);
line.push_str("\\\n");
} else {
line.push_str(s);
let command = std::mem::take(&mut line);
let offset = chrono::Duration::seconds(i as i64);
h.push(History::new(
now - offset, // preserve ordering
command,
String::from("unknown"),
-1,
-1,
None,
None,
))
.await?;
}
}
let time = chrono::Utc::now();
let offset = chrono::Duration::seconds(self.counter);
let time = time - offset;
self.counter += 1;
Some(Ok(History::new(
time,
self.strbuf.trim_end().to_string(),
String::from("unknown"),
-1,
-1,
None,
None,
)))
}
fn size_hint(&self) -> (usize, Option<usize>) {
(0, Some(self.loc))
Ok(())
}
}
#[cfg(test)]
mod tests {
use std::io::Cursor;
use itertools::assert_equal;
use crate::import::{tests::TestLoader, Importer};
use super::Bash;
#[test]
fn test_parse_file() {
let input = r"cargo install atuin
#[tokio::test]
async fn test_parse_file() {
let bytes = r"cargo install atuin
cargo install atuin; \
cargo update
cargo :b̷i̶t̴r̵o̴t̴ ̵i̷s̴ ̷r̶e̵a̸l̷
";
"
.as_bytes()
.to_owned();
let cursor = Cursor::new(input);
let mut bash = Bash::new(cursor).unwrap();
assert_eq!(bash.loc, 4);
assert_eq!(bash.size_hint(), (0, Some(4)));
let mut bash = Bash { bytes };
assert_eq!(bash.entries().await.unwrap(), 4);
assert_eq!(
&bash.next().unwrap().unwrap().command,
"cargo install atuin"
);
assert_eq!(
&bash.next().unwrap().unwrap().command,
"cargo install atuin; \\\ncargo update"
);
assert_eq!(
&bash.next().unwrap().unwrap().command,
"cargo :b̷i̶t̴r̵o̴t̴ ̵i̷s̴ ̷r̶e̵a̸l̷"
);
assert!(bash.next().is_none());
let mut loader = TestLoader::default();
bash.load(&mut loader).await.unwrap();
assert_eq!(bash.size_hint(), (0, Some(0)));
assert_equal(
loader.buf.iter().map(|h| h.command.as_str()),
[
"cargo install atuin",
"cargo install atuin; \\\ncargo update",
"cargo :b̷i̶t̴r̵o̴t̴ ̵i̷s̴ ̷r̶e̵a̸l̷",
],
);
}
}

192
atuin-client/src/import/fish.rs
View file

@ -1,99 +1,90 @@
// import old shell history!
// automatically hoover up all that we can find
use std::{
fs::File,
io::{self, BufRead, BufReader, Read, Seek},
path::{Path, PathBuf},
};
use std::{fs::File, io::Read, path::PathBuf};
use async_trait::async_trait;
use chrono::{prelude::*, Utc};
use directories::BaseDirs;
use eyre::{eyre, Result};
use super::{count_lines, Importer};
use super::{get_histpath, unix_byte_lines, Importer, Loader};
use crate::history::History;
#[derive(Debug)]
pub struct Fish<R> {
file: BufReader<R>,
strbuf: String,
loc: usize,
pub struct Fish {
bytes: Vec<u8>,
}
impl<R: Read + Seek> Fish<R> {
fn new(r: R) -> Result<Self> {
let mut buf = BufReader::new(r);
let loc = count_lines(&mut buf)?;
/// see https://fishshell.com/docs/current/interactive.html#searchable-command-history
fn default_histpath() -> Result<PathBuf> {
let base = BaseDirs::new().ok_or_else(|| eyre!("could not determine data directory"))?;
let data = base.data_local_dir();
Ok(Self {
file: buf,
strbuf: String::new(),
loc,
})
// fish supports multiple history sessions
// If `fish_history` var is missing, or set to `default`, use `fish` as the session
let session = std::env::var("fish_history").unwrap_or_else(|_| String::from("fish"));
let session = if session == "default" {
String::from("fish")
} else {
session
};
let mut histpath = data.join("fish");
histpath.push(format!("{}_history", session));
if histpath.exists() {
Ok(histpath)
} else {
Err(eyre!("Could not find history file. Try setting $HISTFILE"))
}
}
impl<R: Read> Fish<R> {
fn new_entry(&mut self) -> io::Result<bool> {
let inner = self.file.fill_buf()?;
Ok(inner.starts_with(b"- "))
}
}
impl Importer for Fish<File> {
#[async_trait]
impl Importer for Fish {
const NAME: &'static str = "fish";
/// see https://fishshell.com/docs/current/interactive.html#searchable-command-history
fn histpath() -> Result<PathBuf> {
let base = BaseDirs::new().ok_or_else(|| eyre!("could not determine data directory"))?;
let data = base.data_local_dir();
// fish supports multiple history sessions
// If `fish_history` var is missing, or set to `default`, use `fish` as the session
let session = std::env::var("fish_history").unwrap_or_else(|_| String::from("fish"));
let session = if session == "default" {
String::from("fish")
} else {
session
};
let mut histpath = data.join("fish");
histpath.push(format!("{}_history", session));
if histpath.exists() {
Ok(histpath)
} else {
Err(eyre!("Could not find history file. Try setting $HISTFILE"))
}
async fn new() -> Result<Self> {
let mut bytes = Vec::new();
let path = get_histpath(default_histpath)?;
let mut f = File::open(path)?;
f.read_to_end(&mut bytes)?;
Ok(Self { bytes })
}
fn parse(path: impl AsRef<Path>) -> Result<Self> {
Self::new(File::open(path)?)
async fn entries(&mut self) -> Result<usize> {
Ok(super::count_lines(&self.bytes))
}
}
impl<R: Read> Iterator for Fish<R> {
type Item = Result<History>;
fn next(&mut self) -> Option<Self::Item> {
async fn load(self, loader: &mut impl Loader) -> Result<()> {
let now = Utc::now();
let mut time: Option<DateTime<Utc>> = None;
let mut cmd: Option<String> = None;
loop {
self.strbuf.clear();
match self.file.read_line(&mut self.strbuf) {
// no more content to read
Ok(0) => break,
// bail on IO error
Err(e) => return Some(Err(e.into())),
_ => (),
}
for b in unix_byte_lines(&self.bytes) {
let s = match std::str::from_utf8(b) {
Ok(s) => s,
Err(_) => continue, // we can skip past things like invalid utf8
};
// `read_line` adds the line delimeter to the string. No thanks
self.strbuf.pop();
if let Some(c) = s.strip_prefix("- cmd: ") {
// first, we must deal with the prev cmd
if let Some(cmd) = cmd.take() {
let time = time.unwrap_or(now);
loader
.push(History::new(
time,
cmd,
"unknown".into(),
-1,
-1,
None,
None,
))
.await?;
}
if let Some(c) = self.strbuf.strip_prefix("- cmd: ") {
// using raw strings to avoid needing escaping.
// replaces double backslashes with single backslashes
let c = c.replace(r"\\", r"\");
@ -102,7 +93,7 @@ impl<R: Read> Iterator for Fish<R> {
// TODO: any other escape characters?
cmd = Some(c);
} else if let Some(t) = self.strbuf.strip_prefix(" when: ") {
} else if let Some(t) = s.strip_prefix(" when: ") {
// if t is not an int, just ignore this line
if let Ok(t) = t.parse::<i64>() {
time = Some(Utc.timestamp(t, 0));
@ -110,47 +101,40 @@ impl<R: Read> Iterator for Fish<R> {
} else {
// ... ignore paths lines
}
match self.new_entry() {
// next line is a new entry, so let's stop here
// only if we have found a command though
Ok(true) if cmd.is_some() => break,
// bail on IO error
Err(e) => return Some(Err(e.into())),
_ => (),
}
}
let cmd = cmd?;
let time = time.unwrap_or_else(Utc::now);
// we might have a trailing cmd
if let Some(cmd) = cmd.take() {
let time = time.unwrap_or(now);
Some(Ok(History::new(
time,
cmd,
"unknown".into(),
-1,
-1,
None,
None,
)))
}
loader
.push(History::new(
time,
cmd,
"unknown".into(),
-1,
-1,
None,
None,
))
.await?;
}
fn size_hint(&self) -> (usize, Option<usize>) {
// worst case, entry per line
(0, Some(self.loc))
Ok(())
}
}
#[cfg(test)]
mod test {
use std::io::Cursor;
use crate::import::{tests::TestLoader, Importer};
use super::Fish;
#[test]
fn parse_complex() {
#[tokio::test]
async fn parse_complex() {
// complicated input with varying contents and escaped strings.
let input = r#"- cmd: history --help
let bytes = r#"- cmd: history --help
when: 1639162832
- cmd: cat ~/.bash_history
when: 1639162851
@ -181,14 +165,20 @@ ERROR
when: 1639163066
paths:
- ~/.local/share/fish/fish_history
"#;
let cursor = Cursor::new(input);
let mut fish = Fish::new(cursor).unwrap();
"#
.as_bytes()
.to_owned();
let fish = Fish { bytes };
let mut loader = TestLoader::default();
fish.load(&mut loader).await.unwrap();
let mut history = loader.buf.into_iter();
// simple wrapper for fish history entry
macro_rules! fishtory {
($timestamp:expr, $command:expr) => {
let h = fish.next().expect("missing entry in history").unwrap();
let h = history.next().expect("missing entry in history");
assert_eq!(h.command.as_str(), $command);
assert_eq!(h.timestamp.timestamp(), $timestamp);
};

103
atuin-client/src/import/mod.rs
View file

@ -1,9 +1,8 @@
use std::{
io::{BufRead, BufReader, Read, Seek, SeekFrom},
path::{Path, PathBuf},
};
use std::path::PathBuf;
use eyre::Result;
use async_trait::async_trait;
use eyre::{bail, Result};
use memchr::Memchr;
use crate::history::History;
@ -12,16 +11,88 @@ pub mod fish;
pub mod resh;
pub mod zsh;
// this could probably be sped up
fn count_lines(buf: &mut BufReader<impl Read + Seek>) -> Result<usize> {
let lines = buf.lines().count();
buf.seek(SeekFrom::Start(0))?;
Ok(lines)
}
pub trait Importer: IntoIterator<Item = Result<History>> + Sized {
#[async_trait]
pub trait Importer: Sized {
const NAME: &'static str;
fn histpath() -> Result<PathBuf>;
fn parse(path: impl AsRef<Path>) -> Result<Self>;
async fn new() -> Result<Self>;
async fn entries(&mut self) -> Result<usize>;
async fn load(self, loader: &mut impl Loader) -> Result<()>;
}
#[async_trait]
pub trait Loader: Sync + Send {
async fn push(&mut self, hist: History) -> eyre::Result<()>;
}
fn unix_byte_lines(input: &[u8]) -> impl Iterator<Item = &[u8]> {
UnixByteLines {
iter: memchr::memchr_iter(b'\n', input),
bytes: input,
i: 0,
}
}
struct UnixByteLines<'a> {
iter: Memchr<'a>,
bytes: &'a [u8],
i: usize,
}
impl<'a> Iterator for UnixByteLines<'a> {
type Item = &'a [u8];
fn next(&mut self) -> Option<Self::Item> {
let j = self.iter.next()?;
let out = &self.bytes[self.i..j];
self.i = j + 1;
Some(out)
}
fn count(self) -> usize
where
Self: Sized,
{
self.iter.count()
}
}
fn count_lines(input: &[u8]) -> usize {
unix_byte_lines(input).count()
}
fn get_histpath<D>(def: D) -> Result<PathBuf>
where
D: FnOnce() -> Result<PathBuf>,
{
if let Ok(p) = std::env::var("HISTFILE") {
is_file(PathBuf::from(p))
} else {
is_file(def()?)
}
}
fn is_file(p: PathBuf) -> Result<PathBuf> {
if p.is_file() {
Ok(p)
} else {
bail!("Could not find history file {:?}. Try setting $HISTFILE", p)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[derive(Default)]
pub struct TestLoader {
pub buf: Vec<History>,
}
#[async_trait]
impl Loader for TestLoader {
async fn push(&mut self, hist: History) -> Result<()> {
self.buf.push(hist);
Ok(())
}
}
}

135
atuin-client/src/import/resh.rs
View file

@ -1,9 +1,6 @@
use std::{
fs::File,
io::{BufRead, BufReader},
path::{Path, PathBuf},
};
use std::{fs::File, io::Read, path::PathBuf};
use async_trait::async_trait;
use chrono::{TimeZone, Utc};
use directories::UserDirs;
use eyre::{eyre, Result};
@ -11,7 +8,7 @@ use serde::Deserialize;
use atuin_common::utils::uuid_v4;
use super::{count_lines, Importer};
use super::{get_histpath, unix_byte_lines, Importer, Loader};
use crate::history::History;
#[derive(Deserialize, Debug)]
@ -72,88 +69,72 @@ pub struct ReshEntry {
#[derive(Debug)]
pub struct Resh {
file: BufReader<File>,
strbuf: String,
loc: usize,
bytes: Vec<u8>,
}
fn default_histpath() -> Result<PathBuf> {
let user_dirs = UserDirs::new().ok_or_else(|| eyre!("could not find user directories"))?;
let home_dir = user_dirs.home_dir();
Ok(home_dir.join(".resh_history.json"))
}
#[async_trait]
impl Importer for Resh {
const NAME: &'static str = "resh";
fn histpath() -> Result<PathBuf> {
let user_dirs = UserDirs::new().unwrap();
let home_dir = user_dirs.home_dir();
Ok(home_dir.join(".resh_history.json"))
async fn new() -> Result<Self> {
let mut bytes = Vec::new();
let path = get_histpath(default_histpath)?;
let mut f = File::open(path)?;
f.read_to_end(&mut bytes)?;
Ok(Self { bytes })
}
fn parse(path: impl AsRef<Path>) -> Result<Self> {
let file = File::open(path)?;
let mut buf = BufReader::new(file);
let loc = count_lines(&mut buf)?;
Ok(Self {
file: buf,
strbuf: String::new(),
loc,
})
async fn entries(&mut self) -> Result<usize> {
Ok(super::count_lines(&self.bytes))
}
}
impl Iterator for Resh {
type Item = Result<History>;
async fn load(self, h: &mut impl Loader) -> Result<()> {
for b in unix_byte_lines(&self.bytes) {
let s = match std::str::from_utf8(b) {
Ok(s) => s,
Err(_) => continue, // we can skip past things like invalid utf8
};
let entry = match serde_json::from_str::<ReshEntry>(s) {
Ok(e) => e,
Err(_) => continue, // skip invalid json :shrug:
};
fn next(&mut self) -> Option<Self::Item> {
self.strbuf.clear();
match self.file.read_line(&mut self.strbuf) {
Ok(0) => return None,
Ok(_) => (),
Err(e) => return Some(Err(eyre!("failed to read line: {}", e))), // we can skip past things like invalid utf8
#[allow(clippy::cast_possible_truncation)]
#[allow(clippy::cast_sign_loss)]
let timestamp = {
let secs = entry.realtime_before.floor() as i64;
let nanosecs = (entry.realtime_before.fract() * 1_000_000_000_f64).round() as u32;
Utc.timestamp(secs, nanosecs)
};
#[allow(clippy::cast_possible_truncation)]
#[allow(clippy::cast_sign_loss)]
let duration = {
let secs = entry.realtime_after.floor() as i64;
let nanosecs = (entry.realtime_after.fract() * 1_000_000_000_f64).round() as u32;
let difference = Utc.timestamp(secs, nanosecs) - timestamp;
difference.num_nanoseconds().unwrap_or(0)
};
h.push(History {
id: uuid_v4(),
timestamp,
duration,
exit: entry.exit_code,
command: entry.cmd_line,
cwd: entry.pwd,
session: uuid_v4(),
hostname: entry.host,
})
.await?;
}
// .resh_history.json lies about being a json. It is actually a file containing valid json
// on every line. This means that the last line will throw an error, as it is just an EOF.
// Without the special case here, that will crash the importer.
let entry = match serde_json::from_str::<ReshEntry>(&self.strbuf) {
Ok(e) => e,
Err(e) if e.is_eof() => return None,
Err(e) => {
return Some(Err(eyre!(
"Invalid entry found in resh_history file: {}",
e
)))
}
};
#[allow(clippy::cast_possible_truncation)]
#[allow(clippy::cast_sign_loss)]
let timestamp = {
let secs = entry.realtime_before.floor() as i64;
let nanosecs = (entry.realtime_before.fract() * 1_000_000_000_f64).round() as u32;
Utc.timestamp(secs, nanosecs)
};
#[allow(clippy::cast_possible_truncation)]
#[allow(clippy::cast_sign_loss)]
let duration = {
let secs = entry.realtime_after.floor() as i64;
let nanosecs = (entry.realtime_after.fract() * 1_000_000_000_f64).round() as u32;
let difference = Utc.timestamp(secs, nanosecs) - timestamp;
difference.num_nanoseconds().unwrap_or(0)
};
Some(Ok(History {
id: uuid_v4(),
timestamp,
duration,
exit: entry.exit_code,
command: entry.cmd_line,
cwd: entry.pwd,
session: uuid_v4(),
hostname: entry.host,
}))
}
fn size_hint(&self) -> (usize, Option<usize>) {
(self.loc, Some(self.loc))
Ok(())
}
}

224
atuin-client/src/import/zsh.rs
View file

@ -1,138 +1,104 @@
// import old shell history!
// automatically hoover up all that we can find
use std::{
fs::File,
io::{BufRead, BufReader, Read, Seek},
path::{Path, PathBuf},
};
use std::{fs::File, io::Read, path::PathBuf};
use async_trait::async_trait;
use chrono::{prelude::*, Utc};
use directories::UserDirs;
use eyre::{eyre, Result};
use itertools::Itertools;
use super::{count_lines, Importer};
use super::{get_histpath, unix_byte_lines, Importer, Loader};
use crate::history::History;
#[derive(Debug)]
pub struct Zsh<R> {
file: BufReader<R>,
strbuf: String,
loc: usize,
counter: i64,
pub struct Zsh {
bytes: Vec<u8>,
}
impl<R: Read + Seek> Zsh<R> {
fn new(r: R) -> Result<Self> {
let mut buf = BufReader::new(r);
let loc = count_lines(&mut buf)?;
fn default_histpath() -> Result<PathBuf> {
// oh-my-zsh sets HISTFILE=~/.zhistory
// zsh has no default value for this var, but uses ~/.zhistory.
// we could maybe be smarter about this in the future :)
let user_dirs = UserDirs::new().ok_or_else(|| eyre!("could not find user directories"))?;
let home_dir = user_dirs.home_dir();
Ok(Self {
file: buf,
strbuf: String::new(),
loc,
counter: 0,
})
}
}
impl Importer for Zsh<File> {
const NAME: &'static str = "zsh";
fn histpath() -> Result<PathBuf> {
// oh-my-zsh sets HISTFILE=~/.zhistory
// zsh has no default value for this var, but uses ~/.zhistory.
// we could maybe be smarter about this in the future :)
let user_dirs = UserDirs::new().unwrap();
let home_dir = user_dirs.home_dir();
let mut candidates = [".zhistory", ".zsh_history"].iter();
loop {
match candidates.next() {
Some(candidate) => {
let histpath = home_dir.join(candidate);
if histpath.exists() {
break Ok(histpath);
}
let mut candidates = [".zhistory", ".zsh_history"].iter();
loop {
match candidates.next() {
Some(candidate) => {
let histpath = home_dir.join(candidate);
if histpath.exists() {
break Ok(histpath);
}
None => break Err(eyre!("Could not find history file. Try setting $HISTFILE")),
}
None => break Err(eyre!("Could not find history file. Try setting $HISTFILE")),
}
}
fn parse(path: impl AsRef<Path>) -> Result<Self> {
Self::new(File::open(path)?)
}
}
impl<R: Read> Iterator for Zsh<R> {
type Item = Result<History>;
#[async_trait]
impl Importer for Zsh {
const NAME: &'static str = "bash";
fn next(&mut self) -> Option<Self::Item> {
// ZSH extended history records the timestamp + command duration
// These lines begin with :
// So, if the line begins with :, parse it. Otherwise it's just
// the command
self.strbuf.clear();
match self.file.read_line(&mut self.strbuf) {
Ok(0) => return None,
Ok(_) => (),
Err(e) => return Some(Err(eyre!("failed to read line: {}", e))), // we can skip past things like invalid utf8
}
async fn new() -> Result<Self> {
let mut bytes = Vec::new();
let path = get_histpath(default_histpath)?;
let mut f = File::open(path)?;
f.read_to_end(&mut bytes)?;
Ok(Self { bytes })
}
self.loc -= 1;
async fn entries(&mut self) -> Result<usize> {
Ok(super::count_lines(&self.bytes))
}
while self.strbuf.ends_with("\\\n") {
if self.file.read_line(&mut self.strbuf).is_err() {
// There's a chance that the last line of a command has invalid
// characters, the only safe thing to do is break :/
// usually just invalid utf8 or smth
// however, we really need to avoid missing history, so it's
// better to have some items that should have been part of
// something else, than to miss things. So break.
break;
async fn load(self, h: &mut impl Loader) -> Result<()> {
let now = chrono::Utc::now();
let mut line = String::new();
let mut counter = 0;
for b in unix_byte_lines(&self.bytes) {
let s = match std::str::from_utf8(b) {
Ok(s) => s,
Err(_) => continue, // we can skip past things like invalid utf8
};
self.loc -= 1;
if let Some(s) = s.strip_suffix('\\') {
line.push_str(s);
line.push_str("\\\n");
} else {
line.push_str(s);
let command = std::mem::take(&mut line);
if let Some(command) = command.strip_prefix(": ") {
counter += 1;
h.push(parse_extended(command, counter)).await?;
} else {
let offset = chrono::Duration::seconds(counter);
counter += 1;
h.push(History::new(
now - offset, // preserve ordering
command.trim_end().to_string(),
String::from("unknown"),
-1,
-1,
None,
None,
))
.await?;
}
}
}
// We have to handle the case where a line has escaped newlines.
// Keep reading until we have a non-escaped newline
let extended = self.strbuf.starts_with(':');
if extended {
self.counter += 1;
Some(Ok(parse_extended(&self.strbuf, self.counter)))
} else {
let time = chrono::Utc::now();
let offset = chrono::Duration::seconds(self.counter);
let time = time - offset;
self.counter += 1;
Some(Ok(History::new(
time,
self.strbuf.trim_end().to_string(),
String::from("unknown"),
-1,
-1,
None,
None,
)))
}
}
fn size_hint(&self) -> (usize, Option<usize>) {
(0, Some(self.loc))
Ok(())
}
}
fn parse_extended(line: &str, counter: i64) -> History {
let line = line.replacen(": ", "", 2);
let (time, duration) = line.splitn(2, ':').collect_tuple().unwrap();
let (duration, command) = duration.splitn(2, ';').collect_tuple().unwrap();
let (time, duration) = line.split_once(':').unwrap();
let (duration, command) = duration.split_once(';').unwrap();
let time = time
.parse::<i64>()
@ -158,64 +124,64 @@ fn parse_extended(line: &str, counter: i64) -> History {
#[cfg(test)]
mod test {
use std::io::Cursor;
use chrono::prelude::*;
use chrono::Utc;
use itertools::assert_equal;
use crate::import::tests::TestLoader;
use super::*;
#[test]
fn test_parse_extended_simple() {
let parsed = parse_extended(": 1613322469:0;cargo install atuin", 0);
let parsed = parse_extended("1613322469:0;cargo install atuin", 0);
assert_eq!(parsed.command, "cargo install atuin");
assert_eq!(parsed.duration, 0);
assert_eq!(parsed.timestamp, Utc.timestamp(1_613_322_469, 0));
let parsed = parse_extended(": 1613322469:10;cargo install atuin;cargo update", 0);
let parsed = parse_extended("1613322469:10;cargo install atuin;cargo update", 0);
assert_eq!(parsed.command, "cargo install atuin;cargo update");
assert_eq!(parsed.duration, 10_000_000_000);
assert_eq!(parsed.timestamp, Utc.timestamp(1_613_322_469, 0));
let parsed = parse_extended(": 1613322469:10;cargo :b̷i̶t̴r̵o̴t̴ ̵i̷s̴ ̷r̶e̵a̸l̷", 0);
let parsed = parse_extended("1613322469:10;cargo :b̷i̶t̴r̵o̴t̴ ̵i̷s̴ ̷r̶e̵a̸l̷", 0);
assert_eq!(parsed.command, "cargo :b̷i̶t̴r̵o̴t̴ ̵i̷s̴ ̷r̶e̵a̸l̷");
assert_eq!(parsed.duration, 10_000_000_000);
assert_eq!(parsed.timestamp, Utc.timestamp(1_613_322_469, 0));
let parsed = parse_extended(": 1613322469:10;cargo install \\n atuin\n", 0);
let parsed = parse_extended("1613322469:10;cargo install \\n atuin\n", 0);
assert_eq!(parsed.command, "cargo install \\n atuin");
assert_eq!(parsed.duration, 10_000_000_000);
assert_eq!(parsed.timestamp, Utc.timestamp(1_613_322_469, 0));
}
#[test]
fn test_parse_file() {
let input = r": 1613322469:0;cargo install atuin
#[tokio::test]
async fn test_parse_file() {
let bytes = r": 1613322469:0;cargo install atuin
: 1613322469:10;cargo install atuin; \
cargo update
: 1613322469:10;cargo :b̷i̶t̴r̵o̴t̴ ̵i̷s̴ ̷r̶e̵a̸l̷
";
"
.as_bytes()
.to_owned();
let cursor = Cursor::new(input);
let mut zsh = Zsh::new(cursor).unwrap();
assert_eq!(zsh.loc, 4);
assert_eq!(zsh.size_hint(), (0, Some(4)));
let mut zsh = Zsh { bytes };
assert_eq!(zsh.entries().await.unwrap(), 4);
assert_eq!(&zsh.next().unwrap().unwrap().command, "cargo install atuin");
assert_eq!(
&zsh.next().unwrap().unwrap().command,
"cargo install atuin; \\\ncargo update"
let mut loader = TestLoader::default();
zsh.load(&mut loader).await.unwrap();
assert_equal(
loader.buf.iter().map(|h| h.command.as_str()),
[
"cargo install atuin",
"cargo install atuin; \\\ncargo update",
"cargo :b̷i̶t̴r̵o̴t̴ ̵i̷s̴ ̷r̶e̵a̸l̷",
],
);
assert_eq!(
&zsh.next().unwrap().unwrap().command,
"cargo :b̷i̶t̴r̵o̴t̴ ̵i̷s̴ ̷r̶e̵a̸l̷"
);
assert!(zsh.next().is_none());
assert_eq!(zsh.size_hint(), (0, Some(0)));
}
}

1
src/command/client.rs
View file

@ -58,6 +58,7 @@ pub enum Cmd {
}
impl Cmd {
#[tokio::main(flavor = "current_thread")]
pub async fn run(self) -> Result<()> {
pretty_env_logger::init();

6
src/command/client/history.rs
View file

@ -128,11 +128,7 @@ pub fn print_cmd_only(w: &mut StdoutLock, h: &[History]) {
}
impl Cmd {
pub async fn run(
&self,
settings: &Settings,
db: &mut (impl Database + Send + Sync),
) -> Result<()> {
pub async fn run(&self, settings: &Settings, db: &mut impl Database) -> Result<()> {
let context = current_context();
match self {

163
src/command/client/import.rs
View file

@ -1,13 +1,14 @@
use std::{env, path::PathBuf};
use std::env;
use async_trait::async_trait;
use clap::Parser;
use eyre::{eyre, Result};
use eyre::Result;
use indicatif::ProgressBar;
use atuin_client::{
database::Database,
history::History,
import::{bash::Bash, fish::Fish, resh::Resh, zsh::Zsh, Importer},
import::{bash::Bash, fish::Fish, resh::Resh, zsh::Zsh, Importer, Loader},
};
#[derive(Parser)]
@ -18,13 +19,10 @@ pub enum Cmd {
/// Import history from the zsh history file
Zsh,
/// Import history from the bash history file
Bash,
/// Import history from the resh history file
Resh,
/// Import history from the fish history file
Fish,
}
@ -32,7 +30,7 @@ pub enum Cmd {
const BATCH_SIZE: usize = 100;
impl Cmd {
pub async fn run(&self, db: &mut (impl Database + Send + Sync)) -> Result<()> {
pub async fn run<DB: Database>(&self, db: &mut DB) -> Result<()> {
println!(" Atuin ");
println!("======================");
println!(" \u{1f30d} ");
@ -47,124 +45,73 @@ impl Cmd {
if shell.ends_with("/zsh") {
println!("Detected ZSH");
import::<Zsh<_>, _>(db, BATCH_SIZE).await
import::<Zsh, DB>(db).await
} else if shell.ends_with("/fish") {
println!("Detected Fish");
import::<Fish<_>, _>(db, BATCH_SIZE).await
import::<Fish, DB>(db).await
} else if shell.ends_with("/bash") {
println!("Detected Bash");
import::<Bash<_>, _>(db, BATCH_SIZE).await
import::<Bash, DB>(db).await
} else {
println!("cannot import {} history", shell);
Ok(())
}
}
Self::Zsh => import::<Zsh<_>, _>(db, BATCH_SIZE).await,
Self::Bash => import::<Bash<_>, _>(db, BATCH_SIZE).await,
Self::Resh => import::<Resh, _>(db, BATCH_SIZE).await,
Self::Fish => import::<Fish<_>, _>(db, BATCH_SIZE).await,
Self::Zsh => import::<Zsh, DB>(db).await,
Self::Bash => import::<Bash, DB>(db).await,
Self::Resh => import::<Resh, DB>(db).await,
Self::Fish => import::<Fish, DB>(db).await,
}
}
}
async fn import<I: Importer + Send, DB: Database + Send + Sync>(
db: &mut DB,
buf_size: usize,
) -> Result<()>
where
I::IntoIter: Send,
{
pub struct HistoryImporter<'db, DB: Database> {
pb: ProgressBar,
buf: Vec<History>,
db: &'db mut DB,
}
impl<'db, DB: Database> HistoryImporter<'db, DB> {
fn new(db: &'db mut DB, len: usize) -> Self {
Self {
pb: ProgressBar::new(len as u64),
buf: Vec::with_capacity(BATCH_SIZE),
db,
}
}
async fn flush(self) -> Result<()> {
if !self.buf.is_empty() {
self.db.save_bulk(&self.buf).await?;
}
self.pb.finish();
Ok(())
}
}
#[async_trait]
impl<'db, DB: Database> Loader for HistoryImporter<'db, DB> {
async fn push(&mut self, hist: History) -> Result<()> {
self.pb.inc(1);
self.buf.push(hist);
if self.buf.len() == self.buf.capacity() {
self.db.save_bulk(&self.buf).await?;
self.buf.clear();
}
Ok(())
}
}
async fn import<I: Importer + Send, DB: Database>(db: &mut DB) -> Result<()> {
println!("Importing history from {}", I::NAME);
let histpath = get_histpath::<I>()?;
let contents = I::parse(histpath)?;
let iter = contents.into_iter();
let progress = if let (_, Some(upper_bound)) = iter.size_hint() {
ProgressBar::new(upper_bound as u64)
} else {
ProgressBar::new_spinner()
};
let mut buf = Vec::<History>::with_capacity(buf_size);
let mut iter = progress.wrap_iter(iter);
loop {
// fill until either no more entries
// or until the buffer is full
let done = fill_buf(&mut buf, &mut iter);
// flush
db.save_bulk(&buf).await?;
if done {
break;
}
}
let mut importer = I::new().await?;
let len = importer.entries().await.unwrap();
let mut loader = HistoryImporter::new(db, len);
importer.load(&mut loader).await?;
loader.flush().await?;
println!("Import complete!");
Ok(())
}
fn get_histpath<I: Importer>() -> Result<PathBuf> {
if let Ok(p) = env::var("HISTFILE") {
is_file(PathBuf::from(p))
} else {
is_file(I::histpath()?)
}
}
fn is_file(p: PathBuf) -> Result<PathBuf> {
if p.is_file() {
Ok(p)
} else {
Err(eyre!(
"Could not find history file {:?}. Try setting $HISTFILE",
p
))
}
}
fn fill_buf<T, E>(buf: &mut Vec<T>, iter: &mut impl Iterator<Item = Result<T, E>>) -> bool {
buf.clear();
loop {
match iter.next() {
Some(Ok(t)) => buf.push(t),
Some(Err(_)) => (),
None => break true,
}
if buf.len() == buf.capacity() {
break false;
}
}
}
#[cfg(test)]
mod tests {
use super::fill_buf;
#[test]
fn test_fill_buf() {
let mut buf = Vec::with_capacity(4);
let mut iter = vec![
Ok(1),
Err(2),
Ok(3),
Ok(4),
Err(5),
Ok(6),
Ok(7),
Err(8),
Ok(9),
]
.into_iter();
assert!(!fill_buf(&mut buf, &mut iter));
assert_eq!(buf, vec![1, 3, 4, 6]);
assert!(fill_buf(&mut buf, &mut iter));
assert_eq!(buf, vec![7, 9]);
}
}

14
src/command/client/search.rs
View file

@ -75,11 +75,7 @@ pub struct Cmd {
}
impl Cmd {
pub async fn run(
self,
db: &mut (impl Database + Send + Sync),
settings: &Settings,
) -> Result<()> {
pub async fn run(self, db: &mut impl Database, settings: &Settings) -> Result<()> {
if self.interactive {
let item = select_history(
&self.query,
@ -257,7 +253,7 @@ impl State {
async fn query_results(
app: &mut State,
search_mode: SearchMode,
db: &mut (impl Database + Send + Sync),
db: &mut impl Database,
) -> Result<()> {
let results = match app.input.as_str() {
"" => {
@ -284,7 +280,7 @@ async fn query_results(
async fn key_handler(
input: Key,
search_mode: SearchMode,
db: &mut (impl Database + Send + Sync),
db: &mut impl Database,
app: &mut State,
) -> Option<String> {
match input {
@ -537,7 +533,7 @@ async fn select_history(
search_mode: SearchMode,
filter_mode: FilterMode,
style: atuin_client::settings::Style,
db: &mut (impl Database + Send + Sync),
db: &mut impl Database,
) -> Result<String> {
let stdout = stdout().into_raw_mode()?;
let stdout = MouseTerminal::from(stdout);
@ -596,7 +592,7 @@ async fn run_non_interactive(
after: Option<String>,
limit: Option<i64>,
query: &[String],
db: &mut (impl Database + Send + Sync),
db: &mut impl Database,
) -> Result<()> {
let dir = if cwd.as_deref() == Some(".") {
let current = std::env::current_dir()?;

6
src/command/client/stats.rs
View file

@ -62,11 +62,7 @@ fn compute_stats(history: &[History]) -> Result<()> {
}
impl Cmd {
pub async fn run(
&self,
db: &mut (impl Database + Send + Sync),
settings: &Settings,
) -> Result<()> {
pub async fn run(&self, db: &mut impl Database, settings: &Settings) -> Result<()> {
let context = current_context();
let words = if self.period.is_empty() {
String::from("all")

12
src/command/client/sync.rs
View file

@ -31,11 +31,7 @@ pub enum Cmd {
}
impl Cmd {
pub async fn run(
self,
settings: Settings,
db: &mut (impl Database + Send + Sync),
) -> Result<()> {
pub async fn run(self, settings: Settings, db: &mut impl Database) -> Result<()> {
match self {
Self::Sync { force } => run(&settings, force, db).await,
Self::Login(l) => l.run(&settings).await,
@ -52,11 +48,7 @@ impl Cmd {
}
}
async fn run(
settings: &Settings,
force: bool,
db: &mut (impl Database + Send + Sync),
) -> Result<()> {
async fn run(settings: &Settings, force: bool, db: &mut impl Database) -> Result<()> {
atuin_client::sync::sync(settings, force, db).await?;
println!(
"Sync complete! {} items in database, force: {}",

6
src/command/mod.rs
View file

@ -19,11 +19,11 @@ pub enum AtuinCmd {
}
impl AtuinCmd {
pub async fn run(self) -> Result<()> {
pub fn run(self) -> Result<()> {
match self {
Self::Client(client) => client.run().await,
Self::Client(client) => client.run(),
#[cfg(feature = "server")]
Self::Server(server) => server.run().await,
Self::Server(server) => server.run(),
}
}
}

1
src/command/server.rs
View file

@ -21,6 +21,7 @@ pub enum Cmd {
}
impl Cmd {
#[tokio::main]
pub async fn run(self) -> Result<()> {
tracing_subscriber::registry()
.with(fmt::layer())

9
src/main.rs
View file

@ -25,12 +25,11 @@ struct Atuin {
}
impl Atuin {
async fn run(self) -> Result<()> {
self.atuin.run().await
fn run(self) -> Result<()> {
self.atuin.run()
}
}
#[tokio::main]
async fn main() -> Result<()> {
Atuin::parse().run().await
fn main() -> Result<()> {
Atuin::parse().run()
}