Automatically filter out secrets (#1182)
I'd like to extend the regex list here very soon, but start off by automatically filtering out secrets. Do not store them in history! I've included regex for: 1. AWS key id 2. Github pat (old and new) 3. Slack oauth tokens (bot, user) 4. Slack webhooks 5. Stripe live/test keys Will need updating after #806
This commit is contained in:
Ellie Huxtable
GitHub
parent
aa8e5f5c04
commit
73bd8015c3
6 changed files with 201 additions and 28 deletions
|
|
@ -3,6 +3,9 @@ use std::env;
|
|||
use chrono::Utc;
|
||||
|
||||
use atuin_common::utils::uuid_v7;
|
||||
use regex::RegexSet;
|
||||
|
||||
use crate::{secrets::SECRET_PATTERNS, settings::Settings};
|
||||
|
||||
mod builder;
|
||||
|
||||
|
|
@ -185,4 +188,87 @@ impl History {
|
|||
pub fn success(&self) -> bool {
|
||||
self.exit == 0 || self.duration == -1
|
||||
}
|
||||
|
||||
pub fn should_save(&self, settings: &Settings) -> bool {
|
||||
let secret_regex = SECRET_PATTERNS.iter().map(|f| f.1);
|
||||
let secret_regex = RegexSet::new(secret_regex).expect("Failed to build secrets regex");
|
||||
|
||||
!(self.command.starts_with(' ')
|
||||
|| settings.history_filter.is_match(&self.command)
|
||||
|| settings.cwd_filter.is_match(&self.cwd)
|
||||
|| (secret_regex.is_match(&self.command)) && settings.secrets_filter)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use regex::RegexSet;
|
||||
|
||||
use crate::settings::Settings;
|
||||
|
||||
use super::History;
|
||||
|
||||
// Test that we don't save history where necessary
|
||||
#[test]
|
||||
fn privacy_test() {
|
||||
let mut settings = Settings::default();
|
||||
settings.cwd_filter = RegexSet::new(["^/supasecret"]).unwrap();
|
||||
settings.history_filter = RegexSet::new(["^psql"]).unwrap();
|
||||
|
||||
let normal_command: History = History::capture()
|
||||
.timestamp(chrono::Utc::now())
|
||||
.command("echo foo")
|
||||
.cwd("/")
|
||||
.build()
|
||||
.into();
|
||||
|
||||
let with_space: History = History::capture()
|
||||
.timestamp(chrono::Utc::now())
|
||||
.command(" echo bar")
|
||||
.cwd("/")
|
||||
.build()
|
||||
.into();
|
||||
|
||||
let stripe_key: History = History::capture()
|
||||
.timestamp(chrono::Utc::now())
|
||||
.command("curl foo.com/bar?key=sk_test_1234567890abcdefghijklmnop")
|
||||
.cwd("/")
|
||||
.build()
|
||||
.into();
|
||||
|
||||
let secret_dir: History = History::capture()
|
||||
.timestamp(chrono::Utc::now())
|
||||
.command("echo ohno")
|
||||
.cwd("/supasecret")
|
||||
.build()
|
||||
.into();
|
||||
|
||||
let with_psql: History = History::capture()
|
||||
.timestamp(chrono::Utc::now())
|
||||
.command("psql")
|
||||
.cwd("/supasecret")
|
||||
.build()
|
||||
.into();
|
||||
|
||||
assert!(normal_command.should_save(&settings));
|
||||
assert!(!with_space.should_save(&settings));
|
||||
assert!(!stripe_key.should_save(&settings));
|
||||
assert!(!secret_dir.should_save(&settings));
|
||||
assert!(!with_psql.should_save(&settings));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disable_secrets() {
|
||||
let mut settings = Settings::default();
|
||||
settings.secrets_filter = false;
|
||||
|
||||
let stripe_key: History = History::capture()
|
||||
.timestamp(chrono::Utc::now())
|
||||
.command("curl foo.com/bar?key=sk_test_1234567890abcdefghijklmnop")
|
||||
.cwd("/")
|
||||
.build()
|
||||
.into();
|
||||
|
||||
assert!(stripe_key.should_save(&settings));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,4 +15,5 @@ pub mod import;
|
|||
pub mod kv;
|
||||
pub mod ordering;
|
||||
pub mod record;
|
||||
pub mod secrets;
|
||||
pub mod settings;
|
||||
|
|
|
|||
54
atuin-client/src/secrets.rs
Normal file
54
atuin-client/src/secrets.rs
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
// This file will probably trigger a lot of scanners. Sorry.
|
||||
|
||||
// A list of (name, regex, test), where test should match against regex
|
||||
pub static SECRET_PATTERNS: &[(&str, &str, &str)] = &[
|
||||
(
|
||||
"AWS Access Key ID",
|
||||
"AKIA[0-9A-Z]{16}",
|
||||
"AKIAIOSFODNN7EXAMPLE",
|
||||
),
|
||||
(
|
||||
"GitHub PAT (old)",
|
||||
"^ghp_[a-zA-Z0-9]{36}$",
|
||||
"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", // legit, I expired it
|
||||
),
|
||||
(
|
||||
"GitHub PAT (new)",
|
||||
"^github_pat_[a-zA-Z0-9]{22}_[a-zA-Z0-9]{59}$",
|
||||
"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", // also legit, also expired
|
||||
),
|
||||
(
|
||||
"Slack OAuth v2 bot",
|
||||
"xoxb-[0-9]{11}-[0-9]{11}-[0-9a-zA-Z]{24}",
|
||||
"xoxb-17653672481-19874698323-pdFZKVeTuE8sk7oOcBrzbqgy",
|
||||
),
|
||||
(
|
||||
"Slack OAuth v2 user token",
|
||||
"xoxp-[0-9]{11}-[0-9]{11}-[0-9a-zA-Z]{24}",
|
||||
"xoxp-17653672481-19874698323-pdFZKVeTuE8sk7oOcBrzbqgy",
|
||||
),
|
||||
(
|
||||
"Slack webhook",
|
||||
"T[a-zA-Z0-9_]{8}/B[a-zA-Z0-9_]{8}/[a-zA-Z0-9_]{24}",
|
||||
"https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX",
|
||||
),
|
||||
("Stripe test key", "sk_test_[0-9a-zA-Z]{24}", "sk_test_1234567890abcdefghijklmnop"),
|
||||
("Stripe live key", "sk_live_[0-9a-zA-Z]{24}", "sk_live_1234567890abcdefghijklmnop"),
|
||||
];
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use regex::Regex;
|
||||
|
||||
use crate::secrets::SECRET_PATTERNS;
|
||||
|
||||
#[test]
|
||||
fn test_secrets() {
|
||||
for (name, regex, test) in SECRET_PATTERNS {
|
||||
let re =
|
||||
Regex::new(regex).expect(format!("Failed to compile regex for {name}").as_str());
|
||||
|
||||
assert!(re.is_match(test), "{name} test failed!");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -7,7 +7,9 @@ use std::{
|
|||
use atuin_common::record::HostId;
|
||||
use chrono::{prelude::*, Utc};
|
||||
use clap::ValueEnum;
|
||||
use config::{Config, Environment, File as ConfigFile, FileFormat};
|
||||
use config::{
|
||||
builder::DefaultState, Config, ConfigBuilder, Environment, File as ConfigFile, FileFormat,
|
||||
};
|
||||
use eyre::{eyre, Context, Result};
|
||||
use fs_err::{create_dir_all, File};
|
||||
use parse_duration::parse;
|
||||
|
|
@ -168,6 +170,7 @@ pub struct Settings {
|
|||
pub history_filter: RegexSet,
|
||||
#[serde(with = "serde_regex", default = "RegexSet::empty")]
|
||||
pub cwd_filter: RegexSet,
|
||||
pub secrets_filter: bool,
|
||||
pub workspaces: bool,
|
||||
pub ctrl_n_shortcuts: bool,
|
||||
|
||||
|
|
@ -330,32 +333,15 @@ impl Settings {
|
|||
None
|
||||
}
|
||||
|
||||
pub fn new() -> Result<Self> {
|
||||
let config_dir = atuin_common::utils::config_dir();
|
||||
|
||||
pub fn builder() -> Result<ConfigBuilder<DefaultState>> {
|
||||
let data_dir = atuin_common::utils::data_dir();
|
||||
|
||||
create_dir_all(&config_dir)
|
||||
.wrap_err_with(|| format!("could not create dir {config_dir:?}"))?;
|
||||
create_dir_all(&data_dir).wrap_err_with(|| format!("could not create dir {data_dir:?}"))?;
|
||||
|
||||
let mut config_file = if let Ok(p) = std::env::var("ATUIN_CONFIG_DIR") {
|
||||
PathBuf::from(p)
|
||||
} else {
|
||||
let mut config_file = PathBuf::new();
|
||||
config_file.push(config_dir);
|
||||
config_file
|
||||
};
|
||||
|
||||
config_file.push("config.toml");
|
||||
|
||||
let db_path = data_dir.join("history.db");
|
||||
let record_store_path = data_dir.join("records.db");
|
||||
|
||||
let key_path = data_dir.join("key");
|
||||
let session_path = data_dir.join("session");
|
||||
|
||||
let mut config_builder = Config::builder()
|
||||
Ok(Config::builder()
|
||||
.set_default("db_path", db_path.to_str())?
|
||||
.set_default("record_store_path", record_store_path.to_str())?
|
||||
.set_default("key_path", key_path.to_str())?
|
||||
|
|
@ -384,11 +370,33 @@ impl Settings {
|
|||
.set_default("session_token", "")?
|
||||
.set_default("workspaces", false)?
|
||||
.set_default("ctrl_n_shortcuts", false)?
|
||||
.set_default("secrets_filter", true)?
|
||||
.add_source(
|
||||
Environment::with_prefix("atuin")
|
||||
.prefix_separator("_")
|
||||
.separator("__"),
|
||||
);
|
||||
))
|
||||
}
|
||||
|
||||
pub fn new() -> Result<Self> {
|
||||
let config_dir = atuin_common::utils::config_dir();
|
||||
let data_dir = atuin_common::utils::data_dir();
|
||||
|
||||
create_dir_all(&config_dir)
|
||||
.wrap_err_with(|| format!("could not create dir {config_dir:?}"))?;
|
||||
create_dir_all(&data_dir).wrap_err_with(|| format!("could not create dir {data_dir:?}"))?;
|
||||
|
||||
let mut config_file = if let Ok(p) = std::env::var("ATUIN_CONFIG_DIR") {
|
||||
PathBuf::from(p)
|
||||
} else {
|
||||
let mut config_file = PathBuf::new();
|
||||
config_file.push(config_dir);
|
||||
config_file
|
||||
};
|
||||
|
||||
config_file.push("config.toml");
|
||||
|
||||
let mut config_builder = Self::builder()?;
|
||||
|
||||
config_builder = if config_file.exists() {
|
||||
config_builder.add_source(ConfigFile::new(
|
||||
|
|
@ -433,3 +441,16 @@ impl Settings {
|
|||
Ok(settings)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Settings {
|
||||
fn default() -> Self {
|
||||
// if this panics something is very wrong, as the default config
|
||||
// does not build or deserialize into the settings struct
|
||||
Self::builder()
|
||||
.expect("Could not build default")
|
||||
.build()
|
||||
.expect("Could not build config")
|
||||
.try_deserialize()
|
||||
.expect("Could not deserialize config")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -191,16 +191,9 @@ impl Cmd {
|
|||
) -> Result<()> {
|
||||
let command = command.join(" ");
|
||||
|
||||
if command.starts_with(' ') || settings.history_filter.is_match(&command) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// It's better for atuin to silently fail here and attempt to
|
||||
// store whatever is ran, than to throw an error to the terminal
|
||||
let cwd = utils::get_current_dir();
|
||||
if !cwd.is_empty() && settings.cwd_filter.is_match(&cwd) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let h: History = History::capture()
|
||||
.timestamp(chrono::Utc::now())
|
||||
|
|
@ -209,6 +202,10 @@ impl Cmd {
|
|||
.build()
|
||||
.into();
|
||||
|
||||
if !h.should_save(settings) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// print the ID
|
||||
// we use this as the key for calling end
|
||||
println!("{}", h.id);
|
||||
|
|
|
|||
|
|
@ -250,6 +250,20 @@ history_filter = [
|
|||
]
|
||||
```
|
||||
|
||||
### secrets_filter
|
||||
|
||||
```
|
||||
secrets_filter = true
|
||||
```
|
||||
|
||||
Defaults to true. This matches history against a set of default regex, and will not save it if we get a match. Defaults include
|
||||
|
||||
1. AWS key id
|
||||
2. Github pat (old and new)
|
||||
3. Slack oauth tokens (bot, user)
|
||||
4. Slack webhooks
|
||||
5. Stripe live/test keys
|
||||
|
||||
## macOS <kbd>Ctrl-n</kbd> key shortcuts
|
||||
|
||||
macOS does not have an <kbd>Alt</kbd> key, although terminal emulators can often be configured to map the <kbd>Option</kbd> key to be used as <kbd>Alt</kbd>. *However*, remapping <kbd>Option</kbd> this way may prevent typing some characters, such as using <kbd>Option-3</kbd> to type `#` on the British English layout. For such a scenario, set the `ctrl_n_shortcuts` option to `true` in your config file to replace <kbd>Alt-0</kbd> to <kbd>Alt-9</kbd> shortcuts with <kbd>Ctrl-0</kbd> to <kbd>Ctrl-9</kbd> instead:
|
||||
|
|
|
|||
Loading…
Reference in a new issue