Allow server configured page size (#994)
* Allow server configured page size * Backwards compat via semver checks * Correct header name
This commit is contained in:
parent
ca263834e9
commit
d2240e1163
10 changed files with 53 additions and 5 deletions
1
Cargo.lock
generated
1
Cargo.lock
generated
|
@ -197,6 +197,7 @@ dependencies = [
|
||||||
"http",
|
"http",
|
||||||
"rand",
|
"rand",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
|
"semver",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sodiumoxide",
|
"sodiumoxide",
|
||||||
|
|
|
@ -11,7 +11,7 @@ use crate::{
|
||||||
api_client,
|
api_client,
|
||||||
database::Database,
|
database::Database,
|
||||||
encryption::{encrypt, load_encoded_key, load_key},
|
encryption::{encrypt, load_encoded_key, load_key},
|
||||||
settings::{Settings, HISTORY_PAGE_SIZE},
|
settings::Settings,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub fn hash_str(string: &str) -> String {
|
pub fn hash_str(string: &str) -> String {
|
||||||
|
@ -72,7 +72,7 @@ async fn sync_download(
|
||||||
|
|
||||||
local_count = db.history_count().await?;
|
local_count = db.history_count().await?;
|
||||||
|
|
||||||
if page.len() < HISTORY_PAGE_SIZE.try_into().unwrap() {
|
if page.len() < remote_status.page_size.try_into().unwrap() {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -134,7 +134,7 @@ async fn sync_upload(
|
||||||
let mut cursor = Utc::now();
|
let mut cursor = Utc::now();
|
||||||
|
|
||||||
while local_count > remote_count {
|
while local_count > remote_count {
|
||||||
let last = db.before(cursor, HISTORY_PAGE_SIZE).await?;
|
let last = db.before(cursor, remote_status.page_size).await?;
|
||||||
let mut buffer = Vec::new();
|
let mut buffer = Vec::new();
|
||||||
|
|
||||||
if last.is_empty() {
|
if last.is_empty() {
|
||||||
|
|
|
@ -74,6 +74,12 @@ pub struct StatusResponse {
|
||||||
pub count: i64,
|
pub count: i64,
|
||||||
pub username: String,
|
pub username: String,
|
||||||
pub deleted: Vec<String>,
|
pub deleted: Vec<String>,
|
||||||
|
|
||||||
|
// These could/should also go on the index of the server
|
||||||
|
// However, we do not request the server index as a part of normal sync
|
||||||
|
// I'd rather slightly increase the size of this response, than add an extra HTTP request
|
||||||
|
pub page_size: i64, // max page size supported by the server
|
||||||
|
pub version: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize)]
|
#[derive(Debug, Serialize, Deserialize)]
|
||||||
|
|
|
@ -34,3 +34,4 @@ tower = "0.4"
|
||||||
tower-http = { version = "0.3", features = ["trace"] }
|
tower-http = { version = "0.3", features = ["trace"] }
|
||||||
reqwest = { workspace = true }
|
reqwest = { workspace = true }
|
||||||
argon2 = "0.5.0"
|
argon2 = "0.5.0"
|
||||||
|
semver = { workspace = true }
|
||||||
|
|
|
@ -14,7 +14,6 @@ use super::{
|
||||||
models::{History, NewHistory, NewSession, NewUser, Session, User},
|
models::{History, NewHistory, NewSession, NewUser, Session, User},
|
||||||
};
|
};
|
||||||
use crate::settings::Settings;
|
use crate::settings::Settings;
|
||||||
use crate::settings::HISTORY_PAGE_SIZE;
|
|
||||||
|
|
||||||
use atuin_common::utils::get_days_from_month;
|
use atuin_common::utils::get_days_from_month;
|
||||||
|
|
||||||
|
@ -51,6 +50,7 @@ pub trait Database {
|
||||||
created_after: chrono::NaiveDateTime,
|
created_after: chrono::NaiveDateTime,
|
||||||
since: chrono::NaiveDateTime,
|
since: chrono::NaiveDateTime,
|
||||||
host: &str,
|
host: &str,
|
||||||
|
page_size: i64,
|
||||||
) -> Result<Vec<History>>;
|
) -> Result<Vec<History>>;
|
||||||
|
|
||||||
async fn add_history(&self, history: &[NewHistory]) -> Result<()>;
|
async fn add_history(&self, history: &[NewHistory]) -> Result<()>;
|
||||||
|
@ -271,6 +271,7 @@ impl Database for Postgres {
|
||||||
created_after: chrono::NaiveDateTime,
|
created_after: chrono::NaiveDateTime,
|
||||||
since: chrono::NaiveDateTime,
|
since: chrono::NaiveDateTime,
|
||||||
host: &str,
|
host: &str,
|
||||||
|
page_size: i64,
|
||||||
) -> Result<Vec<History>> {
|
) -> Result<Vec<History>> {
|
||||||
let res = sqlx::query_as::<_, History>(
|
let res = sqlx::query_as::<_, History>(
|
||||||
"select id, client_id, user_id, hostname, timestamp, data, created_at from history
|
"select id, client_id, user_id, hostname, timestamp, data, created_at from history
|
||||||
|
@ -285,7 +286,7 @@ impl Database for Postgres {
|
||||||
.bind(host)
|
.bind(host)
|
||||||
.bind(created_after)
|
.bind(created_after)
|
||||||
.bind(since)
|
.bind(since)
|
||||||
.bind(HISTORY_PAGE_SIZE)
|
.bind(page_size)
|
||||||
.fetch_all(&self.pool)
|
.fetch_all(&self.pool)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
|
|
|
@ -2,6 +2,7 @@ use std::collections::HashMap;
|
||||||
|
|
||||||
use axum::{
|
use axum::{
|
||||||
extract::{Path, Query, State},
|
extract::{Path, Query, State},
|
||||||
|
http::HeaderMap,
|
||||||
Json,
|
Json,
|
||||||
};
|
};
|
||||||
use http::StatusCode;
|
use http::StatusCode;
|
||||||
|
@ -13,6 +14,7 @@ use crate::{
|
||||||
database::Database,
|
database::Database,
|
||||||
models::{NewHistory, User},
|
models::{NewHistory, User},
|
||||||
router::AppState,
|
router::AppState,
|
||||||
|
utils::client_version_min,
|
||||||
};
|
};
|
||||||
|
|
||||||
use atuin_common::api::*;
|
use atuin_common::api::*;
|
||||||
|
@ -41,15 +43,30 @@ pub async fn count<DB: Database>(
|
||||||
pub async fn list<DB: Database>(
|
pub async fn list<DB: Database>(
|
||||||
req: Query<SyncHistoryRequest>,
|
req: Query<SyncHistoryRequest>,
|
||||||
user: User,
|
user: User,
|
||||||
|
headers: HeaderMap,
|
||||||
state: State<AppState<DB>>,
|
state: State<AppState<DB>>,
|
||||||
) -> Result<Json<SyncHistoryResponse>, ErrorResponseStatus<'static>> {
|
) -> Result<Json<SyncHistoryResponse>, ErrorResponseStatus<'static>> {
|
||||||
let db = &state.0.database;
|
let db = &state.0.database;
|
||||||
|
|
||||||
|
let agent = headers
|
||||||
|
.get("user-agent")
|
||||||
|
.map_or("", |v| v.to_str().unwrap_or(""));
|
||||||
|
|
||||||
|
let variable_page_size = client_version_min(agent, ">=15.0.0").unwrap_or(false);
|
||||||
|
|
||||||
|
let page_size = if variable_page_size {
|
||||||
|
state.settings.page_size
|
||||||
|
} else {
|
||||||
|
100
|
||||||
|
};
|
||||||
|
|
||||||
let history = db
|
let history = db
|
||||||
.list_history(
|
.list_history(
|
||||||
&user,
|
&user,
|
||||||
req.sync_ts.naive_utc(),
|
req.sync_ts.naive_utc(),
|
||||||
req.history_ts.naive_utc(),
|
req.history_ts.naive_utc(),
|
||||||
&req.host,
|
&req.host,
|
||||||
|
page_size,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
|
|
|
@ -7,6 +7,8 @@ use crate::{database::Database, models::User, router::AppState};
|
||||||
|
|
||||||
use atuin_common::api::*;
|
use atuin_common::api::*;
|
||||||
|
|
||||||
|
const VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||||
|
|
||||||
#[instrument(skip_all, fields(user.id = user.id))]
|
#[instrument(skip_all, fields(user.id = user.id))]
|
||||||
pub async fn status<DB: Database>(
|
pub async fn status<DB: Database>(
|
||||||
user: User,
|
user: User,
|
||||||
|
@ -35,5 +37,7 @@ pub async fn status<DB: Database>(
|
||||||
count,
|
count,
|
||||||
deleted,
|
deleted,
|
||||||
username: user.username,
|
username: user.username,
|
||||||
|
version: VERSION.to_string(),
|
||||||
|
page_size: state.settings.page_size,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
|
@ -15,6 +15,7 @@ pub mod handlers;
|
||||||
pub mod models;
|
pub mod models;
|
||||||
pub mod router;
|
pub mod router;
|
||||||
pub mod settings;
|
pub mod settings;
|
||||||
|
pub mod utils;
|
||||||
|
|
||||||
pub async fn launch(settings: Settings, host: String, port: u16) -> Result<()> {
|
pub async fn launch(settings: Settings, host: String, port: u16) -> Result<()> {
|
||||||
let host = host.parse::<IpAddr>()?;
|
let host = host.parse::<IpAddr>()?;
|
||||||
|
|
|
@ -15,6 +15,7 @@ pub struct Settings {
|
||||||
pub db_uri: String,
|
pub db_uri: String,
|
||||||
pub open_registration: bool,
|
pub open_registration: bool,
|
||||||
pub max_history_length: usize,
|
pub max_history_length: usize,
|
||||||
|
pub page_size: i64,
|
||||||
pub register_webhook_url: Option<String>,
|
pub register_webhook_url: Option<String>,
|
||||||
pub register_webhook_username: String,
|
pub register_webhook_username: String,
|
||||||
}
|
}
|
||||||
|
@ -40,6 +41,7 @@ impl Settings {
|
||||||
.set_default("max_history_length", 8192)?
|
.set_default("max_history_length", 8192)?
|
||||||
.set_default("path", "")?
|
.set_default("path", "")?
|
||||||
.set_default("register_webhook_username", "")?
|
.set_default("register_webhook_username", "")?
|
||||||
|
.set_default("page_size", 1100)?
|
||||||
.add_source(
|
.add_source(
|
||||||
Environment::with_prefix("atuin")
|
Environment::with_prefix("atuin")
|
||||||
.prefix_separator("_")
|
.prefix_separator("_")
|
||||||
|
|
15
atuin-server/src/utils.rs
Normal file
15
atuin-server/src/utils.rs
Normal file
|
@ -0,0 +1,15 @@
|
||||||
|
use eyre::Result;
|
||||||
|
use semver::{Version, VersionReq};
|
||||||
|
|
||||||
|
pub fn client_version_min(user_agent: &str, req: &str) -> Result<bool> {
|
||||||
|
if user_agent.is_empty() {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
let version = user_agent.replace("atuin/", "");
|
||||||
|
|
||||||
|
let req = VersionReq::parse(req)?;
|
||||||
|
let version = Version::parse(version.as_str())?;
|
||||||
|
|
||||||
|
Ok(req.matches(&version))
|
||||||
|
}
|
Loading…
Reference in a new issue