Allow server configured page size (#994)

* Allow server configured page size

* Backwards compat via semver checks

* Correct header name

Cargo.lock

@@ -197,6 +197,7 @@ dependencies = [
  "http",
  "rand",
  "reqwest",
+ "semver",
  "serde",
  "serde_json",
  "sodiumoxide",

atuin-client/src/sync.rs

@@ -11,7 +11,7 @@ use crate::{
     api_client,
     database::Database,
     encryption::{encrypt, load_encoded_key, load_key},
-    settings::{Settings, HISTORY_PAGE_SIZE},
+    settings::Settings,
 };
 
 pub fn hash_str(string: &str) -> String {
@@ -72,7 +72,7 @@ async fn sync_download(
 
         local_count = db.history_count().await?;
 
-        if page.len() < HISTORY_PAGE_SIZE.try_into().unwrap() {
+        if page.len() < remote_status.page_size.try_into().unwrap() {
             break;
         }
 
@@ -134,7 +134,7 @@ async fn sync_upload(
     let mut cursor = Utc::now();
 
     while local_count > remote_count {
-        let last = db.before(cursor, HISTORY_PAGE_SIZE).await?;
+        let last = db.before(cursor, remote_status.page_size).await?;
         let mut buffer = Vec::new();
 
         if last.is_empty() {

atuin-common/src/api.rs

@@ -74,6 +74,12 @@ pub struct StatusResponse {
     pub count: i64,
     pub username: String,
     pub deleted: Vec<String>,
+
+    // These could/should also go on the index of the server
+    // However, we do not request the server index as a part of normal sync
+    // I'd rather slightly increase the size of this response, than add an extra HTTP request
+    pub page_size: i64, // max page size supported by the server
+    pub version: String,
 }
 
 #[derive(Debug, Serialize, Deserialize)]

atuin-server/Cargo.toml

@@ -34,3 +34,4 @@ tower = "0.4"
 tower-http = { version = "0.3", features = ["trace"] }
 reqwest = { workspace = true }
 argon2 = "0.5.0"
+semver = { workspace = true }

atuin-server/src/database.rs

@@ -14,7 +14,6 @@ use super::{
     models::{History, NewHistory, NewSession, NewUser, Session, User},
 };
 use crate::settings::Settings;
-use crate::settings::HISTORY_PAGE_SIZE;
 
 use atuin_common::utils::get_days_from_month;
 
@@ -51,6 +50,7 @@ pub trait Database {
         created_after: chrono::NaiveDateTime,
         since: chrono::NaiveDateTime,
         host: &str,
+        page_size: i64,
     ) -> Result<Vec<History>>;
 
     async fn add_history(&self, history: &[NewHistory]) -> Result<()>;
@@ -271,6 +271,7 @@ impl Database for Postgres {
         created_after: chrono::NaiveDateTime,
         since: chrono::NaiveDateTime,
         host: &str,
+        page_size: i64,
     ) -> Result<Vec<History>> {
         let res = sqlx::query_as::<_, History>(
             "select id, client_id, user_id, hostname, timestamp, data, created_at from history 
@@ -285,7 +286,7 @@ impl Database for Postgres {
         .bind(host)
         .bind(created_after)
         .bind(since)
-        .bind(HISTORY_PAGE_SIZE)
+        .bind(page_size)
         .fetch_all(&self.pool)
         .await?;
 

atuin-server/src/handlers/history.rs

@@ -2,6 +2,7 @@ use std::collections::HashMap;
 
 use axum::{
     extract::{Path, Query, State},
+    http::HeaderMap,
     Json,
 };
 use http::StatusCode;
@@ -13,6 +14,7 @@ use crate::{
     database::Database,
     models::{NewHistory, User},
     router::AppState,
+    utils::client_version_min,
 };
 
 use atuin_common::api::*;
@@ -41,15 +43,30 @@ pub async fn count<DB: Database>(
 pub async fn list<DB: Database>(
     req: Query<SyncHistoryRequest>,
     user: User,
+    headers: HeaderMap,
     state: State<AppState<DB>>,
 ) -> Result<Json<SyncHistoryResponse>, ErrorResponseStatus<'static>> {
     let db = &state.0.database;
+
+    let agent = headers
+        .get("user-agent")
+        .map_or("", |v| v.to_str().unwrap_or(""));
+
+    let variable_page_size = client_version_min(agent, ">=15.0.0").unwrap_or(false);
+
+    let page_size = if variable_page_size {
+        state.settings.page_size
+    } else {
+        100
+    };
+
     let history = db
         .list_history(
             &user,
             req.sync_ts.naive_utc(),
             req.history_ts.naive_utc(),
             &req.host,
+            page_size,
         )
         .await;
 

atuin-server/src/handlers/status.rs

@@ -7,6 +7,8 @@ use crate::{database::Database, models::User, router::AppState};
 
 use atuin_common::api::*;
 
+const VERSION: &str = env!("CARGO_PKG_VERSION");
+
 #[instrument(skip_all, fields(user.id = user.id))]
 pub async fn status<DB: Database>(
     user: User,
@@ -35,5 +37,7 @@ pub async fn status<DB: Database>(
         count,
         deleted,
         username: user.username,
+        version: VERSION.to_string(),
+        page_size: state.settings.page_size,
     }))
 }

atuin-server/src/lib.rs

@@ -15,6 +15,7 @@ pub mod handlers;
 pub mod models;
 pub mod router;
 pub mod settings;
+pub mod utils;
 
 pub async fn launch(settings: Settings, host: String, port: u16) -> Result<()> {
     let host = host.parse::<IpAddr>()?;

atuin-server/src/settings.rs

@@ -15,6 +15,7 @@ pub struct Settings {
     pub db_uri: String,
     pub open_registration: bool,
     pub max_history_length: usize,
+    pub page_size: i64,
     pub register_webhook_url: Option<String>,
     pub register_webhook_username: String,
 }
@@ -40,6 +41,7 @@ impl Settings {
             .set_default("max_history_length", 8192)?
             .set_default("path", "")?
             .set_default("register_webhook_username", "")?
+            .set_default("page_size", 1100)?
             .add_source(
                 Environment::with_prefix("atuin")
                     .prefix_separator("_")

atuin-server/src/utils.rs

@@ -0,0 +1,15 @@
+use eyre::Result;
+use semver::{Version, VersionReq};
+
+pub fn client_version_min(user_agent: &str, req: &str) -> Result<bool> {
+    if user_agent.is_empty() {
+        return Ok(false);
+    }
+
+    let version = user_agent.replace("atuin/", "");
+
+    let req = VersionReq::parse(req)?;
+    let version = Version::parse(version.as_str())?;
+
+    Ok(req.matches(&version))
+}