From 5d73115c78681d9a98fa4068d583468969100c87 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 25 Jun 2026 09:07:00 +0200 Subject: [PATCH 01/29] initial logic --- crates/core/src/macros.rs | 46 ++++++ crates/core/src/sync/interface.rs | 81 ++++++++++- crates/core/src/sync/storage_adapter.rs | 106 ++++++++++++++ crates/core/src/sync/streaming_sync.rs | 31 +++- crates/core/src/sync/sync_status.rs | 18 ++- dart/test/goldens/simple_iteration.json | 14 +- dart/test/goldens/starting_stream.json | 5 +- dart/test/sync_test.dart | 180 ++++++++++++++++++++---- dart/test/utils/test_utils.dart | 7 +- docs/sync.md | 15 ++ 10 files changed, 462 insertions(+), 41 deletions(-) diff --git a/crates/core/src/macros.rs b/crates/core/src/macros.rs index 4e60c953..0bc8dc5a 100644 --- a/crates/core/src/macros.rs +++ b/crates/core/src/macros.rs @@ -43,3 +43,49 @@ macro_rules! create_sqlite_optional_text_fn { } }; } + +#[macro_export] +macro_rules! create_sqlite_int_fn { + ($fn_name:ident, $fn_impl_name:ident, $description:literal) => { + extern "C" fn $fn_name( + ctx: *mut sqlite::context, + argc: c_int, + argv: *mut *mut sqlite::value, + ) { + let args = sqlite::args!(argc, argv); + + let result = $fn_impl_name(ctx, args); + + if let Err(err) = result { + PowerSyncError::from(err).apply_to_ctx($description, ctx); + } else if let Ok(r) = result { + ctx.result_int64(r); + } + } + }; +} + +#[macro_export] +macro_rules! create_sqlite_optional_int_fn { + ($fn_name:ident, $fn_impl_name:ident, $description:literal) => { + extern "C" fn $fn_name( + ctx: *mut sqlite::context, + argc: c_int, + argv: *mut *mut sqlite::value, + ) { + let args = sqlite::args!(argc, argv); + + let result = $fn_impl_name(ctx, args); + + if let Err(err) = result { + PowerSyncError::from(err).apply_to_ctx($description, ctx); + } else if let Ok(r) = result { + if let Some(i) = r { + ctx.result_int64(i); + } else { + ctx.result_null(); + } + } + } + }; +} diff --git a/crates/core/src/sync/interface.rs b/crates/core/src/sync/interface.rs index e09eba94..e1fd66e9 100644 --- a/crates/core/src/sync/interface.rs +++ b/crates/core/src/sync/interface.rs @@ -4,12 +4,12 @@ use core::ffi::{c_int, c_void}; use super::streaming_sync::SyncClient; use super::sync_status::DownloadSyncStatus; use crate::constants::SUBTYPE_JSON; -use crate::create_sqlite_text_fn; use crate::error::PowerSyncError; use crate::schema::Schema; use crate::state::DatabaseState; use crate::sync::diagnostics::{DiagnosticOptions, DiagnosticsEvent}; use crate::sync::subscriptions::{StreamKey, apply_subscriptions}; +use crate::{create_sqlite_int_fn, create_sqlite_optional_int_fn, create_sqlite_text_fn}; use alloc::borrow::Cow; use alloc::boxed::Box; use alloc::rc::Rc; @@ -324,6 +324,28 @@ pub fn register(db: *mut sqlite::sqlite3, state: Rc) -> Result<() Some(DatabaseState::destroy_rc), )?; + db.create_function_v2( + "powersync_probe_local_target_op", + 1, + sqlite::UTF8 | sqlite::DIRECTONLY, + Some(Rc::into_raw(state.clone()) as *mut c_void), + Some(powersync_probe_local_target_op), + None, + None, + Some(DatabaseState::destroy_rc), + )?; + + db.create_function_v2( + "powersync_next_checkpoint_request_id", + 0, + sqlite::UTF8 | sqlite::DIRECTONLY, + Some(Rc::into_raw(state) as *mut c_void), + Some(powersync_next_checkpoint_request_id), + None, + None, + Some(DatabaseState::destroy_rc), + )?; + Ok(()) } @@ -340,8 +362,65 @@ fn powersync_offline_sync_status_impl( Ok(serialized) } +fn powersync_probe_local_target_op_impl( + ctx: *mut sqlite::context, + args: &[*mut sqlite::value], +) -> Result, PowerSyncError> { + if args.len() != 1 { + return Err(PowerSyncError::argument_error( + "powersync_probe_local_target_op takes one argument", + )); + } + + let arg = args[0]; + let new_target_op = match arg.value_type() { + ColumnType::Null => None, + ColumnType::Integer => Some(arg.int64()), + _ => { + return Err(PowerSyncError::argument_error( + "target op must be an integer or null", + )); + } + }; + + let db = ctx.db_handle(); + let db_state = unsafe { DatabaseState::from_context(&ctx) }; + let adapter = db_state.storage_adapter(db)?; + adapter.probe_local_target_op(new_target_op) +} + +fn powersync_next_checkpoint_request_id_impl( + ctx: *mut sqlite::context, + args: &[*mut sqlite::value], +) -> Result { + if !args.is_empty() { + return Err(PowerSyncError::argument_error( + "powersync_next_checkpoint_request_id does not take arguments", + )); + } + + let db = ctx.db_handle(); + verify_in_transaction(db)?; + + let db_state = unsafe { DatabaseState::from_context(&ctx) }; + let adapter = db_state.storage_adapter(db)?; + adapter.next_checkpoint_request_id() +} + create_sqlite_text_fn!( powersync_offline_sync_status, powersync_offline_sync_status_impl, "powersync_offline_sync_status" ); + +create_sqlite_optional_int_fn!( + powersync_probe_local_target_op, + powersync_probe_local_target_op_impl, + "powersync_probe_local_target_op" +); + +create_sqlite_int_fn!( + powersync_next_checkpoint_request_id, + powersync_next_checkpoint_request_id_impl, + "powersync_next_checkpoint_request_id" +); diff --git a/crates/core/src/sync/storage_adapter.rs b/crates/core/src/sync/storage_adapter.rs index 7b5298f3..2801afee 100644 --- a/crates/core/src/sync/storage_adapter.rs +++ b/crates/core/src/sync/storage_adapter.rs @@ -26,6 +26,9 @@ use super::{ bucket_priority::BucketPriority, interface::BucketRequest, streaming_sync::OwnedCheckpoint, }; +const LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY: &str = "last_requested_checkpoint_request_id"; +const LAST_SYNCED_CHECKPOINT_REQUEST_ID_KEY: &str = "last_synced_checkpoint_request_id"; + /// An adapter for storing sync state. /// /// This is used to encapsulate some SQL queries used for the sync implementation, making the code @@ -112,6 +115,9 @@ impl StorageAdapter { items }; + let last_synced_checkpoint_request_id = + self.read_i64_kv(LAST_SYNCED_CHECKPOINT_REQUEST_ID_KEY)?; + let mut streams = Vec::new(); self.iterate_local_subscriptions(|sub| { streams.push(ActiveStreamSubscription::from_local(&sub)); @@ -123,6 +129,7 @@ impl StorageAdapter { priority_status: priority_items, downloading: None, streams, + last_synced_checkpoint_request_id, }) } @@ -512,6 +519,105 @@ WHERE bucket = ?1", None }) } + + /// Probes and optionally updates the local bucket target op for older services where the SDK + /// cannot create checkpoint requests explicitly. + /// + /// The target op can also be used internally as a sentinel value such as max op id while local + /// writes are pending, so it must not be interpreted as a checkpoint request id. + /// + /// When the target op is a positive, non-sentinel checkpoint request id, it also updates + /// `last_requested_checkpoint_request_id` so clients can migrate from the legacy target-op + /// flow to client-created checkpoint requests. `0` and sentinel values such as max op id must + /// not update the last requested id. + /// + /// Returns the target op value from before this call. When `target_op` is `None`, this only + /// reads the current value. + pub fn probe_local_target_op( + &self, + target_op: Option, + ) -> Result, PowerSyncError> { + let previous_target_op = self.local_state()?.map(|state| state.target_op); + + let Some(target_op) = target_op else { + return Ok(previous_target_op); + }; + + if target_op < 0 { + return Err(PowerSyncError::argument_error( + "target op must be a non-negative integer", + )); + } + + let stmt = self.db.prepare_v2( + "INSERT INTO ps_buckets(name, pending_delete, last_op, target_op) +VALUES('$local', 1, 0, ?1) +ON CONFLICT(name) DO UPDATE SET target_op = excluded.target_op", + )?; + stmt.bind_int64(1, target_op)?; + stmt.exec()?; + + if target_op > 0 && target_op != i64::MAX { + self.write_i64_kv(LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY, target_op)?; + } + + Ok(previous_target_op) + } + + /// Persists the checkpoint request id returned in a fully applied checkpoint. + pub fn persist_last_synced_checkpoint_request_id( + &self, + request_id: i64, + ) -> Result<(), PowerSyncError> { + self.write_i64_kv(LAST_SYNCED_CHECKPOINT_REQUEST_ID_KEY, request_id) + } + + /// Increments, persists and returns the next client-created checkpoint request id. + pub fn next_checkpoint_request_id(&self) -> Result { + let statement = self.db.prepare_v2( + "INSERT INTO ps_kv(key, value) +VALUES(?1, 1) +ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + 1 +RETURNING value", + )?; + statement.bind_text( + 1, + LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY, + sqlite::Destructor::STATIC, + )?; + + if statement.step()? == ResultCode::ROW { + Ok(statement.column_int64(0)) + } else { + Err(PowerSyncError::unknown_internal()) + } + } + + fn read_i64_kv(&self, key: &'static str) -> Result, PowerSyncError> { + let statement = self + .db + .prepare_v2("SELECT value FROM ps_kv WHERE key = ?1") + .into_db_result(self.db)?; + statement.bind_text(1, key, sqlite::Destructor::STATIC)?; + + Ok(if statement.step()? == ResultCode::ROW { + Some(statement.column_int64(0)) + } else { + None + }) + } + + fn write_i64_kv(&self, key: &'static str, value: i64) -> Result<(), PowerSyncError> { + let stmt = self.db.prepare_v2( + "INSERT INTO ps_kv(key, value) +VALUES(?1, ?2) +ON CONFLICT(key) DO UPDATE SET value = excluded.value ", + )?; + stmt.bind_text(1, key, sqlite::Destructor::STATIC)?; + stmt.bind_int64(2, value)?; + stmt.exec()?; + Ok(()) + } } pub struct LocalState { diff --git a/crates/core/src/sync/streaming_sync.rs b/crates/core/src/sync/streaming_sync.rs index 5c680ce3..c39129dd 100644 --- a/crates/core/src/sync/streaming_sync.rs +++ b/crates/core/src/sync/streaming_sync.rs @@ -363,7 +363,9 @@ impl StreamingSyncIteration { line: "Validated and applied checkpoint".into(), }); event.instructions.push(Instruction::FlushFileSystem {}); + SyncStateMachineTransition::SyncLocalChangesApplied { + synced_checkpoint_request_id: target.write_checkpoint, partial: None, timestamp, } @@ -400,6 +402,7 @@ impl StreamingSyncIteration { } SyncLocalResult::ChangesApplied { timestamp } => { SyncStateMachineTransition::SyncLocalChangesApplied { + synced_checkpoint_request_id: target.checkpoint.write_checkpoint, partial: Some(priority), timestamp, } @@ -497,7 +500,11 @@ impl StreamingSyncIteration { } => { self.validated_but_not_applied = Some(validated_but_not_applied); } - SyncStateMachineTransition::SyncLocalChangesApplied { partial, timestamp } => { + SyncStateMachineTransition::SyncLocalChangesApplied { + synced_checkpoint_request_id, + partial, + timestamp, + } => { if let Some(priority) = partial { self.status.update( |status| { @@ -506,7 +513,7 @@ impl StreamingSyncIteration { &mut event.instructions, ); } else { - self.handle_checkpoint_applied(event, timestamp); + self.handle_checkpoint_applied(event, timestamp, synced_checkpoint_request_id); } } SyncStateMachineTransition::Empty => {} @@ -647,7 +654,7 @@ impl StreamingSyncIteration { line: "Applied pending checkpoint after completed upload".into(), }); - self.handle_checkpoint_applied(event, timestamp); + self.handle_checkpoint_applied(event, timestamp, checkpoint.write_checkpoint); } _ => { event.instructions.push(Instruction::LogLine { @@ -871,6 +878,14 @@ impl StreamingSyncIteration { } } } + + // This is restored into the sync status when initializing a new client. + if priority.is_none() { + if let Some(write_checkpoint) = target.write_checkpoint { + self.adapter + .persist_last_synced_checkpoint_request_id(write_checkpoint)?; + } + } } Ok(result) @@ -931,11 +946,16 @@ impl StreamingSyncIteration { }) } - fn handle_checkpoint_applied(&mut self, event: &mut ActiveEvent, timestamp: TimestampMicros) { + fn handle_checkpoint_applied( + &mut self, + event: &mut ActiveEvent, + timestamp: TimestampMicros, + synced_checkpoint_request_id: Option, + ) { event.instructions.push(Instruction::DidCompleteSync {}); self.status.update( - |status| status.applied_checkpoint(timestamp), + |status| status.applied_checkpoint(timestamp, synced_checkpoint_request_id), &mut event.instructions, ); } @@ -1113,6 +1133,7 @@ enum SyncStateMachineTransition<'a> { validated_but_not_applied: OwnedCheckpoint, }, SyncLocalChangesApplied { + synced_checkpoint_request_id: Option, partial: Option, timestamp: TimestampMicros, }, diff --git a/crates/core/src/sync/sync_status.rs b/crates/core/src/sync/sync_status.rs index eb7daeea..123fb47e 100644 --- a/crates/core/src/sync/sync_status.rs +++ b/crates/core/src/sync/sync_status.rs @@ -53,6 +53,8 @@ pub struct DownloadSyncStatus { /// received), information about how far the download has progressed. pub downloading: Option, pub streams: Vec, + /// The last fully applied checkpoint request id acknowledged by the sync stream. + pub last_synced_checkpoint_request_id: Option, } impl DownloadSyncStatus { @@ -117,9 +119,16 @@ impl DownloadSyncStatus { self.debug_assert_priority_status_is_sorted(); } - pub fn applied_checkpoint(&mut self, now: TimestampMicros) { + pub fn applied_checkpoint( + &mut self, + now: TimestampMicros, + synced_checkpoint_request_id: Option, + ) { self.downloading = None; self.priority_status.clear(); + if let Some(request_id) = synced_checkpoint_request_id { + self.last_synced_checkpoint_request_id = Some(request_id); + } self.priority_status.push(SyncPriorityStatus { priority: BucketPriority::SENTINEL, @@ -137,6 +146,7 @@ impl Default for DownloadSyncStatus { downloading: None, priority_status: Vec::new(), streams: Vec::new(), + last_synced_checkpoint_request_id: None, } } } @@ -180,12 +190,16 @@ impl Serialize for DownloadSyncStatus { } } - let mut serializer = serializer.serialize_struct("DownloadSyncStatus", 4)?; + let mut serializer = serializer.serialize_struct("DownloadSyncStatus", 6)?; serializer.serialize_field("connected", &self.connected)?; serializer.serialize_field("connecting", &self.connecting)?; serializer.serialize_field("priority_status", &self.priority_status)?; serializer.serialize_field("downloading", &self.downloading)?; serializer.serialize_field("streams", &SerializeStreamsWithProgress(self))?; + serializer.serialize_field( + "last_synced_checkpoint_request_id", + &self.last_synced_checkpoint_request_id, + )?; serializer.end() } diff --git a/dart/test/goldens/simple_iteration.json b/dart/test/goldens/simple_iteration.json index 32c1db89..5bc1bb3f 100644 --- a/dart/test/goldens/simple_iteration.json +++ b/dart/test/goldens/simple_iteration.json @@ -10,7 +10,8 @@ "connecting": true, "priority_status": [], "downloading": null, - "streams": [] + "streams": [], + "last_synced_checkpoint_request_id": null } } }, @@ -65,7 +66,8 @@ } } }, - "streams": [] + "streams": [], + "last_synced_checkpoint_request_id": null } } } @@ -115,7 +117,8 @@ } } }, - "streams": [] + "streams": [], + "last_synced_checkpoint_request_id": null } } } @@ -154,7 +157,8 @@ } ], "downloading": null, - "streams": [] + "streams": [], + "last_synced_checkpoint_request_id": null } } } @@ -173,4 +177,4 @@ } ] } -] \ No newline at end of file +] diff --git a/dart/test/goldens/starting_stream.json b/dart/test/goldens/starting_stream.json index 2549905a..8d55ebee 100644 --- a/dart/test/goldens/starting_stream.json +++ b/dart/test/goldens/starting_stream.json @@ -14,7 +14,8 @@ "connecting": true, "priority_status": [], "downloading": null, - "streams": [] + "streams": [], + "last_synced_checkpoint_request_id": null } } }, @@ -38,4 +39,4 @@ } ] } -] \ No newline at end of file +] diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index 579ece80..b14896c4 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -4,12 +4,12 @@ import 'dart:typed_data'; import 'package:bson/bson.dart'; import 'package:file/local.dart'; +import 'package:path/path.dart'; import 'package:sqlite3/common.dart'; import 'package:sqlite3/sqlite3.dart'; import 'package:sqlite3_test/sqlite3_test.dart'; import 'package:test/test.dart'; import 'package:test_descriptor/test_descriptor.dart' as d; -import 'package:path/path.dart'; import 'utils/native_test_utils.dart'; import 'utils/test_utils.dart'; @@ -135,6 +135,32 @@ void _syncTests({ return syncLine(checkpointComplete(priority: priority, lastOpId: lastOpId)); } + Object? lastRequestedCheckpointRequestId() { + final rows = db.select( + "SELECT value FROM ps_kv WHERE key = 'last_requested_checkpoint_request_id'"); + return rows.isEmpty ? null : rows.single.columnAt(0); + } + + int nextCheckpointRequestId() { + db.execute('begin'); + + try { + final [row] = db.select('SELECT powersync_next_checkpoint_request_id()'); + db.execute('commit'); + return row.columnAt(0) as int; + } catch (e) { + db.execute('rollback'); + rethrow; + } + } + + Object? probeLocalTargetOp([int? opId]) { + final [row] = db.select('SELECT powersync_probe_local_target_op(?)', [ + opId, + ]); + return row.columnAt(0); + } + ResultSet fetchRows() { return db.select('select * from items'); } @@ -318,7 +344,7 @@ void _syncTests({ syncTest('remembers sync state', (controller) { invokeControl('start', null); - pushCheckpoint(buckets: priorityBuckets); + pushCheckpoint(buckets: priorityBuckets, writeCheckpoint: '1'); pushCheckpointComplete(); controller.elapse(Duration(minutes: 10)); @@ -331,26 +357,28 @@ void _syncTests({ instructions, contains( containsPair( - 'UpdateSyncStatus', - containsPair( - 'status', + 'UpdateSyncStatus', containsPair( - 'priority_status', - [ - { - 'priority': 2, - 'last_synced_at': timestamp(), - 'has_synced': true - }, - { - 'priority': 2147483647, - 'last_synced_at': timestamp(plusMinutes: -10), - 'has_synced': true - }, - ], - ), - ), - ), + 'status', + allOf( + containsPair('last_synced_checkpoint_request_id', 1), + containsPair( + 'priority_status', + [ + { + 'priority': 2, + 'last_synced_at': timestamp(), + 'has_synced': true + }, + { + 'priority': 2147483647, + 'last_synced_at': timestamp(plusMinutes: -10), + 'has_synced': true + }, + ], + ), + ), + )), ), ); @@ -358,6 +386,7 @@ void _syncTests({ expect(json.decode(row[0]), { 'connected': false, 'connecting': false, + 'last_synced_checkpoint_request_id': 1, 'priority_status': [ {'priority': 2, 'last_synced_at': timestamp(), 'has_synced': true}, { @@ -371,6 +400,106 @@ void _syncTests({ }); }); + syncTest('allocates requested checkpoint request ids', (_) { + expect(nextCheckpointRequestId(), 1); + expect(lastRequestedCheckpointRequestId(), 1); + + expect(nextCheckpointRequestId(), 2); + expect(lastRequestedCheckpointRequestId(), 2); + }); + + syncTest('probes and updates local target op with checkpoint request id', (_) { + expect(probeLocalTargetOp(), isNull); + expect(probeLocalTargetOp(1), isNull); + expect(lastRequestedCheckpointRequestId(), 1); + expect(probeLocalTargetOp(), 1); + expect(probeLocalTargetOp(2), 1); + expect(lastRequestedCheckpointRequestId(), 2); + expect(probeLocalTargetOp(), 2); + + invokeControl('start', null); + + expect(db.select(r"SELECT target_op FROM ps_buckets WHERE name = '$local'"), + [ + {'target_op': 2} + ]); + }); + + syncTest('does not store non-request target ops as checkpoint request id', (_) { + expect(probeLocalTargetOp(0), isNull); + expect(lastRequestedCheckpointRequestId(), isNull); + + expect(probeLocalTargetOp(9223372036854775807), 0); + expect(lastRequestedCheckpointRequestId(), isNull); + expect(probeLocalTargetOp(), 9223372036854775807); + }); + + syncTest('does not persist placeholder checkpoint request id', (_) { + db.execute("insert into items (id, col) values ('local', 'data');"); + + invokeControl('start', null); + + expect(lastRequestedCheckpointRequestId(), isNull); + }); + + syncTest( + 'does not mark checkpoint request id as synced for partial checkpoint', + (_) { + invokeControl('start', null); + + pushCheckpoint(buckets: priorityBuckets, writeCheckpoint: '1'); + final instructions = pushCheckpointComplete(priority: 2); + + expect( + instructions, + contains( + containsPair( + 'UpdateSyncStatus', + containsPair( + 'status', + containsPair('last_synced_checkpoint_request_id', null), + ), + ), + ), + ); + + final [row] = db.select('select powersync_offline_sync_status();'); + expect( + json.decode(row[0]), + containsPair('last_synced_checkpoint_request_id', null), + ); + }, + ); + + syncTest('keeps synced checkpoint request id across normal checkpoints', (_) { + invokeControl('start', null); + + pushCheckpoint(buckets: priorityBuckets, writeCheckpoint: '1'); + pushCheckpointComplete(); + + pushCheckpoint(buckets: priorityBuckets); + final instructions = pushCheckpointComplete(); + + expect( + instructions, + contains( + containsPair( + 'UpdateSyncStatus', + containsPair( + 'status', + containsPair('last_synced_checkpoint_request_id', 1), + ), + ), + ), + ); + + final [row] = db.select('select powersync_offline_sync_status();'); + expect( + json.decode(row[0]), + containsPair('last_synced_checkpoint_request_id', 1), + ); + }); + test('clearing database clears sync status', () { invokeControl('start', null); pushCheckpoint(buckets: priorityBuckets); @@ -816,7 +945,7 @@ void _syncTests({ ]); // Now complete the upload process. - db.execute(r"UPDATE ps_buckets SET target_op = 1 WHERE name = '$local'"); + probeLocalTargetOp(1); invokeControl('completed_upload', null); // This should apply the pending write checkpoint. @@ -832,8 +961,9 @@ void _syncTests({ // Complete upload process db.execute('DELETE FROM ps_crud'); - db.execute(r"UPDATE ps_buckets SET target_op = 1 WHERE name = '$local'"); + probeLocalTargetOp(1); expect(invokeControl('completed_upload', null), isEmpty); + expect(lastRequestedCheckpointRequestId(), 1); // Sync afterwards containing data and write checkpoint. pushCheckpoint(buckets: priorityBuckets, writeCheckpoint: '1'); @@ -861,7 +991,7 @@ void _syncTests({ ]); // Now the upload is complete and requests a write checkpoint - db.execute(r"UPDATE ps_buckets SET target_op = 1 WHERE name = '$local'"); + probeLocalTargetOp(1); expect(invokeControl('completed_upload', null), isEmpty); // Which triggers a new iteration @@ -898,7 +1028,7 @@ void _syncTests({ db.execute("insert into items (id, col) values ('local2', 'data2');"); // Now the upload is complete and requests a write checkpoint - db.execute(r"UPDATE ps_buckets SET target_op = 1 WHERE name = '$local'"); + probeLocalTargetOp(1); expect(invokeControl('completed_upload', null), [ containsPair('LogLine', { 'severity': 'WARNING', diff --git a/dart/test/utils/test_utils.dart b/dart/test/utils/test_utils.dart index 943f632d..eb310ac2 100644 --- a/dart/test/utils/test_utils.dart +++ b/dart/test/utils/test_utils.dart @@ -23,11 +23,16 @@ Object stream(String name, bool isDefault, {List errors = const []}) { } /// Creates a `checkpoint_complete` or `partial_checkpoint_complete` line. -Object checkpointComplete({int? priority, String lastOpId = '1'}) { +Object checkpointComplete({ + int? priority, + String lastOpId = '1', + String? writeCheckpoint, +}) { return { priority == null ? 'checkpoint_complete' : 'partial_checkpoint_complete': { 'last_op_id': lastOpId, if (priority != null) 'priority': priority, + if (writeCheckpoint != null) 'write_checkpoint': writeCheckpoint, }, }; } diff --git a/docs/sync.md b/docs/sync.md index 8737cee0..bc33520c 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -37,6 +37,19 @@ The following commands are supported: If a new subscription is created, or when a subscription without a TTL has been removed, the client will ask to restart the connection. +When uploads request a write checkpoint, SDKs should call +`powersync_next_checkpoint_request_id()` inside a transaction to allocate the id to pass to the +request-checkpoint API. + +`powersync_probe_local_target_op(op_id)` is only for compatibility when a new SDK is used with an +older PowerSync service that does not yet support client-created checkpoint requests. In that mode, +call it after the service-side request is made. Pass `NULL` to probe the current internal `$local` +bucket target op without updating it, or pass an integer to update that target op. In both cases it +returns the value from before the call, or `NULL` if no value existed. Updating to a positive, +non-sentinel target op also stores it as `last_requested_checkpoint_request_id` to support migrating +to client-created checkpoint requests. `0` and sentinel values such as max op id are not stored as +requested checkpoint ids. + `powersync_control` returns a JSON-encoded array of instructions for the client: ```typescript @@ -68,6 +81,8 @@ interface UpdateSyncStatus { connecting: boolean, priority_status: [], downloading: null | DownloadProgress, + streams: [], + last_synced_checkpoint_request_id: null | number, } // Instructs SDKs to refresh credentials from the backend connector. From 696d9502fbda45a34f1ed441838547a543629c93 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 26 Jun 2026 09:10:23 +0200 Subject: [PATCH 02/29] migrations --- crates/core/src/crud_vtab.rs | 4 +- crates/core/src/migrations.rs | 90 +++++++++++++++++++++++- crates/core/src/sync/interface.rs | 22 +++--- crates/core/src/sync/storage_adapter.rs | 80 +++++++++++++-------- crates/core/src/sync/sync_local.rs | 6 +- dart/test/crud_test.dart | 19 +++-- dart/test/migration_test.dart | 92 +++++++++++++++++++++++++ dart/test/sync_test.dart | 17 +++-- dart/test/utils/migration_fixtures.dart | 24 +++++-- docs/sync.md | 41 ++++++++--- 10 files changed, 326 insertions(+), 69 deletions(-) diff --git a/crates/core/src/crud_vtab.rs b/crates/core/src/crud_vtab.rs index 4c0f4070..e14122a9 100644 --- a/crates/core/src/crud_vtab.rs +++ b/crates/core/src/crud_vtab.rs @@ -247,7 +247,9 @@ impl SimpleCrudTransactionMode { fn record_local_write(&mut self, db: *mut sqlite::sqlite3) -> Result<(), ResultCode> { if !self.had_writes { - db.exec_safe(formatcp!("INSERT OR REPLACE INTO ps_buckets(name, last_op, target_op) VALUES('$local', 0, {MAX_OP_ID})"))?; + db.exec_safe(formatcp!( + "INSERT OR REPLACE INTO ps_kv(key, value) VALUES('local_target_op', {MAX_OP_ID})" + ))?; self.had_writes = true; } diff --git a/crates/core/src/migrations.rs b/crates/core/src/migrations.rs index 06f22146..d3a10138 100644 --- a/crates/core/src/migrations.rs +++ b/crates/core/src/migrations.rs @@ -16,7 +16,7 @@ use crate::fix_data::apply_v035_fix; use crate::schema::inspection::ExistingView; use crate::sync::BucketPriority; -pub const LATEST_VERSION: i32 = 13; +pub const LATEST_VERSION: i32 = 14; pub fn powersync_migrate( ctx: *mut sqlite::context, @@ -460,6 +460,94 @@ DROP TABLE ps_sync_state_old; track_migration.exec()?; } + if current_version < 14 && target_version >= 14 { + // Move the legacy `$local` checkpoint bookkeeping into ps_kv. + // + // In older databases, `$local.last_applied_op` represented the latest legacy write + // checkpoint that was actually applied, so it becomes the last synced checkpoint request. + // `$local.last_op` represented the latest legacy write checkpoint seen in the sync stream, + // so it becomes the last seen checkpoint request. + // + // `$local.target_op` can either be a concrete checkpoint request id or a sentinel such as + // i64::MAX while local writes are pending. Store it separately as `local_target_op`, but + // only treat concrete values as requested checkpoint ids. We intentionally don't seed + // `last_requested_checkpoint_request_id` from `$local.last_applied_op` because that is a + // synced value, not necessarily the current requested target. + // + // When the target op is not concrete and there is no existing requested checkpoint id, + // `last_requested_checkpoint_request_id` remains undefined. That migration path is + // ambiguous: a new client-created request would start at 1, or another value lower than + // the service's legacy counter. SDKs should detect the undefined value, create one old + // write checkpoint, persist that returned concrete target through + // `powersync_probe_local_target_op`, and only then start creating checkpoint requests. + let up = "\ +INSERT INTO ps_kv(key, value) +SELECT 'last_synced_checkpoint_request_id', last_applied_op + FROM ps_buckets + WHERE name = '$local' + AND last_applied_op > 0 + AND last_applied_op != 9223372036854775807; + +INSERT INTO ps_kv(key, value) +SELECT 'last_seen_checkpoint_request_id', last_op + FROM ps_buckets + WHERE name = '$local' + AND last_op > 0 + AND last_op != 9223372036854775807; + +INSERT INTO ps_kv(key, value) +SELECT 'last_requested_checkpoint_request_id', target_op + FROM ps_buckets + WHERE name = '$local' + AND target_op > 0 + AND target_op != 9223372036854775807; + +INSERT INTO ps_kv(key, value) +SELECT 'local_target_op', target_op + FROM ps_buckets + WHERE name = '$local' + AND target_op > 0; +"; + local_db.exec_safe(up).into_db_result(local_db)?; + + // Downgrading needs to rebuild the old `$local` row from the new ps_kv state so older SDKs + // can keep using their target-op based blocking behavior. In that model, `$local.last_op` + // tracked the latest seen legacy write checkpoint and was compared with `$local.target_op` + // to decide whether downloaded changes could be applied. The `$local.last_applied_op` + // value represented the synced checkpoint that had actually been applied locally. + // `$local.pending_delete = 1` marked this as a synthetic local-only bucket instead of a + // normal service bucket. Restore each old progress column from its matching ps_kv key. + // The 0 defaults cover a local target that exists before any checkpoint has been seen or + // applied. If `local_target_op` is absent, don't create a `$local` row: the old + // implementation also didn't have a `$local` bucket unless there was local target state to + // track. + const DOWN_STATEMENTS: &[&str] = &[ + "INSERT INTO ps_buckets(name, pending_delete, last_op, last_applied_op, target_op) +SELECT '$local', 1, seen, synced, target + FROM ( + SELECT + IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = 'last_seen_checkpoint_request_id'), 0) AS seen, + IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = 'last_synced_checkpoint_request_id'), 0) AS synced, + (SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = 'local_target_op') AS target + ) + WHERE EXISTS ( + SELECT 1 FROM ps_kv WHERE key = 'local_target_op' + ) +ON CONFLICT(name) DO UPDATE SET + pending_delete = excluded.pending_delete, + last_op = excluded.last_op, + last_applied_op = excluded.last_applied_op, + target_op = excluded.target_op", + "DELETE FROM ps_migration WHERE id >= 14", + ]; + let down = serialize_down_statements(DOWN_STATEMENTS)?; + let track_migration = + local_db.prepare_v2("INSERT INTO ps_migration(id, down_migrations) VALUES (?, ?)")?; + track_migration.bind_int(1, 14)?; + track_migration.bind_text(2, &down, Destructor::STATIC)?; + track_migration.exec()?; + } + Ok(()) } diff --git a/crates/core/src/sync/interface.rs b/crates/core/src/sync/interface.rs index e1fd66e9..967cfd66 100644 --- a/crates/core/src/sync/interface.rs +++ b/crates/core/src/sync/interface.rs @@ -373,15 +373,19 @@ fn powersync_probe_local_target_op_impl( } let arg = args[0]; - let new_target_op = match arg.value_type() { - ColumnType::Null => None, - ColumnType::Integer => Some(arg.int64()), - _ => { - return Err(PowerSyncError::argument_error( - "target op must be an integer or null", - )); - } - }; + let new_target_op = + match arg.value_type() { + ColumnType::Null => None, + ColumnType::Integer => Some(arg.int64()), + ColumnType::Text => Some(arg.text().parse::().map_err(|_| { + PowerSyncError::argument_error("target op must be an integer string") + })?), + _ => { + return Err(PowerSyncError::argument_error( + "target op must be an integer, integer string, or null", + )); + } + }; let db = ctx.db_handle(); let db_state = unsafe { DatabaseState::from_context(&ctx) }; diff --git a/crates/core/src/sync/storage_adapter.rs b/crates/core/src/sync/storage_adapter.rs index 2801afee..5c5eeae6 100644 --- a/crates/core/src/sync/storage_adapter.rs +++ b/crates/core/src/sync/storage_adapter.rs @@ -27,8 +27,14 @@ use super::{ }; const LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY: &str = "last_requested_checkpoint_request_id"; +const LAST_SEEN_CHECKPOINT_REQUEST_ID_KEY: &str = "last_seen_checkpoint_request_id"; const LAST_SYNCED_CHECKPOINT_REQUEST_ID_KEY: &str = "last_synced_checkpoint_request_id"; +// Tracks the local target used to block applying downloaded rows while local writes are +// outstanding. When present, this is normally either the max-op sentinel for pending local writes or +// a concrete checkpoint request id also stored in LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY. +const LOCAL_TARGET_OP_KEY: &str = "local_target_op"; + /// An adapter for storing sync state. /// /// This is used to encapsulate some SQL queries used for the sync implementation, making the code @@ -267,10 +273,8 @@ WHERE bucket = ?1", } } - if let (None, Some(write_checkpoint)) = (&priority, &checkpoint.write_checkpoint) { - update_bucket.bind_int64(1, *write_checkpoint)?; - update_bucket.bind_text(2, "$local", sqlite::Destructor::STATIC)?; - update_bucket.exec()?; + if let (None, Some(checkpoint_request_id)) = (&priority, &checkpoint.write_checkpoint) { + self.persist_last_seen_checkpoint_request_id(*checkpoint_request_id)?; } #[derive(Serialize)] @@ -507,29 +511,26 @@ WHERE bucket = ?1", } pub fn local_state(&self) -> Result, PowerSyncError> { - let stmt = self - .db - .prepare_v2("SELECT target_op FROM ps_buckets WHERE name = ?")?; - stmt.bind_text(1, "$local", sqlite::Destructor::STATIC)?; - - Ok(if stmt.step()? == ResultCode::ROW { - let target_op = stmt.column_int64(0); - Some(LocalState { target_op }) - } else { - None - }) + Ok(self + .read_i64_kv(LOCAL_TARGET_OP_KEY)? + .map(|target_op| LocalState { target_op })) } - /// Probes and optionally updates the local bucket target op for older services where the SDK - /// cannot create checkpoint requests explicitly. + /// Probes and optionally updates the local target op used to block applying downloaded rows + /// while local writes are outstanding. + /// + /// In the write-checkpoint flow, callers allocate a checkpoint request id, post it to the + /// service, and then update this from the max-op sentinel to the concrete checkpoint request id + /// once the request succeeds. This is also used for older services where the SDK cannot create + /// checkpoint requests explicitly. /// /// The target op can also be used internally as a sentinel value such as max op id while local - /// writes are pending, so it must not be interpreted as a checkpoint request id. + /// writes are pending, so it must not always be interpreted as a checkpoint request id. /// /// When the target op is a positive, non-sentinel checkpoint request id, it also updates /// `last_requested_checkpoint_request_id` so clients can migrate from the legacy target-op - /// flow to client-created checkpoint requests. `0` and sentinel values such as max op id must - /// not update the last requested id. + /// flow to client-created checkpoint requests. `0` clears the local target, and sentinel + /// values such as max op id must not update the last requested id. /// /// Returns the target op value from before this call. When `target_op` is `None`, this only /// reads the current value. @@ -539,32 +540,45 @@ WHERE bucket = ?1", ) -> Result, PowerSyncError> { let previous_target_op = self.local_state()?.map(|state| state.target_op); + // Return the previous op if no new target_op has been provided let Some(target_op) = target_op else { return Ok(previous_target_op); }; + // Set the new target op + if target_op < 0 { return Err(PowerSyncError::argument_error( "target op must be a non-negative integer", )); } - let stmt = self.db.prepare_v2( - "INSERT INTO ps_buckets(name, pending_delete, last_op, target_op) -VALUES('$local', 1, 0, ?1) -ON CONFLICT(name) DO UPDATE SET target_op = excluded.target_op", - )?; - stmt.bind_int64(1, target_op)?; - stmt.exec()?; + if target_op == 0 { + self.delete_kv(LOCAL_TARGET_OP_KEY)?; + return Ok(previous_target_op); + } - if target_op > 0 && target_op != i64::MAX { + self.write_i64_kv(LOCAL_TARGET_OP_KEY, target_op)?; + + if target_op != i64::MAX { self.write_i64_kv(LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY, target_op)?; } Ok(previous_target_op) } - /// Persists the checkpoint request id returned in a fully applied checkpoint. + /// Persists the checkpoint request id observed in a complete sync checkpoint. + /// + /// This replaces the legacy `$local.last_op` bookkeeping used to decide whether downloaded + /// data can be applied after local uploads complete. + pub fn persist_last_seen_checkpoint_request_id( + &self, + request_id: i64, + ) -> Result<(), PowerSyncError> { + self.write_i64_kv(LAST_SEEN_CHECKPOINT_REQUEST_ID_KEY, request_id) + } + + /// Persists the checkpoint request id that was applied locally. pub fn persist_last_synced_checkpoint_request_id( &self, request_id: i64, @@ -574,6 +588,7 @@ ON CONFLICT(name) DO UPDATE SET target_op = excluded.target_op", /// Increments, persists and returns the next client-created checkpoint request id. pub fn next_checkpoint_request_id(&self) -> Result { + // Wonder, should this be a sequence instead let statement = self.db.prepare_v2( "INSERT INTO ps_kv(key, value) VALUES(?1, 1) @@ -618,6 +633,13 @@ ON CONFLICT(key) DO UPDATE SET value = excluded.value ", stmt.exec()?; Ok(()) } + + fn delete_kv(&self, key: &'static str) -> Result<(), PowerSyncError> { + let stmt = self.db.prepare_v2("DELETE FROM ps_kv WHERE key = ?1")?; + stmt.bind_text(1, key, sqlite::Destructor::STATIC)?; + stmt.exec()?; + Ok(()) + } } pub struct LocalState { diff --git a/crates/core/src/sync/sync_local.rs b/crates/core/src/sync/sync_local.rs index 1293ef33..aef8f8d4 100644 --- a/crates/core/src/sync/sync_local.rs +++ b/crates/core/src/sync/sync_local.rs @@ -67,7 +67,11 @@ impl<'a> SyncOperation<'a> { if needs_check { // language=SQLite let statement = self.db.prepare_v2( - "SELECT 1 FROM ps_buckets WHERE target_op > last_op AND name = '$local'", + "SELECT 1 +FROM ps_kv AS target +LEFT JOIN ps_kv AS seen ON seen.key = 'last_seen_checkpoint_request_id' +WHERE target.key = 'local_target_op' + AND CAST(target.value AS INTEGER) > COALESCE(CAST(seen.value AS INTEGER), 0)", )?; if statement.step()? == ResultCode::ROW { diff --git a/dart/test/crud_test.dart b/dart/test/crud_test.dart index 5d6d379b..0f561a48 100644 --- a/dart/test/crud_test.dart +++ b/dart/test/crud_test.dart @@ -249,7 +249,7 @@ void main() { }); }); - test('updates local bucket and updated rows', () { + test('updates local target op and updated rows', () { db.execute( 'INSERT INTO powersync_crud (op, id, type, data) VALUES (?, ?, ?, ?)', [ @@ -262,12 +262,17 @@ void main() { expect(db.select('SELECT * FROM ps_updated_rows'), [ {'row_type': 'users', 'row_id': 'foo'} ]); - expect(db.select('SELECT * FROM ps_buckets'), [ - allOf( - containsPair('name', r'$local'), - containsPair('target_op', 9223372036854775807), - ) - ]); + expect(db.select(r"SELECT * FROM ps_buckets WHERE name = '$local'"), + isEmpty); + expect( + db.select( + "SELECT key, value FROM ps_kv WHERE key = 'local_target_op'"), + [ + { + 'key': 'local_target_op', + 'value': 9223372036854775807, + } + ]); }); test('does not require data', () { diff --git a/dart/test/migration_test.dart b/dart/test/migration_test.dart index 9203f74f..b8d6b7cf 100644 --- a/dart/test/migration_test.dart +++ b/dart/test/migration_test.dart @@ -97,6 +97,98 @@ void main() { }); } + test('migrates local checkpoint state to ps_kv', () async { + db.execute(fixtures.expectedState[13]!); + db.execute(r''' +INSERT INTO ps_buckets(id, name, last_applied_op, last_op, target_op, add_checksum, op_checksum, pending_delete, count_at_last, count_since_last, downloaded_size) +VALUES(1, '$local', 5, 6, 7, 0, 0, 1, 0, 0, 0); +'''); + + db.executeInTx('select powersync_init()'); + + expect(db.select('SELECT key, value FROM ps_kv ORDER BY key'), containsAll([ + {'key': 'last_seen_checkpoint_request_id', 'value': 6}, + {'key': 'last_requested_checkpoint_request_id', 'value': 7}, + {'key': 'last_synced_checkpoint_request_id', 'value': 5}, + {'key': 'local_target_op', 'value': 7}, + ])); + }); + + test('does not migrate last applied op as requested checkpoint id', + () async { + db.execute(fixtures.expectedState[13]!); + // Simulate pending local writes during migration. The legacy target op is the max-op + // sentinel, while last_applied_op is the last write checkpoint applied locally. + db.execute(r''' +INSERT INTO ps_buckets(id, name, last_applied_op, last_op, target_op, add_checksum, op_checksum, pending_delete, count_at_last, count_since_last, downloaded_size) +VALUES(1, '$local', 5, 6, 9223372036854775807, 0, 0, 1, 0, 0, 0); +'''); + + db.executeInTx('select powersync_init()'); + + // last_applied_op becomes the synced checkpoint id, but it must not seed the requested + // checkpoint counter. The sentinel target is preserved for blocking, but is not concrete + // enough to become last_requested_checkpoint_request_id. + expect(db.select('SELECT key, value FROM ps_kv ORDER BY key'), [ + {'key': 'last_seen_checkpoint_request_id', 'value': 6}, + {'key': 'last_synced_checkpoint_request_id', 'value': 5}, + {'key': 'local_target_op', 'value': 9223372036854775807}, + ]); + }); + + test('does not migrate sentinel target op as requested checkpoint id', + () async { + db.execute(fixtures.expectedState[13]!); + db.execute(r''' +INSERT INTO ps_buckets(id, name, last_applied_op, last_op, target_op, add_checksum, op_checksum, pending_delete, count_at_last, count_since_last, downloaded_size) +VALUES(1, '$local', 0, 0, 9223372036854775807, 0, 0, 1, 0, 0, 0); +'''); + + db.executeInTx('select powersync_init()'); + + // The max-op sentinel is valid local target state, but it is not a concrete checkpoint + // request id and must not seed last_requested_checkpoint_request_id. + expect(db.select('SELECT key, value FROM ps_kv ORDER BY key'), [ + {'key': 'local_target_op', 'value': 9223372036854775807}, + ]); + }); + + test('restores local checkpoint state on downgrade', () async { + db.execute(fixtures.finalState); + db.execute(r''' +INSERT INTO ps_kv(key, value) VALUES + ('last_requested_checkpoint_request_id', 7), + ('last_seen_checkpoint_request_id', 6), + ('last_synced_checkpoint_request_id', 5), + ('local_target_op', 7); +'''); + + db.executeInTx('select powersync_test_migration(13)'); + + expect( + db.select( + r"SELECT pending_delete, last_op, last_applied_op, target_op FROM ps_buckets WHERE name = '$local'"), + [ + { + 'pending_delete': 1, + 'last_op': 6, + 'last_applied_op': 5, + 'target_op': 7, + } + ], + ); + }); + + test('does not restore local bucket without local target on downgrade', + () async { + db.execute(fixtures.finalState); + + db.executeInTx('select powersync_test_migration(13)'); + + expect(db.select(r"SELECT * FROM ps_buckets WHERE name = '$local'"), + isEmpty); + }); + /// Here we apply a developer schema _after_ migrating test('schema after migration', () async { db.execute(fixtures.expectedState[2]!); diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index b14896c4..b1628b57 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -154,7 +154,7 @@ void _syncTests({ } } - Object? probeLocalTargetOp([int? opId]) { + Object? probeLocalTargetOp([Object? opId]) { final [row] = db.select('SELECT powersync_probe_local_target_op(?)', [ opId, ]); @@ -418,18 +418,20 @@ void _syncTests({ expect(probeLocalTargetOp(), 2); invokeControl('start', null); + }); - expect(db.select(r"SELECT target_op FROM ps_buckets WHERE name = '$local'"), - [ - {'target_op': 2} - ]); + syncTest('accepts text checkpoint request ids for local target op', (_) { + expect(probeLocalTargetOp('1'), isNull); + expect(lastRequestedCheckpointRequestId(), 1); + expect(probeLocalTargetOp(), 1); }); syncTest('does not store non-request target ops as checkpoint request id', (_) { expect(probeLocalTargetOp(0), isNull); expect(lastRequestedCheckpointRequestId(), isNull); + expect(probeLocalTargetOp(), isNull); - expect(probeLocalTargetOp(9223372036854775807), 0); + expect(probeLocalTargetOp(9223372036854775807), isNull); expect(lastRequestedCheckpointRequestId(), isNull); expect(probeLocalTargetOp(), 9223372036854775807); }); @@ -498,6 +500,9 @@ void _syncTests({ json.decode(row[0]), containsPair('last_synced_checkpoint_request_id', 1), ); + + expect(db.select(r"SELECT * FROM ps_buckets WHERE name = '$local'"), + isEmpty); }); test('clearing database clears sync status', () { diff --git a/dart/test/utils/migration_fixtures.dart b/dart/test/utils/migration_fixtures.dart index 9566a182..c52ab845 100644 --- a/dart/test/utils/migration_fixtures.dart +++ b/dart/test/utils/migration_fixtures.dart @@ -1,9 +1,12 @@ /// The current database version -const databaseVersion = 13; +const databaseVersion = 14; /// This is the base database state that we expect at various schema versions. /// Generated by loading the specific library version, and exporting the schema. -const expectedState = { +final expectedState = _expectedState(); + +Map _expectedState() { + final state = { 2: r''' ;CREATE TABLE ps_buckets( name TEXT PRIMARY KEY, @@ -537,12 +540,19 @@ const expectedState = { ;INSERT INTO ps_migration(id, down_migrations) VALUES(11, '[{"sql":"DROP TABLE ps_stream_subscriptions"},{"sql":"DELETE FROM ps_migration WHERE id >= 11"}]') ;INSERT INTO ps_migration(id, down_migrations) VALUES(12, '[{"sql":"ALTER TABLE ps_buckets DROP COLUMN downloaded_size"},{"sql":"DELETE FROM ps_migration WHERE id >= 12"}]') ;INSERT INTO ps_migration(id, down_migrations) VALUES(13, '[{"sql":"UPDATE ps_stream_subscriptions SET expires_at = expires_at / 1000000, last_synced_at = last_synced_at / 1000000"},{"sql":"ALTER TABLE ps_sync_state RENAME TO ps_sync_state_new"},{"sql":"CREATE TABLE ps_sync_state (\n priority INTEGER NOT NULL PRIMARY KEY,\n last_synced_at TEXT NOT NULL\n) STRICT;"},{"sql":"INSERT INTO ps_sync_state (priority, last_synced_at) SELECT priority, datetime(last_synced_at / 1000000, ''unixepoch'') FROM ps_sync_state_new"},{"sql":"DROP TABLE ps_sync_state_new"},{"sql":"DELETE FROM ps_migration WHERE id >= 13"}]')''', -}; + }; + state[14] = '''${state[13]!.trim()} +;INSERT INTO ps_migration(id, down_migrations) VALUES(14, '[{"sql":"INSERT INTO ps_buckets(name, pending_delete, last_op, last_applied_op, target_op)\\nSELECT ''\$local'', 1, seen, synced, target\\n FROM (\\n SELECT\\n IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''last_seen_checkpoint_request_id''), 0) AS seen,\\n IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''last_synced_checkpoint_request_id''), 0) AS synced,\\n (SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''local_target_op'') AS target\\n )\\n WHERE EXISTS (\\n SELECT 1 FROM ps_kv WHERE key = ''local_target_op''\\n )\\nON CONFLICT(name) DO UPDATE SET\\n pending_delete = excluded.pending_delete,\\n last_op = excluded.last_op,\\n last_applied_op = excluded.last_applied_op,\\n target_op = excluded.target_op"},{"sql":"DELETE FROM ps_migration WHERE id >= 14"}]')'''; + return state; +} final finalState = expectedState[databaseVersion]!; /// data to test "up" migrations -const data1 = { +final data1 = _data1(); + +Map _data1() { + final data = { 2: r''' ;INSERT INTO ps_buckets(name, last_applied_op, last_op, target_op, add_checksum, pending_delete) VALUES ('b1', 0, 0, 0, 0, 0), @@ -672,7 +682,10 @@ const data1 = { ;INSERT INTO ps_updated_rows(row_type, row_id) VALUES ('lists', 'l2') ''' -}; + }; + data[14] = data[13]!; + return data; +} /// data to test "down" migrations /// This is slightly different from the above, @@ -719,6 +732,7 @@ final dataDown1 = { 10: data1[9]!, 11: data1[10]!, 12: data1[12]!, + 13: data1[13]!, }; final finalData1 = data1[databaseVersion]!; diff --git a/docs/sync.md b/docs/sync.md index bc33520c..0d681917 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -39,16 +39,37 @@ The following commands are supported: When uploads request a write checkpoint, SDKs should call `powersync_next_checkpoint_request_id()` inside a transaction to allocate the id to pass to the -request-checkpoint API. - -`powersync_probe_local_target_op(op_id)` is only for compatibility when a new SDK is used with an -older PowerSync service that does not yet support client-created checkpoint requests. In that mode, -call it after the service-side request is made. Pass `NULL` to probe the current internal `$local` -bucket target op without updating it, or pass an integer to update that target op. In both cases it -returns the value from before the call, or `NULL` if no value existed. Updating to a positive, -non-sentinel target op also stores it as `last_requested_checkpoint_request_id` to support migrating -to client-created checkpoint requests. `0` and sentinel values such as max op id are not stored as -requested checkpoint ids. +request-checkpoint API. In checkpoint-request mode, the SDK should first allocate the id, then post +that id to the service, and then call `powersync_probe_local_target_op(id)` with the same id once +the service accepts the request. This sets the local target op to the request op, replacing the +pending-write sentinel with the concrete checkpoint request id that the sync stream can satisfy. +`powersync_next_checkpoint_request_id()` only advances the request counter; it does not update the +local target op used to block applying downloaded rows. + +`powersync_probe_local_target_op(op_id)` reads and optionally updates the internal local target op. +The same function is used for compatibility when a new SDK is used with an older PowerSync service +that does not yet support client-created checkpoint requests; after the service-side write +checkpoint request returns a concrete id, call `powersync_probe_local_target_op(id)` with that id. +Pass `NULL` to probe the current internal `$local` target op from `ps_kv` without updating it, or +pass an integer or integer string to update that target op. In both cases it returns the value from +before the call, or `NULL` if no value existed. Updating to a positive, non-sentinel target op also +stores it as `last_requested_checkpoint_request_id` to support migrating to client-created +checkpoint requests. Passing `0` clears the local target, and sentinel values such as max op id are +not stored as requested checkpoint ids. + +Database migration v14 moves legacy `$local` checkpoint state into `ps_kv`: `$local.last_applied_op` +becomes `last_synced_checkpoint_request_id`, `$local.last_op` becomes the internal +`last_seen_checkpoint_request_id`, a concrete `$local.target_op` advances the request counter, and +`$local.target_op` is stored as `local_target_op`. Downgrading restores a `$local` row only when +`local_target_op` exists, so older SDKs can keep using target-op based blocking without inventing a +synthetic local bucket when there was no local target state. + +If the migrated target op is not concrete, for example max op id while local writes are pending, +`last_requested_checkpoint_request_id` may be undefined. SDKs must detect that before calling +`powersync_next_checkpoint_request_id()`, since that function would allocate `1` or another value +lower than the legacy service-side counter. In that ambiguous state, create one old-style write +checkpoint first, store the returned concrete id with `powersync_probe_local_target_op(id)`, and +then switch to client-created checkpoint requests. `powersync_control` returns a JSON-encoded array of instructions for the client: From 4e4f932b8048ddeb23d5586df81e6b4d586a8492 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 26 Jun 2026 10:59:54 +0200 Subject: [PATCH 03/29] add temporary docs --- docs/historic-write-checkpoints.md | 208 ++++++++++++++++++++++ docs/sync.md | 17 +- docs/write-checkpoint-requests.md | 269 +++++++++++++++++++++++++++++ 3 files changed, 488 insertions(+), 6 deletions(-) create mode 100644 docs/historic-write-checkpoints.md create mode 100644 docs/write-checkpoint-requests.md diff --git a/docs/historic-write-checkpoints.md b/docs/historic-write-checkpoints.md new file mode 100644 index 00000000..dd697108 --- /dev/null +++ b/docs/historic-write-checkpoints.md @@ -0,0 +1,208 @@ +# Write Checkpointing + +The general flow for mutations is as follows. + +A client makes a write to a table/view. Triggers are used to populate the `ps_crud` table with an entry for the operation. Every local write marks the `$local` bucket in `ps_buckets` as having a `target_op` of the maximum i64 value - this effectively blocks incoming synced checkpoints from being applied. + +```sql +INSERT OR REPLACE INTO ps_buckets(name, last_op, target_op) VALUES('$local', 0, {MAX_OP_ID}) +``` + +A connected client SDK monitors the `ps_crud` table - or the sync state machine triggers CRUD uploads when ready. The user's `uploadData` gets CRUD transactions with `getNextCrudTransaction` or some equivalent method. Calling the `complete` method on a CRUD transaction-like object will: + +- Remove the entries from `ps_crud` +- Depending on the write checkpoint method used: + - Optionally apply a custom_write_checkpoint as the target_op - ONLY if the `ps_crud` queue is empty + - Ensure that the `target_op` is at the MAX_OP_ID + +```TypeScript + await tx.execute(`DELETE FROM ${PSInternalTable.CRUD} WHERE id <= ?`, [lastClientId]); + if (writeCheckpoint) { + const check = await tx.execute(`SELECT 1 FROM ${PSInternalTable.CRUD} LIMIT 1`); + if (!check.rows?.length) { + await tx.execute(`UPDATE ${PSInternalTable.BUCKETS} SET target_op = CAST(? as INTEGER) WHERE name='$local'`, [ + writeCheckpoint + ]); + } + } else { + await tx.execute(`UPDATE ${PSInternalTable.BUCKETS} SET target_op = CAST(? as INTEGER) WHERE name='$local'`, [ + this.bucketStorageAdapter.getMaxOpId() + ]); + } +``` + +Once all the uploads have completed, the Sync implementation will attempt to update the local target. + +```typescript +// private async _uploadAllCrud(signal: AbortSignal): Promise { + +// AbstractStreamingSyncImplementation.ts line 418 +// Uploading is completed +const neededUpdate = await this.options.adapter.updateLocalTarget(() => this.getWriteCheckpoint()); + +// ... +// } + +// SqliteBucketStorage.ts line 67 +// async updateLocalTarget(cb: () => Promise): Promise { +const rs1 = await this.db.getAll( + "SELECT target_op FROM ps_buckets WHERE name = '$local' AND target_op = CAST(? as INTEGER)", + [MAX_OP_ID] +); + +// If the target op is not the MAX_OP_ID (it's a concrete checkpoint ID) +// Then: Don't fetch a new write checkpoint from the service, leave it as is. +// This essentially caters for the custom write checkpoint case where a concrete `writeCheckpoint` +// is set after all items have been uploaded. +// In the managed checkpoint flow, the target_op should be the MAX_OP_ID here. +if (!rs1.length) { + // Nothing to update + return false; +} + +// The logic below tries to ensure that no uploads happened in-between async operations, +// like fetching a write-checkpoint from the PowerSync service +const rs = await this.db.getAll<{ seq: number }>("SELECT seq FROM main.sqlite_sequence WHERE name = 'ps_crud'"); +if (!rs.length) { + // Nothing to update + return false; +} + +const seqBefore: number = rs[0]['seq']; + +// This callback usually connects to the PowerSync service write-checkpoint2.json endpoint +const opId = await cb(); + +// Now we apply the target_op, only if no other CRUD items have dirtied the local state meanwhile. +return this.writeTransaction(async (tx) => { + const anyData = await tx.execute('SELECT 1 FROM ps_crud LIMIT 1'); + if (anyData.rows?.length) { + // if isNotEmpty + this.logger.debug(`New data uploaded since write checkpoint ${opId} - need new write checkpoint`); + return false; + } + + const rs = await tx.execute("SELECT seq FROM main.sqlite_sequence WHERE name = 'ps_crud'"); + if (!rs.rows?.length) { + // assert isNotEmpty + throw new Error('SQLite Sequence should not be empty'); + } + + const seqAfter: number = rs.rows?.item(0)['seq']; + if (seqAfter != seqBefore) { + this.logger.debug( + `New data uploaded since write checkpoint ${opId} - need new write checkpoint (sequence updated)` + ); + + // New crud data may have been uploaded since we got the checkpoint. Abort. + return false; + } + + this.logger.debug(`Updating target write checkpoint to ${opId}`); + await tx.execute("UPDATE ps_buckets SET target_op = CAST(? as INTEGER) WHERE name='$local'", [opId]); + return true; +}); +// } +``` + +Concurrently, the streaming sync implementation is reading checkpoints from the PowerSync service `/sync/stream/` endpoint. The PowerSync service reports which checkpoints are associated with a corresponding write checkpoint. + +The client does not publish guarded changes as soon as it sees that value. After a full checkpoint has completed and its checksums have been validated, `sync_local` stores the checkpoint's `write_checkpoint` as `$local.last_op`. Incoming changes can then be applied locally only when `$local.target_op <= $local.last_op` and the `ps_crud` queue is empty. Partial priority 0 syncs are the exception: they may publish while uploads are outstanding. + +```Rust +// sync_local.rs + +fn can_apply_sync_changes(&self) -> Result { + // Don't publish downloaded data until the upload queue is empty (except for downloaded data + // in priority 0, which is published earlier). + + let needs_check = match &self.partial { + Some(p) => !p.priority.may_publish_with_outstanding_uploads(), + None => true, + }; + + if needs_check { + // language=SQLite + let statement = self.db.prepare_v2( + "SELECT 1 FROM ps_buckets WHERE target_op > last_op AND name = '$local'", + )?; + + if statement.step()? == ResultCode::ROW { + return Ok(false); + } + + let statement = self.db.prepare_v2("SELECT 1 FROM ps_crud LIMIT 1")?; + if statement.step()? != ResultCode::DONE { + return Ok(false); + } + } + + Ok(true) + } + +// storage_adapter.rs + +pub fn sync_local( + // ... +) { + +// ... + if let (None, Some(write_checkpoint)) = (&priority, &checkpoint.write_checkpoint) { + update_bucket.bind_int64(1, *write_checkpoint)?; + update_bucket.bind_text(2, "$local", sqlite::Destructor::STATIC)?; + update_bucket.exec()?; + } + +// ... +} +``` + +## The $local bucket + +`$local` is a special row in `ps_buckets` used to track whether downloaded changes are safe to +publish while local writes are being uploaded. It is stored in the same table as real sync buckets, +but it is not sent to the sync service as a bucket request. + +- `target_op`: The write checkpoint that must be reached before guarded upstream changes may be + published locally. Local writes set this to `MAX_OP_ID`; custom write checkpoints or managed + write-checkpoint requests replace it with a concrete checkpoint id once the relevant upload has + completed. + + ```sql + -- Local writes block guarded publishes until a concrete write checkpoint is known. + INSERT OR REPLACE INTO ps_buckets(name, last_op, target_op) + VALUES('$local', 0, {MAX_OP_ID}); + + -- After upload completion, custom or managed checkpointing stores the target checkpoint. + UPDATE ps_buckets SET target_op = CAST(? AS INTEGER) WHERE name = '$local'; + ``` + +- `last_op`: The latest write checkpoint observed from the sync service. This is updated from + `checkpoint.write_checkpoint` when a full checkpoint is validated. + + ```rust + let update_bucket = self.db.prepare_v2("UPDATE ps_buckets SET last_op = ? WHERE name = ?")?; + + if let (None, Some(write_checkpoint)) = (&priority, &checkpoint.write_checkpoint) { + update_bucket.bind_int64(1, *write_checkpoint)?; + update_bucket.bind_text(2, "$local", sqlite::Destructor::STATIC)?; + update_bucket.exec()?; + } + ``` + +- `last_applied_op`: The latest write checkpoint whose guarded changes have actually been published + locally. This advances to `last_op` after a full `sync_local` apply succeeds. + + ```sql + UPDATE ps_buckets + SET last_applied_op = last_op + WHERE last_applied_op != last_op; + ``` + +The apply gate checks `$local.target_op > $local.last_op` before publishing full checkpoints and +non-priority-0 partial checkpoints. It also checks that `ps_crud` is empty. This means downloaded +changes remain buffered until the client has both uploaded local CRUD and seen the corresponding +write checkpoint in the sync stream. + +Clearing the database removes `$local`: a hard clear deletes all rows from `ps_buckets`, while a +soft clear deletes only the `$local` row and keeps reusable remote bucket state. diff --git a/docs/sync.md b/docs/sync.md index 0d681917..84a70ade 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -64,12 +64,17 @@ becomes `last_synced_checkpoint_request_id`, `$local.last_op` becomes the intern `local_target_op` exists, so older SDKs can keep using target-op based blocking without inventing a synthetic local bucket when there was no local target state. -If the migrated target op is not concrete, for example max op id while local writes are pending, -`last_requested_checkpoint_request_id` may be undefined. SDKs must detect that before calling -`powersync_next_checkpoint_request_id()`, since that function would allocate `1` or another value -lower than the legacy service-side counter. In that ambiguous state, create one old-style write -checkpoint first, store the returned concrete id with `powersync_probe_local_target_op(id)`, and -then switch to client-created checkpoint requests. +If `local_target_op` is absent after migration, there is no local write gate waiting for a +checkpoint. In that case, SDKs can start client-created checkpoint requests normally, even when +`last_requested_checkpoint_request_id` is undefined and the first allocated id is `1`. + +The ambiguous case is a migrated `local_target_op` of max op id with no +`last_requested_checkpoint_request_id`: local writes are pending, but there is no concrete request +id to wait for yet. The max-op sentinel may also cover earlier pending uploads that were already +associated with legacy service-created write checkpoints, so restarting the client-created counter +from `1` could create a target lower than those existing associations. In that state, create one +old-style write checkpoint first, store the returned concrete id with +`powersync_probe_local_target_op(id)`, and then switch to client-created checkpoint requests. `powersync_control` returns a JSON-encoded array of instructions for the client: diff --git a/docs/write-checkpoint-requests.md b/docs/write-checkpoint-requests.md new file mode 100644 index 00000000..739c06c1 --- /dev/null +++ b/docs/write-checkpoint-requests.md @@ -0,0 +1,269 @@ +# Write Checkpoint State in `ps_kv` + +The new write checkpoint logic moves the historic `$local` bucket bookkeeping into `ps_kv`. +`ps_buckets` now tracks real sync buckets, while local upload gating and checkpoint-request +progress are represented as key/value state. + +At a high level: + +- `local_target_op` replaces `$local.target_op` as the local write apply gate. +- `last_seen_checkpoint_request_id` replaces `$local.last_op`. +- `last_synced_checkpoint_request_id` replaces `$local.last_applied_op`. +- `last_requested_checkpoint_request_id` tracks the latest concrete checkpoint request id known to + the client. + +SDKs should not write these keys directly. They update the local target through +`powersync_probe_local_target_op()`, which is the shared helper for both legacy write checkpoints +and new client-created checkpoint requests. The newer `powersync_next_checkpoint_request_id()` +function only allocates a checkpoint request id; after the service accepts that request, the SDK +uses `powersync_probe_local_target_op(id)` to make the accepted id the local target. + +For the historic `$local` bucket flow, see `historic-write-checkpoints.md`. + +## Local writes + +A client write to a synced table/view records an entry in `ps_crud`. For simple CRUD triggers, the +same transaction also records the affected row in `ps_updated_rows` and sets `local_target_op` to +the maximum i64 value. This is the `ps_kv` equivalent of the old `$local.target_op` sentinel: it +means "there are local writes, but we do not yet know the concrete checkpoint id that will +acknowledge them". + +The sentinel is stored in `ps_kv`, not in `ps_buckets`: + +```sql +INSERT OR REPLACE INTO ps_kv(key, value) +VALUES('local_target_op', MAX_OP_ID); +``` + +## Completing uploaded CRUD + +SDK upload code removes uploaded items from `ps_crud`. If the connector supplies a custom write +checkpoint and the queue is empty, that concrete checkpoint becomes the local target immediately. +Otherwise the target is reset to `MAX_OP_ID`, allowing the sync client to create a standard +checkpoint request after the queue drains. + +```text +transaction { + deleteUploadedCrud(upTo: lastUploadedId) + + if let customCheckpoint, crudQueueIsEmpty { + powersync_probe_local_target_op(customCheckpoint) + } else { + powersync_probe_local_target_op(MAX_OP_ID) + } +} +``` + +## Updating the local target + +Once uploads are complete, the sync client updates the local target through +`powersync_probe_local_target_op()`. It only does this when the current target is still +`MAX_OP_ID`, which avoids overwriting a custom checkpoint that was already stored by +`complete(writeCheckpoint:)`. + +The SDK implementation: + +1. Probes the current target with `powersync_probe_local_target_op(NULL)`. +2. Reads `sqlite_sequence.seq` for `ps_crud`. +3. Gets a concrete checkpoint id from either the new or legacy service API. +4. Re-enters a write transaction. +5. Verifies that `ps_crud` is still empty and that its sequence did not change. +6. Stores the concrete target with `powersync_probe_local_target_op(opId)`. + +```text +if powersync_probe_local_target_op(NULL) == MAX_OP_ID { + let seqBefore = psCrudSequence() + let checkpointId = await createOrFetchCheckpointId() + + transaction { + guard ps_crud.isEmpty && psCrudSequence() == seqBefore else { + return + } + + powersync_probe_local_target_op(checkpointId) + } +} +``` + +In checkpoint-request mode, `getWriteCheckpoint()` calls `requestCheckpoint()`. That allocates an +id locally, sends it to `/sync/checkpoint-request`, and returns the same id once the service accepts +the request. Only then does the upload path store that id as `local_target_op` with +`powersync_probe_local_target_op(id)`. + +```text +let requestId = transaction { + powersync_next_checkpoint_request_id() +} + +POST /sync/checkpoint-request { + client_id, + checkpoint_request_id: requestId +} + +return requestId +``` + +The legacy fallback still calls `/write-checkpoint2.json`; the returned write checkpoint is stored +through the same `powersync_probe_local_target_op(opId)` helper. This keeps SDK target updates +consistent across both protocols. + +## Helper functions + +These SQL functions are the SDK-facing API for the new `ps_kv` checkpoint state. + +`powersync_next_checkpoint_request_id()` must be called inside a transaction. It increments and +returns `last_requested_checkpoint_request_id` in `ps_kv`. + +```sql +INSERT INTO ps_kv(key, value) +VALUES('last_requested_checkpoint_request_id', 1) +ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + 1 +RETURNING value; +``` + +This function only allocates an id. It does not update `local_target_op`. + +Note on sequences: SQLite does not have standalone sequences. The sequence-like alternatives are +either an `AUTOINCREMENT` table backed by SQLite's internal `sqlite_sequence`, or a dedicated +single-row counter table like the existing `ps_tx` transaction counter. The checkpoint request +counter currently lives in `ps_kv` because it is also migrated and seeded from legacy/custom +concrete targets via `powersync_probe_local_target_op()`. If we want stricter structure later, a +dedicated checkpoint-request counter table would be the closest match to a sequence. + +`powersync_probe_local_target_op(op_id)` reads and optionally updates the local target: + +- `NULL` returns the current `local_target_op` without changing it. +- `0` clears `local_target_op`. +- A positive value stores `local_target_op`. +- A positive value other than `i64::MAX` also stores `last_requested_checkpoint_request_id`. +- Negative values and non-integer inputs are rejected. + +The function returns the previous target value, or `NULL` if there was no target. + +```text +previous = ps_kv['local_target_op'] + +if target_op == NULL: + return previous +if target_op == 0: + delete ps_kv['local_target_op'] +else: + ps_kv['local_target_op'] = target_op + if target_op != MAX_OP_ID: + ps_kv['last_requested_checkpoint_request_id'] = target_op + +return previous +``` + +## Applying downloaded checkpoints + +The sync stream reports the checkpoint request id in `checkpoint.write_checkpoint`. After a full +checkpoint validates, core persists it as `last_seen_checkpoint_request_id`. + +```text +on full checkpoint with write_checkpoint: + ps_kv['last_seen_checkpoint_request_id'] = checkpoint.write_checkpoint +``` + +Before publishing downloaded rows, `sync_local` checks the local gate. Full checkpoints and +non-priority-0 partial checkpoints can only apply when: + +- `local_target_op` is absent, or it is less than or equal to `last_seen_checkpoint_request_id`. +- `ps_crud` is empty. + +Priority 0 partial syncs are the exception: they may publish while uploads are outstanding. + +```sql +SELECT 1 +FROM ps_kv AS target +LEFT JOIN ps_kv AS seen ON seen.key = 'last_seen_checkpoint_request_id' +WHERE target.key = 'local_target_op' + AND CAST(target.value AS INTEGER) > COALESCE(CAST(seen.value AS INTEGER), 0); +``` + +If a full checkpoint validated but cannot apply because local CRUD is pending, the state machine +keeps it as `validated_but_not_applied`. When the SDK later sends `completed_upload`, core retries +that checkpoint unless its `write_checkpoint` is older than the current `local_target_op`. + +```text +on completed_upload: + if pending_checkpoint.write_checkpoint >= local_target_op: + retry applying pending_checkpoint +``` + +After a full checkpoint applies, core stores the applied checkpoint request id as +`last_synced_checkpoint_request_id` and emits it in sync status. + +```text +after full checkpoint apply: + ps_kv['last_synced_checkpoint_request_id'] = checkpoint.write_checkpoint +``` + +## Explicit checkpoint requests + +Swift exposes `PowerSyncDatabaseProtocol.requestCheckpoint()` for callers that want to wait until +the local database has caught up to the service. This creates a checkpoint request id through the +connected sync client and returns a `CheckpointRequest`. + +This explicit API does not update `local_target_op`: it is a wait marker, not a local upload gate. +The returned object waits until sync status reports `last_synced_checkpoint_request_id >= requestId`. + +```text +isSynced = status.lastSyncedCheckpointRequestId >= requestId + +waitForSync() { + for status in syncStatusUpdates { + return when status.lastSyncedCheckpointRequestId >= requestId + throw if status reports a sync error + } +} +``` + +The public database method requires an active or connecting sync client, because a disconnected +request could not be delivered to the service or observed in the sync stream. + +## `ps_kv` checkpoint state + +- `local_target_op`: The current apply gate. It is either `MAX_OP_ID` while local writes are + pending, a concrete checkpoint request id after upload completion, or absent when there is no + local write gate. +- `last_requested_checkpoint_request_id`: The last client-created checkpoint request id allocated + by `powersync_next_checkpoint_request_id()`. `powersync_probe_local_target_op()` also writes this + key for positive, non-sentinel targets so migrated or legacy-created concrete targets can seed the + client request counter. +- `last_seen_checkpoint_request_id`: The latest full checkpoint `write_checkpoint` observed and + validated from the sync stream. +- `last_synced_checkpoint_request_id`: The latest full checkpoint `write_checkpoint` that has been + applied locally. SDKs expose this in sync status and use it to resolve `CheckpointRequest` waits. + +## Migration from `$local` + +Migration v14 moves the old `$local` bucket state into `ps_kv`: + +- `$local.last_applied_op` becomes `last_synced_checkpoint_request_id`. +- `$local.last_op` becomes `last_seen_checkpoint_request_id`. +- A concrete `$local.target_op` becomes `last_requested_checkpoint_request_id`. +- Any positive `$local.target_op`, including `MAX_OP_ID`, becomes `local_target_op`. + +An absent `local_target_op` is safe: there is no local write gate waiting for a checkpoint, so an +SDK can start client-created checkpoint requests from `1`. The sync stream will only report that +request id after the service has accepted and reached it. + +The ambiguous case is a migrated `local_target_op` of `MAX_OP_ID` with no +`last_requested_checkpoint_request_id`. That means there is a pending local write gate but no +concrete request id to wait for yet. The `MAX_OP_ID` sentinel only says that local writes dirtied +the gate; it does not prove that no earlier uploads were already associated with legacy +service-created write checkpoints. Those existing checkpoint ids may be higher than a restarted +client counter such as `1`, and using a lower target could let an older seen checkpoint satisfy the +gate too early. In that state, the SDK should create one legacy write checkpoint first, store the +concrete id with `powersync_probe_local_target_op(id)`, and then switch to client-created +checkpoint requests. + +The down migration rebuilds a `$local` row only when `local_target_op` exists, using: + +- `last_seen_checkpoint_request_id` as `$local.last_op` +- `last_synced_checkpoint_request_id` as `$local.last_applied_op` +- `local_target_op` as `$local.target_op` + +This keeps older SDKs able to use the historic target-op gate after a downgrade without inventing a +synthetic `$local` bucket when there was no local target state. From 093bd8acc24a464af37ed0cf7d4721d67b111658 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Fri, 26 Jun 2026 16:38:15 +0200 Subject: [PATCH 04/29] cleanup wording --- crates/core/src/migrations.rs | 31 ++++++++--------- crates/core/src/sync/interface.rs | 5 +++ crates/core/src/sync/storage_adapter.rs | 17 ++++----- crates/core/src/sync/streaming_sync.rs | 46 ++++++++++++++----------- crates/core/src/sync/sync_status.rs | 16 ++++----- dart/test/goldens/simple_iteration.json | 8 ++--- dart/test/goldens/starting_stream.json | 2 +- dart/test/migration_test.dart | 8 ++--- dart/test/sync_test.dart | 12 +++---- dart/test/utils/migration_fixtures.dart | 2 +- dart/test/utils/test_utils.dart | 2 -- docs/sync.md | 4 +-- docs/write-checkpoint-requests.md | 18 +++++----- 13 files changed, 86 insertions(+), 85 deletions(-) diff --git a/crates/core/src/migrations.rs b/crates/core/src/migrations.rs index d3a10138..04996084 100644 --- a/crates/core/src/migrations.rs +++ b/crates/core/src/migrations.rs @@ -464,36 +464,33 @@ DROP TABLE ps_sync_state_old; // Move the legacy `$local` checkpoint bookkeeping into ps_kv. // // In older databases, `$local.last_applied_op` represented the latest legacy write - // checkpoint that was actually applied, so it becomes the last synced checkpoint request. + // checkpoint that was actually applied, so it becomes the last applied checkpoint request. // `$local.last_op` represented the latest legacy write checkpoint seen in the sync stream, // so it becomes the last seen checkpoint request. // // `$local.target_op` can either be a concrete checkpoint request id or a sentinel such as // i64::MAX while local writes are pending. Store it separately as `local_target_op`, but // only treat concrete values as requested checkpoint ids. We intentionally don't seed - // `last_requested_checkpoint_request_id` from `$local.last_applied_op` because that is a - // synced value, not necessarily the current requested target. + // `last_requested_checkpoint_request_id` from `$local.last_applied_op` because that is an + // applied value, not necessarily the current requested target. // - // When the target op is not concrete and there is no existing requested checkpoint id, - // `last_requested_checkpoint_request_id` remains undefined. That migration path is - // ambiguous: a new client-created request would start at 1, or another value lower than - // the service's legacy counter. SDKs should detect the undefined value, create one old - // write checkpoint, persist that returned concrete target through - // `powersync_probe_local_target_op`, and only then start creating checkpoint requests. + // An absent local target can safely start client-created checkpoint requests from 1. The + // ambiguous case is an existing max-op local target without a concrete requested id: + // pending local writes may already be associated with legacy service-created write + // checkpoints, so SDKs should bridge once through the legacy endpoint before starting + // client-created checkpoint requests. let up = "\ INSERT INTO ps_kv(key, value) -SELECT 'last_synced_checkpoint_request_id', last_applied_op +SELECT 'last_applied_checkpoint_request_id', last_applied_op FROM ps_buckets WHERE name = '$local' - AND last_applied_op > 0 - AND last_applied_op != 9223372036854775807; + AND last_applied_op > 0; INSERT INTO ps_kv(key, value) SELECT 'last_seen_checkpoint_request_id', last_op FROM ps_buckets WHERE name = '$local' - AND last_op > 0 - AND last_op != 9223372036854775807; + AND last_op > 0; INSERT INTO ps_kv(key, value) SELECT 'last_requested_checkpoint_request_id', target_op @@ -514,7 +511,7 @@ SELECT 'local_target_op', target_op // can keep using their target-op based blocking behavior. In that model, `$local.last_op` // tracked the latest seen legacy write checkpoint and was compared with `$local.target_op` // to decide whether downloaded changes could be applied. The `$local.last_applied_op` - // value represented the synced checkpoint that had actually been applied locally. + // value represented the checkpoint that had actually been applied locally. // `$local.pending_delete = 1` marked this as a synthetic local-only bucket instead of a // normal service bucket. Restore each old progress column from its matching ps_kv key. // The 0 defaults cover a local target that exists before any checkpoint has been seen or @@ -523,11 +520,11 @@ SELECT 'local_target_op', target_op // track. const DOWN_STATEMENTS: &[&str] = &[ "INSERT INTO ps_buckets(name, pending_delete, last_op, last_applied_op, target_op) -SELECT '$local', 1, seen, synced, target +SELECT '$local', 1, seen, applied, target FROM ( SELECT IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = 'last_seen_checkpoint_request_id'), 0) AS seen, - IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = 'last_synced_checkpoint_request_id'), 0) AS synced, + IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = 'last_applied_checkpoint_request_id'), 0) AS applied, (SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = 'local_target_op') AS target ) WHERE EXISTS ( diff --git a/crates/core/src/sync/interface.rs b/crates/core/src/sync/interface.rs index 967cfd66..3ad63933 100644 --- a/crates/core/src/sync/interface.rs +++ b/crates/core/src/sync/interface.rs @@ -388,6 +388,11 @@ fn powersync_probe_local_target_op_impl( }; let db = ctx.db_handle(); + + if new_target_op.is_some() { + verify_in_transaction(db)?; + } + let db_state = unsafe { DatabaseState::from_context(&ctx) }; let adapter = db_state.storage_adapter(db)?; adapter.probe_local_target_op(new_target_op) diff --git a/crates/core/src/sync/storage_adapter.rs b/crates/core/src/sync/storage_adapter.rs index 5c5eeae6..d62dce23 100644 --- a/crates/core/src/sync/storage_adapter.rs +++ b/crates/core/src/sync/storage_adapter.rs @@ -28,7 +28,7 @@ use super::{ const LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY: &str = "last_requested_checkpoint_request_id"; const LAST_SEEN_CHECKPOINT_REQUEST_ID_KEY: &str = "last_seen_checkpoint_request_id"; -const LAST_SYNCED_CHECKPOINT_REQUEST_ID_KEY: &str = "last_synced_checkpoint_request_id"; +const LAST_APPLIED_CHECKPOINT_REQUEST_ID_KEY: &str = "last_applied_checkpoint_request_id"; // Tracks the local target used to block applying downloaded rows while local writes are // outstanding. When present, this is normally either the max-op sentinel for pending local writes or @@ -121,8 +121,8 @@ impl StorageAdapter { items }; - let last_synced_checkpoint_request_id = - self.read_i64_kv(LAST_SYNCED_CHECKPOINT_REQUEST_ID_KEY)?; + let last_applied_checkpoint_request_id = + self.read_i64_kv(LAST_APPLIED_CHECKPOINT_REQUEST_ID_KEY)?; let mut streams = Vec::new(); self.iterate_local_subscriptions(|sub| { @@ -135,7 +135,7 @@ impl StorageAdapter { priority_status: priority_items, downloading: None, streams, - last_synced_checkpoint_request_id, + last_applied_checkpoint_request_id, }) } @@ -545,8 +545,6 @@ WHERE bucket = ?1", return Ok(previous_target_op); }; - // Set the new target op - if target_op < 0 { return Err(PowerSyncError::argument_error( "target op must be a non-negative integer", @@ -579,16 +577,15 @@ WHERE bucket = ?1", } /// Persists the checkpoint request id that was applied locally. - pub fn persist_last_synced_checkpoint_request_id( + pub fn persist_last_applied_checkpoint_request_id( &self, request_id: i64, ) -> Result<(), PowerSyncError> { - self.write_i64_kv(LAST_SYNCED_CHECKPOINT_REQUEST_ID_KEY, request_id) + self.write_i64_kv(LAST_APPLIED_CHECKPOINT_REQUEST_ID_KEY, request_id) } /// Increments, persists and returns the next client-created checkpoint request id. pub fn next_checkpoint_request_id(&self) -> Result { - // Wonder, should this be a sequence instead let statement = self.db.prepare_v2( "INSERT INTO ps_kv(key, value) VALUES(?1, 1) @@ -626,7 +623,7 @@ RETURNING value", let stmt = self.db.prepare_v2( "INSERT INTO ps_kv(key, value) VALUES(?1, ?2) -ON CONFLICT(key) DO UPDATE SET value = excluded.value ", +ON CONFLICT(key) DO UPDATE SET value = excluded.value", )?; stmt.bind_text(1, key, sqlite::Destructor::STATIC)?; stmt.bind_int64(2, value)?; diff --git a/crates/core/src/sync/streaming_sync.rs b/crates/core/src/sync/streaming_sync.rs index c39129dd..8ea26b80 100644 --- a/crates/core/src/sync/streaming_sync.rs +++ b/crates/core/src/sync/streaming_sync.rs @@ -365,7 +365,7 @@ impl StreamingSyncIteration { event.instructions.push(Instruction::FlushFileSystem {}); SyncStateMachineTransition::SyncLocalChangesApplied { - synced_checkpoint_request_id: target.write_checkpoint, + applied_checkpoint_request_id: target.write_checkpoint, partial: None, timestamp, } @@ -402,7 +402,7 @@ impl StreamingSyncIteration { } SyncLocalResult::ChangesApplied { timestamp } => { SyncStateMachineTransition::SyncLocalChangesApplied { - synced_checkpoint_request_id: target.checkpoint.write_checkpoint, + applied_checkpoint_request_id: target.checkpoint.write_checkpoint, partial: Some(priority), timestamp, } @@ -460,7 +460,7 @@ impl StreamingSyncIteration { target: &mut SyncTarget, event: &mut ActiveEvent, transition: SyncStateMachineTransition, - ) -> Option { + ) -> Result, PowerSyncError> { match transition { SyncStateMachineTransition::StartTrackingCheckpoint { progress, @@ -494,14 +494,14 @@ impl StreamingSyncIteration { diagnostics.handle_data_line(line, &*status, &mut event.instructions); } } - SyncStateMachineTransition::CloseIteration(close) => return Some(close), + SyncStateMachineTransition::CloseIteration(close) => return Ok(Some(close)), SyncStateMachineTransition::SyncLocalFailedDueToPendingCrud { validated_but_not_applied, } => { self.validated_but_not_applied = Some(validated_but_not_applied); } SyncStateMachineTransition::SyncLocalChangesApplied { - synced_checkpoint_request_id, + applied_checkpoint_request_id, partial, timestamp, } => { @@ -513,13 +513,17 @@ impl StreamingSyncIteration { &mut event.instructions, ); } else { - self.handle_checkpoint_applied(event, timestamp, synced_checkpoint_request_id); + self.handle_checkpoint_applied( + event, + timestamp, + applied_checkpoint_request_id, + )?; } } SyncStateMachineTransition::Empty => {} }; - None + Ok(None) } /// Handles a single sync line. @@ -533,7 +537,7 @@ impl StreamingSyncIteration { line: &SyncLineWithSource, ) -> Result, PowerSyncError> { let transition = self.prepare_handling_sync_line(target, event, line)?; - Ok(self.apply_transition(target, event, transition)) + self.apply_transition(target, event, transition) } /// Runs a full sync iteration, returning nothing when it completes regularly or an error when @@ -654,7 +658,7 @@ impl StreamingSyncIteration { line: "Applied pending checkpoint after completed upload".into(), }); - self.handle_checkpoint_applied(event, timestamp, checkpoint.write_checkpoint); + self.handle_checkpoint_applied(event, timestamp, checkpoint.write_checkpoint)?; } _ => { event.instructions.push(Instruction::LogLine { @@ -878,14 +882,6 @@ impl StreamingSyncIteration { } } } - - // This is restored into the sync status when initializing a new client. - if priority.is_none() { - if let Some(write_checkpoint) = target.write_checkpoint { - self.adapter - .persist_last_synced_checkpoint_request_id(write_checkpoint)?; - } - } } Ok(result) @@ -950,14 +946,22 @@ impl StreamingSyncIteration { &mut self, event: &mut ActiveEvent, timestamp: TimestampMicros, - synced_checkpoint_request_id: Option, - ) { + applied_checkpoint_request_id: Option, + ) -> Result<(), PowerSyncError> { + if let Some(request_id) = applied_checkpoint_request_id { + // Persisted so it can be restored into sync status when initializing a new client. + self.adapter + .persist_last_applied_checkpoint_request_id(request_id)?; + } + event.instructions.push(Instruction::DidCompleteSync {}); self.status.update( - |status| status.applied_checkpoint(timestamp, synced_checkpoint_request_id), + |status| status.applied_checkpoint(timestamp, applied_checkpoint_request_id), &mut event.instructions, ); + + Ok(()) } } @@ -1133,7 +1137,7 @@ enum SyncStateMachineTransition<'a> { validated_but_not_applied: OwnedCheckpoint, }, SyncLocalChangesApplied { - synced_checkpoint_request_id: Option, + applied_checkpoint_request_id: Option, partial: Option, timestamp: TimestampMicros, }, diff --git a/crates/core/src/sync/sync_status.rs b/crates/core/src/sync/sync_status.rs index 123fb47e..7dba73f0 100644 --- a/crates/core/src/sync/sync_status.rs +++ b/crates/core/src/sync/sync_status.rs @@ -53,8 +53,8 @@ pub struct DownloadSyncStatus { /// received), information about how far the download has progressed. pub downloading: Option, pub streams: Vec, - /// The last fully applied checkpoint request id acknowledged by the sync stream. - pub last_synced_checkpoint_request_id: Option, + /// The latest checkpoint request id whose full checkpoint has been applied locally. + pub last_applied_checkpoint_request_id: Option, } impl DownloadSyncStatus { @@ -122,12 +122,12 @@ impl DownloadSyncStatus { pub fn applied_checkpoint( &mut self, now: TimestampMicros, - synced_checkpoint_request_id: Option, + applied_checkpoint_request_id: Option, ) { self.downloading = None; self.priority_status.clear(); - if let Some(request_id) = synced_checkpoint_request_id { - self.last_synced_checkpoint_request_id = Some(request_id); + if let Some(request_id) = applied_checkpoint_request_id { + self.last_applied_checkpoint_request_id = Some(request_id); } self.priority_status.push(SyncPriorityStatus { @@ -146,7 +146,7 @@ impl Default for DownloadSyncStatus { downloading: None, priority_status: Vec::new(), streams: Vec::new(), - last_synced_checkpoint_request_id: None, + last_applied_checkpoint_request_id: None, } } } @@ -197,8 +197,8 @@ impl Serialize for DownloadSyncStatus { serializer.serialize_field("downloading", &self.downloading)?; serializer.serialize_field("streams", &SerializeStreamsWithProgress(self))?; serializer.serialize_field( - "last_synced_checkpoint_request_id", - &self.last_synced_checkpoint_request_id, + "last_applied_checkpoint_request_id", + &self.last_applied_checkpoint_request_id, )?; serializer.end() diff --git a/dart/test/goldens/simple_iteration.json b/dart/test/goldens/simple_iteration.json index 5bc1bb3f..d3bca3fa 100644 --- a/dart/test/goldens/simple_iteration.json +++ b/dart/test/goldens/simple_iteration.json @@ -11,7 +11,7 @@ "priority_status": [], "downloading": null, "streams": [], - "last_synced_checkpoint_request_id": null + "last_applied_checkpoint_request_id": null } } }, @@ -67,7 +67,7 @@ } }, "streams": [], - "last_synced_checkpoint_request_id": null + "last_applied_checkpoint_request_id": null } } } @@ -118,7 +118,7 @@ } }, "streams": [], - "last_synced_checkpoint_request_id": null + "last_applied_checkpoint_request_id": null } } } @@ -158,7 +158,7 @@ ], "downloading": null, "streams": [], - "last_synced_checkpoint_request_id": null + "last_applied_checkpoint_request_id": null } } } diff --git a/dart/test/goldens/starting_stream.json b/dart/test/goldens/starting_stream.json index 8d55ebee..c5573003 100644 --- a/dart/test/goldens/starting_stream.json +++ b/dart/test/goldens/starting_stream.json @@ -15,7 +15,7 @@ "priority_status": [], "downloading": null, "streams": [], - "last_synced_checkpoint_request_id": null + "last_applied_checkpoint_request_id": null } } }, diff --git a/dart/test/migration_test.dart b/dart/test/migration_test.dart index b8d6b7cf..08ea1536 100644 --- a/dart/test/migration_test.dart +++ b/dart/test/migration_test.dart @@ -109,7 +109,7 @@ VALUES(1, '$local', 5, 6, 7, 0, 0, 1, 0, 0, 0); expect(db.select('SELECT key, value FROM ps_kv ORDER BY key'), containsAll([ {'key': 'last_seen_checkpoint_request_id', 'value': 6}, {'key': 'last_requested_checkpoint_request_id', 'value': 7}, - {'key': 'last_synced_checkpoint_request_id', 'value': 5}, + {'key': 'last_applied_checkpoint_request_id', 'value': 5}, {'key': 'local_target_op', 'value': 7}, ])); }); @@ -126,12 +126,12 @@ VALUES(1, '$local', 5, 6, 9223372036854775807, 0, 0, 1, 0, 0, 0); db.executeInTx('select powersync_init()'); - // last_applied_op becomes the synced checkpoint id, but it must not seed the requested + // last_applied_op becomes the applied checkpoint id, but it must not seed the requested // checkpoint counter. The sentinel target is preserved for blocking, but is not concrete // enough to become last_requested_checkpoint_request_id. expect(db.select('SELECT key, value FROM ps_kv ORDER BY key'), [ {'key': 'last_seen_checkpoint_request_id', 'value': 6}, - {'key': 'last_synced_checkpoint_request_id', 'value': 5}, + {'key': 'last_applied_checkpoint_request_id', 'value': 5}, {'key': 'local_target_op', 'value': 9223372036854775807}, ]); }); @@ -159,7 +159,7 @@ VALUES(1, '$local', 0, 0, 9223372036854775807, 0, 0, 1, 0, 0, 0); INSERT INTO ps_kv(key, value) VALUES ('last_requested_checkpoint_request_id', 7), ('last_seen_checkpoint_request_id', 6), - ('last_synced_checkpoint_request_id', 5), + ('last_applied_checkpoint_request_id', 5), ('local_target_op', 7); '''); diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index b1628b57..d17a10ab 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -361,7 +361,7 @@ void _syncTests({ containsPair( 'status', allOf( - containsPair('last_synced_checkpoint_request_id', 1), + containsPair('last_applied_checkpoint_request_id', 1), containsPair( 'priority_status', [ @@ -386,7 +386,7 @@ void _syncTests({ expect(json.decode(row[0]), { 'connected': false, 'connecting': false, - 'last_synced_checkpoint_request_id': 1, + 'last_applied_checkpoint_request_id': 1, 'priority_status': [ {'priority': 2, 'last_synced_at': timestamp(), 'has_synced': true}, { @@ -459,7 +459,7 @@ void _syncTests({ 'UpdateSyncStatus', containsPair( 'status', - containsPair('last_synced_checkpoint_request_id', null), + containsPair('last_applied_checkpoint_request_id', null), ), ), ), @@ -468,7 +468,7 @@ void _syncTests({ final [row] = db.select('select powersync_offline_sync_status();'); expect( json.decode(row[0]), - containsPair('last_synced_checkpoint_request_id', null), + containsPair('last_applied_checkpoint_request_id', null), ); }, ); @@ -489,7 +489,7 @@ void _syncTests({ 'UpdateSyncStatus', containsPair( 'status', - containsPair('last_synced_checkpoint_request_id', 1), + containsPair('last_applied_checkpoint_request_id', 1), ), ), ), @@ -498,7 +498,7 @@ void _syncTests({ final [row] = db.select('select powersync_offline_sync_status();'); expect( json.decode(row[0]), - containsPair('last_synced_checkpoint_request_id', 1), + containsPair('last_applied_checkpoint_request_id', 1), ); expect(db.select(r"SELECT * FROM ps_buckets WHERE name = '$local'"), diff --git a/dart/test/utils/migration_fixtures.dart b/dart/test/utils/migration_fixtures.dart index c52ab845..555d4681 100644 --- a/dart/test/utils/migration_fixtures.dart +++ b/dart/test/utils/migration_fixtures.dart @@ -542,7 +542,7 @@ Map _expectedState() { ;INSERT INTO ps_migration(id, down_migrations) VALUES(13, '[{"sql":"UPDATE ps_stream_subscriptions SET expires_at = expires_at / 1000000, last_synced_at = last_synced_at / 1000000"},{"sql":"ALTER TABLE ps_sync_state RENAME TO ps_sync_state_new"},{"sql":"CREATE TABLE ps_sync_state (\n priority INTEGER NOT NULL PRIMARY KEY,\n last_synced_at TEXT NOT NULL\n) STRICT;"},{"sql":"INSERT INTO ps_sync_state (priority, last_synced_at) SELECT priority, datetime(last_synced_at / 1000000, ''unixepoch'') FROM ps_sync_state_new"},{"sql":"DROP TABLE ps_sync_state_new"},{"sql":"DELETE FROM ps_migration WHERE id >= 13"}]')''', }; state[14] = '''${state[13]!.trim()} -;INSERT INTO ps_migration(id, down_migrations) VALUES(14, '[{"sql":"INSERT INTO ps_buckets(name, pending_delete, last_op, last_applied_op, target_op)\\nSELECT ''\$local'', 1, seen, synced, target\\n FROM (\\n SELECT\\n IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''last_seen_checkpoint_request_id''), 0) AS seen,\\n IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''last_synced_checkpoint_request_id''), 0) AS synced,\\n (SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''local_target_op'') AS target\\n )\\n WHERE EXISTS (\\n SELECT 1 FROM ps_kv WHERE key = ''local_target_op''\\n )\\nON CONFLICT(name) DO UPDATE SET\\n pending_delete = excluded.pending_delete,\\n last_op = excluded.last_op,\\n last_applied_op = excluded.last_applied_op,\\n target_op = excluded.target_op"},{"sql":"DELETE FROM ps_migration WHERE id >= 14"}]')'''; +;INSERT INTO ps_migration(id, down_migrations) VALUES(14, '[{"sql":"INSERT INTO ps_buckets(name, pending_delete, last_op, last_applied_op, target_op)\\nSELECT ''\$local'', 1, seen, applied, target\\n FROM (\\n SELECT\\n IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''last_seen_checkpoint_request_id''), 0) AS seen,\\n IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''last_applied_checkpoint_request_id''), 0) AS applied,\\n (SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''local_target_op'') AS target\\n )\\n WHERE EXISTS (\\n SELECT 1 FROM ps_kv WHERE key = ''local_target_op''\\n )\\nON CONFLICT(name) DO UPDATE SET\\n pending_delete = excluded.pending_delete,\\n last_op = excluded.last_op,\\n last_applied_op = excluded.last_applied_op,\\n target_op = excluded.target_op"},{"sql":"DELETE FROM ps_migration WHERE id >= 14"}]')'''; return state; } diff --git a/dart/test/utils/test_utils.dart b/dart/test/utils/test_utils.dart index eb310ac2..8c6bd1fc 100644 --- a/dart/test/utils/test_utils.dart +++ b/dart/test/utils/test_utils.dart @@ -26,13 +26,11 @@ Object stream(String name, bool isDefault, {List errors = const []}) { Object checkpointComplete({ int? priority, String lastOpId = '1', - String? writeCheckpoint, }) { return { priority == null ? 'checkpoint_complete' : 'partial_checkpoint_complete': { 'last_op_id': lastOpId, if (priority != null) 'priority': priority, - if (writeCheckpoint != null) 'write_checkpoint': writeCheckpoint, }, }; } diff --git a/docs/sync.md b/docs/sync.md index 84a70ade..e9595d3e 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -58,7 +58,7 @@ checkpoint requests. Passing `0` clears the local target, and sentinel values su not stored as requested checkpoint ids. Database migration v14 moves legacy `$local` checkpoint state into `ps_kv`: `$local.last_applied_op` -becomes `last_synced_checkpoint_request_id`, `$local.last_op` becomes the internal +becomes `last_applied_checkpoint_request_id`, `$local.last_op` becomes the internal `last_seen_checkpoint_request_id`, a concrete `$local.target_op` advances the request counter, and `$local.target_op` is stored as `local_target_op`. Downgrading restores a `$local` row only when `local_target_op` exists, so older SDKs can keep using target-op based blocking without inventing a @@ -108,7 +108,7 @@ interface UpdateSyncStatus { priority_status: [], downloading: null | DownloadProgress, streams: [], - last_synced_checkpoint_request_id: null | number, + last_applied_checkpoint_request_id: null | number, } // Instructs SDKs to refresh credentials from the backend connector. diff --git a/docs/write-checkpoint-requests.md b/docs/write-checkpoint-requests.md index 739c06c1..a03d9356 100644 --- a/docs/write-checkpoint-requests.md +++ b/docs/write-checkpoint-requests.md @@ -8,7 +8,7 @@ At a high level: - `local_target_op` replaces `$local.target_op` as the local write apply gate. - `last_seen_checkpoint_request_id` replaces `$local.last_op`. -- `last_synced_checkpoint_request_id` replaces `$local.last_applied_op`. +- `last_applied_checkpoint_request_id` replaces `$local.last_applied_op`. - `last_requested_checkpoint_request_id` tracks the latest concrete checkpoint request id known to the client. @@ -192,11 +192,11 @@ on completed_upload: ``` After a full checkpoint applies, core stores the applied checkpoint request id as -`last_synced_checkpoint_request_id` and emits it in sync status. +`last_applied_checkpoint_request_id` and emits it in sync status. ```text after full checkpoint apply: - ps_kv['last_synced_checkpoint_request_id'] = checkpoint.write_checkpoint + ps_kv['last_applied_checkpoint_request_id'] = checkpoint.write_checkpoint ``` ## Explicit checkpoint requests @@ -206,14 +206,14 @@ the local database has caught up to the service. This creates a checkpoint reque connected sync client and returns a `CheckpointRequest`. This explicit API does not update `local_target_op`: it is a wait marker, not a local upload gate. -The returned object waits until sync status reports `last_synced_checkpoint_request_id >= requestId`. +The returned object waits until sync status reports `last_applied_checkpoint_request_id >= requestId`. ```text -isSynced = status.lastSyncedCheckpointRequestId >= requestId +isSynced = status.lastAppliedCheckpointRequestId >= requestId waitForSync() { for status in syncStatusUpdates { - return when status.lastSyncedCheckpointRequestId >= requestId + return when status.lastAppliedCheckpointRequestId >= requestId throw if status reports a sync error } } @@ -233,14 +233,14 @@ request could not be delivered to the service or observed in the sync stream. client request counter. - `last_seen_checkpoint_request_id`: The latest full checkpoint `write_checkpoint` observed and validated from the sync stream. -- `last_synced_checkpoint_request_id`: The latest full checkpoint `write_checkpoint` that has been +- `last_applied_checkpoint_request_id`: The latest full checkpoint `write_checkpoint` that has been applied locally. SDKs expose this in sync status and use it to resolve `CheckpointRequest` waits. ## Migration from `$local` Migration v14 moves the old `$local` bucket state into `ps_kv`: -- `$local.last_applied_op` becomes `last_synced_checkpoint_request_id`. +- `$local.last_applied_op` becomes `last_applied_checkpoint_request_id`. - `$local.last_op` becomes `last_seen_checkpoint_request_id`. - A concrete `$local.target_op` becomes `last_requested_checkpoint_request_id`. - Any positive `$local.target_op`, including `MAX_OP_ID`, becomes `local_target_op`. @@ -262,7 +262,7 @@ checkpoint requests. The down migration rebuilds a `$local` row only when `local_target_op` exists, using: - `last_seen_checkpoint_request_id` as `$local.last_op` -- `last_synced_checkpoint_request_id` as `$local.last_applied_op` +- `last_applied_checkpoint_request_id` as `$local.last_applied_op` - `local_target_op` as `$local.target_op` This keeps older SDKs able to use the historic target-op gate after a downgrade without inventing a From 8da12bb12f0077d28383582615fc48e83740b30e Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 2 Jul 2026 12:48:48 +0200 Subject: [PATCH 05/29] wip: move functions to powersync_control invocations --- crates/core/src/sync/interface.rs | 156 +++++++++------------ crates/core/src/sync/storage_adapter.rs | 36 ++++- crates/core/src/sync/streaming_sync.rs | 69 +++++---- dart/benchmark/apply_lines.dart | 4 +- dart/test/error_test.dart | 1 + dart/test/goldens/simple_iteration.json | 8 +- dart/test/goldens/starting_stream.json | 8 +- dart/test/migration_test.dart | 1 + dart/test/sync_local_performance_test.dart | 1 + dart/test/sync_stream_test.dart | 19 ++- dart/test/sync_test.dart | 102 ++++++++++---- docs/sync.md | 66 +++++---- docs/write-checkpoint-requests.md | 73 ++++++---- 13 files changed, 351 insertions(+), 193 deletions(-) diff --git a/crates/core/src/sync/interface.rs b/crates/core/src/sync/interface.rs index 3ad63933..57e559e6 100644 --- a/crates/core/src/sync/interface.rs +++ b/crates/core/src/sync/interface.rs @@ -4,14 +4,15 @@ use core::ffi::{c_int, c_void}; use super::streaming_sync::SyncClient; use super::sync_status::DownloadSyncStatus; use crate::constants::SUBTYPE_JSON; +use crate::create_sqlite_text_fn; use crate::error::PowerSyncError; use crate::schema::Schema; use crate::state::DatabaseState; use crate::sync::diagnostics::{DiagnosticOptions, DiagnosticsEvent}; use crate::sync::subscriptions::{StreamKey, apply_subscriptions}; -use crate::{create_sqlite_int_fn, create_sqlite_optional_int_fn, create_sqlite_text_fn}; use alloc::borrow::Cow; use alloc::boxed::Box; +use alloc::format; use alloc::rc::Rc; use alloc::{string::String, vec::Vec}; use powersync_sqlite_nostd::bindings::SQLITE_RESULT_SUBTYPE; @@ -76,6 +77,8 @@ pub enum SyncControlRequest<'a> { StartSyncStream(StartSyncStream), /// The client requests to stop the current sync iteration. StopSyncStream, + /// The client requests a new checkpoint request id. + NextCheckpointRequestId, /// The client is forwading a sync event to the core extension. SyncEvent(SyncEvent<'a>), } @@ -89,6 +92,10 @@ pub enum SyncEvent<'a> { /// /// In response, we'll stop the current iteration to begin another one with the new token. DidRefreshToken, + /// Seeds the checkpoint request counter from service state. + SeedCheckpointRequestId { + request_id: Option, + }, /// Notifies the sync client that the current CRUD upload (for which the client SDK is /// responsible) has finished. /// @@ -128,13 +135,24 @@ pub enum Instruction { }, /// Connect to the sync service using the [StreamingSyncRequest] created by the core extension, /// and then forward received lines via [SyncEvent::TextLine] and [SyncEvent::BinaryLine]. - EstablishSyncStream { request: StreamingSyncRequest }, + EstablishSyncStream { + request: StreamingSyncRequest, + /// The latest checkpoint request id known locally before opening this stream. + /// + /// SDKs can use a missing value as a cue to fetch checkpoint request state from the service + /// and report it back with `seed_checkpoint_request_id`. + last_checkpoint_request_id: Option, + }, FetchCredentials { /// Whether the credentials currently used have expired. /// /// If false, this is a pre-fetch. did_expire: bool, }, + /// Return a newly allocated checkpoint request id to the SDK. + CheckpointRequestId { request_id: i64 }, + /// Return the local target op value observed before an optional update. + LocalTargetOp { target_op: Option }, // These are defined like this because deserializers in Kotlin can't support either an // object or a literal value /// Close the websocket / HTTP stream to the sync service. @@ -232,6 +250,24 @@ pub fn register(db: *mut sqlite::sqlite3, state: Rc) -> Result<() } }), "stop" => SyncControlRequest::StopSyncStream, + "next_checkpoint_request_id" => SyncControlRequest::NextCheckpointRequestId, + "local_target_op" => { + let target_op = parse_optional_i64_payload( + *payload, + "local target op", + "local target op must be an integer, integer string, or null", + )?; + let adapter = state.storage_adapter(db)?; + let target_op = adapter.probe_local_target_op(target_op)?; + let formatted = + serde_json::to_string(&alloc::vec![Instruction::LocalTargetOp { + target_op + },]) + .map_err(PowerSyncError::internal)?; + ctx.result_text_transient(&formatted); + ctx.result_subtype(SUBTYPE_JSON); + return Ok(()); + } "line_text" => SyncControlRequest::SyncEvent(SyncEvent::TextLine { data: if payload.value_type() == ColumnType::Text { payload.text() @@ -251,6 +287,15 @@ pub fn register(db: *mut sqlite::sqlite3, state: Rc) -> Result<() }, }), "refreshed_token" => SyncControlRequest::SyncEvent(SyncEvent::DidRefreshToken), + "seed_checkpoint_request_id" => { + SyncControlRequest::SyncEvent(SyncEvent::SeedCheckpointRequestId { + request_id: parse_optional_i64_payload( + *payload, + "checkpoint request id", + "checkpoint request id must be an integer, integer string, or null", + )?, + }) + } "completed_upload" => SyncControlRequest::SyncEvent(SyncEvent::UploadFinished), "update_subscriptions" => { SyncControlRequest::SyncEvent(SyncEvent::DidUpdateSubscriptions { @@ -324,28 +369,6 @@ pub fn register(db: *mut sqlite::sqlite3, state: Rc) -> Result<() Some(DatabaseState::destroy_rc), )?; - db.create_function_v2( - "powersync_probe_local_target_op", - 1, - sqlite::UTF8 | sqlite::DIRECTONLY, - Some(Rc::into_raw(state.clone()) as *mut c_void), - Some(powersync_probe_local_target_op), - None, - None, - Some(DatabaseState::destroy_rc), - )?; - - db.create_function_v2( - "powersync_next_checkpoint_request_id", - 0, - sqlite::UTF8 | sqlite::DIRECTONLY, - Some(Rc::into_raw(state) as *mut c_void), - Some(powersync_next_checkpoint_request_id), - None, - None, - Some(DatabaseState::destroy_rc), - )?; - Ok(()) } @@ -362,74 +385,31 @@ fn powersync_offline_sync_status_impl( Ok(serialized) } -fn powersync_probe_local_target_op_impl( - ctx: *mut sqlite::context, - args: &[*mut sqlite::value], -) -> Result, PowerSyncError> { - if args.len() != 1 { - return Err(PowerSyncError::argument_error( - "powersync_probe_local_target_op takes one argument", - )); - } - - let arg = args[0]; - let new_target_op = - match arg.value_type() { - ColumnType::Null => None, - ColumnType::Integer => Some(arg.int64()), - ColumnType::Text => Some(arg.text().parse::().map_err(|_| { - PowerSyncError::argument_error("target op must be an integer string") - })?), - _ => { - return Err(PowerSyncError::argument_error( - "target op must be an integer, integer string, or null", - )); - } - }; - - let db = ctx.db_handle(); - - if new_target_op.is_some() { - verify_in_transaction(db)?; - } - - let db_state = unsafe { DatabaseState::from_context(&ctx) }; - let adapter = db_state.storage_adapter(db)?; - adapter.probe_local_target_op(new_target_op) -} - -fn powersync_next_checkpoint_request_id_impl( - ctx: *mut sqlite::context, - args: &[*mut sqlite::value], -) -> Result { - if !args.is_empty() { - return Err(PowerSyncError::argument_error( - "powersync_next_checkpoint_request_id does not take arguments", - )); - } - - let db = ctx.db_handle(); - verify_in_transaction(db)?; - - let db_state = unsafe { DatabaseState::from_context(&ctx) }; - let adapter = db_state.storage_adapter(db)?; - adapter.next_checkpoint_request_id() -} - create_sqlite_text_fn!( powersync_offline_sync_status, powersync_offline_sync_status_impl, "powersync_offline_sync_status" ); -create_sqlite_optional_int_fn!( - powersync_probe_local_target_op, - powersync_probe_local_target_op_impl, - "powersync_probe_local_target_op" -); +fn parse_optional_i64_payload( + payload: *mut sqlite::value, + name: &'static str, + type_error: &'static str, +) -> Result, PowerSyncError> { + let value = match payload.value_type() { + ColumnType::Null => return Ok(None), + ColumnType::Integer => payload.int64(), + ColumnType::Text => payload.text().parse::().map_err(|_| { + PowerSyncError::argument_error(format!("{name} must be an integer string")) + })?, + _ => return Err(PowerSyncError::argument_error(type_error)), + }; + + if value < 0 { + return Err(PowerSyncError::argument_error(format!( + "{name} must be a non-negative integer" + ))); + } -create_sqlite_int_fn!( - powersync_next_checkpoint_request_id, - powersync_next_checkpoint_request_id_impl, - "powersync_next_checkpoint_request_id" -); + Ok(Some(value)) +} diff --git a/crates/core/src/sync/storage_adapter.rs b/crates/core/src/sync/storage_adapter.rs index d62dce23..34f0c3b3 100644 --- a/crates/core/src/sync/storage_adapter.rs +++ b/crates/core/src/sync/storage_adapter.rs @@ -540,7 +540,6 @@ WHERE bucket = ?1", ) -> Result, PowerSyncError> { let previous_target_op = self.local_state()?.map(|state| state.target_op); - // Return the previous op if no new target_op has been provided let Some(target_op) = target_op else { return Ok(previous_target_op); }; @@ -558,6 +557,8 @@ WHERE bucket = ?1", self.write_i64_kv(LOCAL_TARGET_OP_KEY, target_op)?; + // Concrete target ops also seed the request counter for clients migrating from legacy + // service-created write checkpoints to client-created checkpoint requests. if target_op != i64::MAX { self.write_i64_kv(LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY, target_op)?; } @@ -605,6 +606,39 @@ RETURNING value", } } + /// Returns whether the local checkpoint request counter has been initialized. + pub fn has_checkpoint_request_id(&self) -> Result { + Ok(self.last_checkpoint_request_id()?.is_some()) + } + + /// Returns the latest checkpoint request id known locally. + pub fn last_checkpoint_request_id(&self) -> Result, PowerSyncError> { + self.read_i64_kv(LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY) + } + + /// Seeds the local checkpoint request counter from service state without moving it backwards. + /// + /// A null service value means the service has no record for this client yet. Store zero in + /// that case so the first local allocation returns one while still marking the state as seeded. + pub fn seed_checkpoint_request_id( + &self, + request_id: Option, + ) -> Result<(), PowerSyncError> { + let stmt = self.db.prepare_v2( + "INSERT INTO ps_kv(key, value) +VALUES(?1, ?2) +ON CONFLICT(key) DO UPDATE SET value = max(CAST(value AS INTEGER), excluded.value)", + )?; + stmt.bind_text( + 1, + LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY, + sqlite::Destructor::STATIC, + )?; + stmt.bind_int64(2, request_id.unwrap_or(0))?; + stmt.exec()?; + Ok(()) + } + fn read_i64_kv(&self, key: &'static str) -> Result, PowerSyncError> { let statement = self .db diff --git a/crates/core/src/sync/streaming_sync.rs b/crates/core/src/sync/streaming_sync.rs index 8ea26b80..116d92e7 100644 --- a/crates/core/src/sync/streaming_sync.rs +++ b/crates/core/src/sync/streaming_sync.rs @@ -95,21 +95,24 @@ impl SyncClient { SyncControlRequest::SyncEvent(sync_event) => { let mut active = ActiveEvent::new(sync_event); - let ClientState::IterationActive(handle) = &mut self.state else { - return Err(PowerSyncError::state_error("No iteration is active")); + let run_result = { + let ClientState::IterationActive(handle) = &mut self.state else { + return Err(PowerSyncError::state_error("No iteration is active")); + }; + + handle.run(&mut active) }; - match handle.run(&mut active) { + match run_result { Err(e) => { self.state = ClientState::Idle; return Err(e); } - Ok(done) => { - if done { - self.state = ClientState::Idle; - } + Ok(true) => { + self.state = ClientState::Idle; } - }; + Ok(false) => {} + } if let Some(recoverable) = active.recoverable_error.take() { Err(recoverable) @@ -117,6 +120,19 @@ impl SyncClient { Ok(active.instructions) } } + SyncControlRequest::NextCheckpointRequestId => { + if !self.has_sync_iteration() { + return Err(PowerSyncError::state_error("No iteration is active")); + } + if !self.adapter.has_checkpoint_request_id()? { + return Err(PowerSyncError::state_error( + "Checkpoint request state has not been seeded", + )); + } + + let request_id = self.adapter.next_checkpoint_request_id()?; + Ok(alloc::vec![Instruction::CheckpointRequestId { request_id }]) + } SyncControlRequest::StopSyncStream => self.state.tear_down(), } } @@ -200,18 +216,18 @@ impl SyncIterationHandle { }; let mut context = Context::from_waker(&waker); - Ok( - if let Poll::Ready(result) = self.future.poll(&mut context) { - let close = result?; + let done = if let Poll::Ready(result) = self.future.poll(&mut context) { + let close = result?; - active - .instructions - .push(Instruction::CloseSyncStream(close)); - true - } else { - false - }, - ) + active + .instructions + .push(Instruction::CloseSyncStream(close)); + true + } else { + false + }; + + Ok(done) } } @@ -557,6 +573,10 @@ impl StreamingSyncIteration { .update(|s| s.disconnect(), &mut event.instructions); break false; } + SyncEvent::SeedCheckpointRequestId { request_id } => { + self.adapter.seed_checkpoint_request_id(request_id)?; + continue; + } SyncEvent::TextLine { data } => SyncLineWithSource::from_text(data)?, SyncEvent::BinaryLine { data } => SyncLineWithSource::from_binary(data)?, SyncEvent::UploadFinished => { @@ -897,11 +917,11 @@ impl StreamingSyncIteration { /// subscriptions, used to associate [BucketSubscriptionReason::DerivedFromExplicitSubscription]. async fn prepare_request(&mut self) -> Result { let event = Self::receive_event().await; - let SyncEvent::Initialize = event.event else { + if !matches!(&event.event, SyncEvent::Initialize) { return Err(PowerSyncError::argument_error( "first event must initialize", )); - }; + } let offline_state = self.adapter.offline_sync_state()?; self.status.update( @@ -933,9 +953,10 @@ impl StreamingSyncIteration { app_metadata: self.options.app_metadata.take(), }; - event - .instructions - .push(Instruction::EstablishSyncStream { request }); + event.instructions.push(Instruction::EstablishSyncStream { + request, + last_checkpoint_request_id: self.adapter.last_checkpoint_request_id()?, + }); Ok(BeforeCheckpoint { local_buckets: local_bucket_names, stream_subscriptions: stream_subscriptions, diff --git a/dart/benchmark/apply_lines.dart b/dart/benchmark/apply_lines.dart index ee0654b5..c06d361a 100644 --- a/dart/benchmark/apply_lines.dart +++ b/dart/benchmark/apply_lines.dart @@ -17,8 +17,10 @@ void main(List args) { final db = openTestDatabase(); db + ..execute('BEGIN') ..execute('select powersync_init()') - ..execute('select powersync_control(?, null)', ['start']); + ..execute('select powersync_control(?, null)', ['start']) + ..execute('COMMIT'); final stopwatch = Stopwatch()..start(); diff --git a/dart/test/error_test.dart b/dart/test/error_test.dart index 74da34a6..305a91f4 100644 --- a/dart/test/error_test.dart +++ b/dart/test/error_test.dart @@ -78,6 +78,7 @@ void main() { final control = db.prepare(stmt); control.execute(['start', null]); + control.execute(['seed_checkpoint_request_id', null]); expect( () => control.execute(['line_text', 'invalid sync line']), throwsA(isSqliteException( diff --git a/dart/test/goldens/simple_iteration.json b/dart/test/goldens/simple_iteration.json index d3bca3fa..c6c3ac21 100644 --- a/dart/test/goldens/simple_iteration.json +++ b/dart/test/goldens/simple_iteration.json @@ -28,11 +28,17 @@ "include_defaults": true, "subscriptions": [] } - } + }, + "last_checkpoint_request_id": null } } ] }, + { + "operation": "seed_checkpoint_request_id", + "data": null, + "output": [] + }, { "operation": "line_text", "data": { diff --git a/dart/test/goldens/starting_stream.json b/dart/test/goldens/starting_stream.json index c5573003..bc9b1436 100644 --- a/dart/test/goldens/starting_stream.json +++ b/dart/test/goldens/starting_stream.json @@ -34,9 +34,15 @@ "include_defaults": true, "subscriptions": [] } - } + }, + "last_checkpoint_request_id": null } } ] + }, + { + "operation": "seed_checkpoint_request_id", + "data": null, + "output": [] } ] diff --git a/dart/test/migration_test.dart b/dart/test/migration_test.dart index 08ea1536..5c3256d5 100644 --- a/dart/test/migration_test.dart +++ b/dart/test/migration_test.dart @@ -301,6 +301,7 @@ INSERT INTO ps_kv(key, value) VALUES db.execute('begin'); control('start'); + control('seed_checkpoint_request_id'); control( 'line_text', json.encode(checkpoint(lastOpId: 3, buckets: [ diff --git a/dart/test/sync_local_performance_test.dart b/dart/test/sync_local_performance_test.dart index 92ba1c2b..81546158 100644 --- a/dart/test/sync_local_performance_test.dart +++ b/dart/test/sync_local_performance_test.dart @@ -100,6 +100,7 @@ COMMIT; // Start a fake sync client to apply the changes we've already written to // ps_oplog control('start'); + control('seed_checkpoint_request_id'); final lastOpid = db.select('select max(op_id) from ps_oplog').single.columnAt(0) as int; final allBuckets = db diff --git a/dart/test/sync_stream_test.dart b/dart/test/sync_stream_test.dart index e5ba8ac1..21c02561 100644 --- a/dart/test/sync_stream_test.dart +++ b/dart/test/sync_stream_test.dart @@ -32,7 +32,7 @@ void main() { ['client_id', 'test-test-test-test']); }); - List control(String operation, Object? data) { + List controlRaw(String operation, Object? data) { db.execute('begin'); ResultSet result; @@ -60,6 +60,23 @@ void main() { } } + bool establishesSyncStream(List instructions) { + return instructions.any((instruction) => + instruction is Map && + instruction.containsKey('EstablishSyncStream')); + } + + List control(String operation, Object? data) { + final result = controlRaw(operation, data); + if (operation == 'start' && establishesSyncStream(result)) { + return [ + ...result, + ...controlRaw('seed_checkpoint_request_id', null), + ]; + } + return result; + } + group('default streams', () { syncTest('are created on-demand', (_) { control('start', null); diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index d17a10ab..34c1d5a4 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -63,13 +63,29 @@ void _syncTests({ return jsonDecode(row.columnAt(0)); } + bool establishesSyncStream(List instructions) { + return instructions.any((instruction) => + instruction is Map && + instruction.containsKey('EstablishSyncStream')); + } + List invokeControl(String operation, Object? data) { + List result; if (matcher.enabled) { // Trace through golden matcher - return matcher.invoke(operation, data); + result = matcher.invoke(operation, data); } else { - return invokeControlRaw(operation, data); + result = invokeControlRaw(operation, data); } + + if (operation == 'start' && establishesSyncStream(result)) { + final seedResult = matcher.enabled + ? matcher.invoke('seed_checkpoint_request_id', null) + : invokeControlRaw('seed_checkpoint_request_id', null); + return [...result, ...seedResult]; + } + + return result; } setUp(() async { @@ -141,24 +157,23 @@ void _syncTests({ return rows.isEmpty ? null : rows.single.columnAt(0); } - int nextCheckpointRequestId() { - db.execute('begin'); + Object? streamLastCheckpointRequestId(List instructions) { + final instruction = instructions.whereType().firstWhere( + (instruction) => instruction.containsKey('EstablishSyncStream')); + final establish = instruction['EstablishSyncStream'] as Map; + return establish['last_checkpoint_request_id']; + } - try { - final [row] = db.select('SELECT powersync_next_checkpoint_request_id()'); - db.execute('commit'); - return row.columnAt(0) as int; - } catch (e) { - db.execute('rollback'); - rethrow; - } + int nextCheckpointRequestId() { + final [instruction] = invokeControl('next_checkpoint_request_id', null); + final data = (instruction as Map)['CheckpointRequestId'] as Map; + return data['request_id'] as int; } Object? probeLocalTargetOp([Object? opId]) { - final [row] = db.select('SELECT powersync_probe_local_target_op(?)', [ - opId, - ]); - return row.columnAt(0); + final [instruction] = invokeControl('local_target_op', opId); + final data = (instruction as Map)['LocalTargetOp'] as Map; + return data['target_op']; } ResultSet fetchRows() { @@ -206,12 +221,15 @@ void _syncTests({ }); syncTest('app_metadata is passed to EstablishSyncStream request', (_) { - final startInstructions = invokeControlRaw( - 'start', - json.encode({ - 'app_metadata': {'key1': 'value1', 'key2': 'value2'} - }), - ); + final startInstructions = [ + ...invokeControlRaw( + 'start', + json.encode({ + 'app_metadata': {'key1': 'value1', 'key2': 'value2'} + }), + ), + ...invokeControlRaw('seed_checkpoint_request_id', null), + ]; expect( startInstructions, @@ -401,6 +419,8 @@ void _syncTests({ }); syncTest('allocates requested checkpoint request ids', (_) { + invokeControl('start', null); + expect(nextCheckpointRequestId(), 1); expect(lastRequestedCheckpointRequestId(), 1); @@ -408,16 +428,48 @@ void _syncTests({ expect(lastRequestedCheckpointRequestId(), 2); }); - syncTest('probes and updates local target op with checkpoint request id', (_) { + syncTest('seeds requested checkpoint request ids from service state', (_) { + final startInstructions = invokeControlRaw('start', null); + expect(streamLastCheckpointRequestId(startInstructions), isNull); + invokeControlRaw('seed_checkpoint_request_id', 41); + + expect(nextCheckpointRequestId(), 42); + expect(lastRequestedCheckpointRequestId(), 42); + + final restartInstructions = invokeControlRaw('start', null); + expect( + restartInstructions, + contains(containsPair('EstablishSyncStream', anything)), + ); + expect(streamLastCheckpointRequestId(restartInstructions), 42); + expect(invokeControlRaw('seed_checkpoint_request_id', 100), isEmpty); + + expect(nextCheckpointRequestId(), 101); + expect(lastRequestedCheckpointRequestId(), 101); + }); + + syncTest('requires checkpoint request state before allocating checkpoint ids', (_) { + invokeControlRaw('start', null); + + expect( + () => invokeControlRaw('next_checkpoint_request_id', null), + throwsA(isSqliteException( + 21, + contains('Checkpoint request state has not been seeded'), + )), + ); + expect(probeLocalTargetOp(), isNull); + }); + + syncTest('probes and updates local target op without sync iteration', (_) { expect(probeLocalTargetOp(), isNull); expect(probeLocalTargetOp(1), isNull); expect(lastRequestedCheckpointRequestId(), 1); expect(probeLocalTargetOp(), 1); + expect(probeLocalTargetOp(2), 1); expect(lastRequestedCheckpointRequestId(), 2); expect(probeLocalTargetOp(), 2); - - invokeControl('start', null); }); syncTest('accepts text checkpoint request ids for local target op', (_) { diff --git a/docs/sync.md b/docs/sync.md index e9595d3e..74143cab 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -10,7 +10,7 @@ The function should always be called in a transaction. The following commands are supported: 1. `start`: Payload is a JSON-encoded object. This requests the client to start a sync iteration. - The payload can either be `null` or an JSON object with: + The payload can either be `null` or a JSON object with: - An optional `parameters: Record` entry, specifying parameters to include in the request to the sync service. - A `schema: { tables: Table[], raw_tables: RawTable[] }` entry specifying the schema of the database to @@ -26,36 +26,46 @@ The following commands are supported: - The client will emit an instruction to stop the current stream, clients should restart by sending another `start` command. 6. `completed_upload`: Notify the sync implementation that all local changes have been uploaded. -7. `update_subscriptions`: Notify the sync implementation that subscriptions which are currently active in the app - have changed. Depending on the TTL of caches, this may cause it to request a reconnect. +7. `update_subscriptions`: Payload is a JSON-encoded array of + `{name: string, params: Record}`. Notify the sync implementation that subscriptions + which are currently active in the app have changed. Depending on the TTL of caches, this may + cause it to request a reconnect. 8. `connection`: Notify the sync implementation about the connection being opened (second parameter should be `established`) or the HTTP stream closing (second parameter should be `end`). This is used to set `connected` to true in the sync status without waiting for the first sync line. -9. `subscriptions`: Store a new sync steam subscription in the database or remove it. +9. `subscriptions`: Store a new sync stream subscription in the database or remove it. This command can run outside of a sync iteration and does not affect it. -10. `update_subscriptions`: Second parameter is a JSON-encoded array of `{name: string, params: Record}`. - If a new subscription is created, or when a subscription without a TTL has been removed, the client will ask to - restart the connection. +10. `next_checkpoint_request_id`: No payload. During an active sync iteration after checkpoint + request state exists locally, allocates and returns the next checkpoint request id in a + `CheckpointRequestId` instruction. +11. `local_target_op`: Payload is `null`, an integer, or an integer string. Probes, updates or + clears the local target op and returns the previously-observed value in a `LocalTargetOp` + result. This command can run outside of a sync iteration and does not affect it. +12. `seed_checkpoint_request_id`: Payload is `null`, an integer, or an integer string. After + receiving `EstablishSyncStream`, SDKs can re-request the client's last checkpoint request state + from the service using the provided hint, then seed core with the actual response for + reconciliation. `NULL` means the service has no record for the client yet; core stores `0` only + when no local seed exists. Integer seeds use `max(local, service)` semantics so the local counter + never moves backwards. When uploads request a write checkpoint, SDKs should call -`powersync_next_checkpoint_request_id()` inside a transaction to allocate the id to pass to the -request-checkpoint API. In checkpoint-request mode, the SDK should first allocate the id, then post -that id to the service, and then call `powersync_probe_local_target_op(id)` with the same id once -the service accepts the request. This sets the local target op to the request op, replacing the -pending-write sentinel with the concrete checkpoint request id that the sync stream can satisfy. -`powersync_next_checkpoint_request_id()` only advances the request counter; it does not update the +`powersync_control('next_checkpoint_request_id', NULL)` inside a transaction to allocate the id to +pass to the request-checkpoint API. In checkpoint-request mode, the SDK should first allocate the id, +then post that id to the service, and then call `powersync_control('local_target_op', id)` with the +same id once the service accepts the request. This sets the local target op to the request op, +replacing the pending-write sentinel with the concrete checkpoint request id that the sync stream +can satisfy. `next_checkpoint_request_id` only advances the request counter; it does not update the local target op used to block applying downloaded rows. -`powersync_probe_local_target_op(op_id)` reads and optionally updates the internal local target op. -The same function is used for compatibility when a new SDK is used with an older PowerSync service -that does not yet support client-created checkpoint requests; after the service-side write -checkpoint request returns a concrete id, call `powersync_probe_local_target_op(id)` with that id. -Pass `NULL` to probe the current internal `$local` target op from `ps_kv` without updating it, or -pass an integer or integer string to update that target op. In both cases it returns the value from -before the call, or `NULL` if no value existed. Updating to a positive, non-sentinel target op also -stores it as `last_requested_checkpoint_request_id` to support migrating to client-created -checkpoint requests. Passing `0` clears the local target, and sentinel values such as max op id are -not stored as requested checkpoint ids. +`powersync_control('local_target_op', op_id)` probes and optionally updates the internal local +target op. The same command is used for compatibility when a new SDK is used with an older +PowerSync service that does not yet support client-created checkpoint requests; after the +service-side write checkpoint request returns a concrete id, call +`powersync_control('local_target_op', id)` with that id. Passing `NULL` returns the current target +without changing it, and passing `0` clears the local target. Updating to a positive, non-sentinel +target op also stores it as `last_requested_checkpoint_request_id` to support migrating to +client-created checkpoint requests. Sentinel values such as max op id are not stored as requested +checkpoint ids. Database migration v14 moves legacy `$local` checkpoint state into `ps_kv`: `$local.last_applied_op` becomes `last_applied_checkpoint_request_id`, `$local.last_op` becomes the internal @@ -74,7 +84,7 @@ id to wait for yet. The max-op sentinel may also cover earlier pending uploads t associated with legacy service-created write checkpoints, so restarting the client-created counter from `1` could create a target lower than those existing associations. In that state, create one old-style write checkpoint first, store the returned concrete id with -`powersync_probe_local_target_op(id)`, and then switch to client-created checkpoint requests. +`powersync_control('local_target_op', id)`, and then switch to client-created checkpoint requests. `powersync_control` returns a JSON-encoded array of instructions for the client: @@ -83,6 +93,8 @@ type Instruction = { LogLine: LogLine } | { UpdateSyncStatus: UpdateSyncStatus } | { EstablishSyncStream: EstablishSyncStream } | { FetchCredentials: FetchCredentials } + | { CheckpointRequestId: { request_id: number } } + | { LocalTargetOp: { target_op: null | number } } // Close a connection previously started after EstablishSyncStream | { CloseSyncStream: { hide_disconnect: boolean } } // For the Dart web client, flush the (otherwise non-durable) file system. @@ -97,8 +109,14 @@ interface LogLine { } // Instructs client SDKs to open a connection to the sync service. +// last_checkpoint_request_id is core's local counter value before this stream request. On connect, +// SDKs can use it to re-request this client's last checkpoint request state from the service, then +// call powersync_control('seed_checkpoint_request_id', value) with the actual response for +// reconciliation. `value` may be null when the service has no checkpoint request state for this +// client. interface EstablishSyncStream { request: any // The JSON-encoded StreamingSyncRequest to send to the sync service + last_checkpoint_request_id: null | number } // Instructs SDKS to update the downloading state of their SyncStatus. diff --git a/docs/write-checkpoint-requests.md b/docs/write-checkpoint-requests.md index a03d9356..7edf54c4 100644 --- a/docs/write-checkpoint-requests.md +++ b/docs/write-checkpoint-requests.md @@ -13,10 +13,11 @@ At a high level: the client. SDKs should not write these keys directly. They update the local target through -`powersync_probe_local_target_op()`, which is the shared helper for both legacy write checkpoints -and new client-created checkpoint requests. The newer `powersync_next_checkpoint_request_id()` -function only allocates a checkpoint request id; after the service accepts that request, the SDK -uses `powersync_probe_local_target_op(id)` to make the accepted id the local target. +`powersync_control('local_target_op', value)`, which is the shared helper for both legacy write +checkpoints and new client-created checkpoint requests. The +`powersync_control('next_checkpoint_request_id', NULL)` command only allocates a checkpoint request +id; after the service accepts that request, the SDK uses +`powersync_control('local_target_op', id)` to make the accepted id the local target. For the historic `$local` bucket flow, see `historic-write-checkpoints.md`. @@ -47,9 +48,9 @@ transaction { deleteUploadedCrud(upTo: lastUploadedId) if let customCheckpoint, crudQueueIsEmpty { - powersync_probe_local_target_op(customCheckpoint) + powersync_control('local_target_op', customCheckpoint) } else { - powersync_probe_local_target_op(MAX_OP_ID) + powersync_control('local_target_op', MAX_OP_ID) } } ``` @@ -57,21 +58,25 @@ transaction { ## Updating the local target Once uploads are complete, the sync client updates the local target through -`powersync_probe_local_target_op()`. It only does this when the current target is still +`powersync_control('local_target_op', value)`. It only does this when the current target is still `MAX_OP_ID`, which avoids overwriting a custom checkpoint that was already stored by `complete(writeCheckpoint:)`. The SDK implementation: -1. Probes the current target with `powersync_probe_local_target_op(NULL)`. +1. Probes the current target with `powersync_control('local_target_op', NULL)`. 2. Reads `sqlite_sequence.seq` for `ps_crud`. 3. Gets a concrete checkpoint id from either the new or legacy service API. 4. Re-enters a write transaction. 5. Verifies that `ps_crud` is still empty and that its sequence did not change. -6. Stores the concrete target with `powersync_probe_local_target_op(opId)`. +6. Stores the concrete target with `powersync_control('local_target_op', opId)`. ```text -if powersync_probe_local_target_op(NULL) == MAX_OP_ID { +let previousTarget = transaction { + powersync_control('local_target_op', NULL).LocalTargetOp.target_op +} + +if previousTarget == MAX_OP_ID { let seqBefore = psCrudSequence() let checkpointId = await createOrFetchCheckpointId() @@ -80,7 +85,7 @@ if powersync_probe_local_target_op(NULL) == MAX_OP_ID { return } - powersync_probe_local_target_op(checkpointId) + powersync_control('local_target_op', checkpointId) } } ``` @@ -88,11 +93,11 @@ if powersync_probe_local_target_op(NULL) == MAX_OP_ID { In checkpoint-request mode, `getWriteCheckpoint()` calls `requestCheckpoint()`. That allocates an id locally, sends it to `/sync/checkpoint-request`, and returns the same id once the service accepts the request. Only then does the upload path store that id as `local_target_op` with -`powersync_probe_local_target_op(id)`. +`powersync_control('local_target_op', id)`. ```text let requestId = transaction { - powersync_next_checkpoint_request_id() + powersync_control('next_checkpoint_request_id', NULL).CheckpointRequestId.request_id } POST /sync/checkpoint-request { @@ -104,15 +109,26 @@ return requestId ``` The legacy fallback still calls `/write-checkpoint2.json`; the returned write checkpoint is stored -through the same `powersync_probe_local_target_op(opId)` helper. This keeps SDK target updates +through the same `powersync_control('local_target_op', opId)` helper. This keeps SDK target updates consistent across both protocols. -## Helper functions +## Sync control commands + +These `powersync_control` commands are the SDK-facing API for the new `ps_kv` checkpoint state. -These SQL functions are the SDK-facing API for the new `ps_kv` checkpoint state. +`powersync_control('start', payload)` begins a sync iteration and emits `EstablishSyncStream` with a +`last_checkpoint_request_id` hint. This is core's local `last_requested_checkpoint_request_id` value +before opening the stream, or `NULL` when no local seed exists. On connect, SDKs can use this hint +to re-request the client's last checkpoint request state from the service, then call +`powersync_control('seed_checkpoint_request_id', value)` with the actual response for +reconciliation. `value` may be `NULL` when the service has no record for the client; core stores `0` +only when no local seed exists. Integer seeds use `max(local, service)` semantics so the local +counter never moves backwards. SDKs may also refresh service state when their user/client context +changes. -`powersync_next_checkpoint_request_id()` must be called inside a transaction. It increments and -returns `last_requested_checkpoint_request_id` in `ps_kv`. +`powersync_control('next_checkpoint_request_id', NULL)` must be called inside a transaction during +an active sync iteration after `last_requested_checkpoint_request_id` exists locally. It increments +and returns `last_requested_checkpoint_request_id` in a `CheckpointRequestId` instruction. ```sql INSERT INTO ps_kv(key, value) @@ -121,16 +137,18 @@ ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + 1 RETURNING value; ``` -This function only allocates an id. It does not update `local_target_op`. +This command only allocates an id. It does not update `local_target_op`. Note on sequences: SQLite does not have standalone sequences. The sequence-like alternatives are either an `AUTOINCREMENT` table backed by SQLite's internal `sqlite_sequence`, or a dedicated single-row counter table like the existing `ps_tx` transaction counter. The checkpoint request counter currently lives in `ps_kv` because it is also migrated and seeded from legacy/custom -concrete targets via `powersync_probe_local_target_op()`. If we want stricter structure later, a -dedicated checkpoint-request counter table would be the closest match to a sequence. +concrete targets via `powersync_control('local_target_op', value)`. If we want stricter structure +later, a dedicated checkpoint-request counter table would be the closest match to a sequence. -`powersync_probe_local_target_op(op_id)` reads and optionally updates the local target: +`powersync_control('local_target_op', op_id)` probes and optionally updates the local target. Like +`subscriptions`, this command is handled directly by `powersync_control` and can run outside an +active sync iteration: - `NULL` returns the current `local_target_op` without changing it. - `0` clears `local_target_op`. @@ -138,7 +156,8 @@ dedicated checkpoint-request counter table would be the closest match to a seque - A positive value other than `i64::MAX` also stores `last_requested_checkpoint_request_id`. - Negative values and non-integer inputs are rejected. -The function returns the previous target value, or `NULL` if there was no target. +The command returns the previous target value in a `LocalTargetOp` result, or `NULL` if there was no +target. ```text previous = ps_kv['local_target_op'] @@ -228,9 +247,9 @@ request could not be delivered to the service or observed in the sync stream. pending, a concrete checkpoint request id after upload completion, or absent when there is no local write gate. - `last_requested_checkpoint_request_id`: The last client-created checkpoint request id allocated - by `powersync_next_checkpoint_request_id()`. `powersync_probe_local_target_op()` also writes this - key for positive, non-sentinel targets so migrated or legacy-created concrete targets can seed the - client request counter. + by `powersync_control('next_checkpoint_request_id', NULL)`. + `powersync_control('local_target_op', value)` also writes this key for positive, non-sentinel + targets so migrated or legacy-created concrete targets can seed the client request counter. - `last_seen_checkpoint_request_id`: The latest full checkpoint `write_checkpoint` observed and validated from the sync stream. - `last_applied_checkpoint_request_id`: The latest full checkpoint `write_checkpoint` that has been @@ -256,7 +275,7 @@ the gate; it does not prove that no earlier uploads were already associated with service-created write checkpoints. Those existing checkpoint ids may be higher than a restarted client counter such as `1`, and using a lower target could let an older seen checkpoint satisfy the gate too early. In that state, the SDK should create one legacy write checkpoint first, store the -concrete id with `powersync_probe_local_target_op(id)`, and then switch to client-created +concrete id with `powersync_control('local_target_op', id)`, and then switch to client-created checkpoint requests. The down migration rebuilds a `$local` row only when `local_target_op` exists, using: From 0e11137723fd11b887226fb98f653a9b2b0f8088 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 2 Jul 2026 13:33:43 +0200 Subject: [PATCH 06/29] drop target_op column in migrations --- crates/core/src/migrations.rs | 43 +++++++++++++++++++++++++ dart/test/migration_test.dart | 13 +++++++- dart/test/utils/fix_035_fixtures.dart | 12 +++---- dart/test/utils/migration_fixtures.dart | 16 +++++++-- docs/schema.md | 20 +++++++----- docs/sync.md | 8 +++-- docs/write-checkpoint-requests.md | 7 +++- 7 files changed, 97 insertions(+), 22 deletions(-) diff --git a/crates/core/src/migrations.rs b/crates/core/src/migrations.rs index 04996084..6d6bbfd4 100644 --- a/crates/core/src/migrations.rs +++ b/crates/core/src/migrations.rs @@ -504,6 +504,8 @@ SELECT 'local_target_op', target_op FROM ps_buckets WHERE name = '$local' AND target_op > 0; + +ALTER TABLE ps_buckets DROP COLUMN target_op; "; local_db.exec_safe(up).into_db_result(local_db)?; @@ -519,6 +521,47 @@ SELECT 'local_target_op', target_op // implementation also didn't have a `$local` bucket unless there was local target state to // track. const DOWN_STATEMENTS: &[&str] = &[ + "ALTER TABLE ps_buckets RENAME TO ps_buckets_14", + "DROP INDEX ps_buckets_name", + "CREATE TABLE ps_buckets( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + last_applied_op INTEGER NOT NULL DEFAULT 0, + last_op INTEGER NOT NULL DEFAULT 0, + target_op INTEGER NOT NULL DEFAULT 0, + add_checksum INTEGER NOT NULL DEFAULT 0, + op_checksum INTEGER NOT NULL DEFAULT 0, + pending_delete INTEGER NOT NULL DEFAULT 0 +) STRICT", + "CREATE UNIQUE INDEX ps_buckets_name ON ps_buckets (name)", + "ALTER TABLE ps_buckets ADD COLUMN count_at_last INTEGER NOT NULL DEFAULT 0", + "ALTER TABLE ps_buckets ADD COLUMN count_since_last INTEGER NOT NULL DEFAULT 0", + "ALTER TABLE ps_buckets ADD COLUMN downloaded_size INTEGER NOT NULL DEFAULT 0", + "INSERT INTO ps_buckets( + id, + name, + last_applied_op, + last_op, + add_checksum, + op_checksum, + pending_delete, + count_at_last, + count_since_last, + downloaded_size +) +SELECT + id, + name, + last_applied_op, + last_op, + add_checksum, + op_checksum, + pending_delete, + count_at_last, + count_since_last, + downloaded_size +FROM ps_buckets_14", + "DROP TABLE ps_buckets_14", "INSERT INTO ps_buckets(name, pending_delete, last_op, last_applied_op, target_op) SELECT '$local', 1, seen, applied, target FROM ( diff --git a/dart/test/migration_test.dart b/dart/test/migration_test.dart index 5c3256d5..771ccd77 100644 --- a/dart/test/migration_test.dart +++ b/dart/test/migration_test.dart @@ -112,6 +112,17 @@ VALUES(1, '$local', 5, 6, 7, 0, 0, 1, 0, 0, 0); {'key': 'last_applied_checkpoint_request_id', 'value': 5}, {'key': 'local_target_op', 'value': 7}, ])); + expect( + db + .select("PRAGMA table_info('ps_buckets')") + .map((row) => row['name']), + isNot(contains('target_op')), + ); + expect( + () => db.execute( + r"UPDATE ps_buckets SET target_op = 8 WHERE name = '$local'"), + throwsA(isA()), + ); }); test('does not migrate last applied op as requested checkpoint id', @@ -130,8 +141,8 @@ VALUES(1, '$local', 5, 6, 9223372036854775807, 0, 0, 1, 0, 0, 0); // checkpoint counter. The sentinel target is preserved for blocking, but is not concrete // enough to become last_requested_checkpoint_request_id. expect(db.select('SELECT key, value FROM ps_kv ORDER BY key'), [ - {'key': 'last_seen_checkpoint_request_id', 'value': 6}, {'key': 'last_applied_checkpoint_request_id', 'value': 5}, + {'key': 'last_seen_checkpoint_request_id', 'value': 6}, {'key': 'local_target_op', 'value': 9223372036854775807}, ]); }); diff --git a/dart/test/utils/fix_035_fixtures.dart b/dart/test/utils/fix_035_fixtures.dart index fa912e89..0b600a90 100644 --- a/dart/test/utils/fix_035_fixtures.dart +++ b/dart/test/utils/fix_035_fixtures.dart @@ -18,9 +18,9 @@ const dataBroken = ''' /// Data after applying the migration fix, but before sync_local const dataMigrated = ''' -;INSERT INTO ps_buckets(id, name, last_applied_op, last_op, target_op, add_checksum, op_checksum, pending_delete, count_at_last, count_since_last, downloaded_size) VALUES - (1, 'b1', 0, 0, 0, 0, 120, 0, 0, 0, 0), - (2, 'b2', 0, 0, 0, 0, 3, 0, 0, 0, 0) +;INSERT INTO ps_buckets(id, name, last_applied_op, last_op, add_checksum, op_checksum, pending_delete, count_at_last, count_since_last, downloaded_size) VALUES + (1, 'b1', 0, 0, 0, 120, 0, 0, 0, 0), + (2, 'b2', 0, 0, 0, 3, 0, 0, 0, 0) ;INSERT INTO ps_oplog(bucket, op_id, row_type, row_id, key, data, hash) VALUES (1, 1, 'todos', 't1', '', '{}', 100), (1, 2, 'todos', 't2', '', '{}', 20), @@ -39,9 +39,9 @@ const dataMigrated = ''' /// Data after applying the migration fix and sync_local const dataFixed = ''' -;INSERT INTO ps_buckets(id, name, last_applied_op, last_op, target_op, add_checksum, op_checksum, pending_delete, count_at_last, count_since_last, downloaded_size) VALUES - (1, 'b1', 3, 3, 0, 0, 120, 0, 1, 0, 0), - (2, 'b2', 3, 3, 0, 0, 3, 0, 1, 0, 0) +;INSERT INTO ps_buckets(id, name, last_applied_op, last_op, add_checksum, op_checksum, pending_delete, count_at_last, count_since_last, downloaded_size) VALUES + (1, 'b1', 3, 3, 0, 120, 0, 1, 0, 0), + (2, 'b2', 3, 3, 0, 3, 0, 1, 0, 0) ;INSERT INTO ps_oplog(bucket, op_id, row_type, row_id, key, data, hash) VALUES (1, 1, 'todos', 't1', '', '{}', 100), (1, 2, 'todos', 't2', '', '{}', 20), diff --git a/dart/test/utils/migration_fixtures.dart b/dart/test/utils/migration_fixtures.dart index 555d4681..7f6e6ba8 100644 --- a/dart/test/utils/migration_fixtures.dart +++ b/dart/test/utils/migration_fixtures.dart @@ -541,8 +541,8 @@ Map _expectedState() { ;INSERT INTO ps_migration(id, down_migrations) VALUES(12, '[{"sql":"ALTER TABLE ps_buckets DROP COLUMN downloaded_size"},{"sql":"DELETE FROM ps_migration WHERE id >= 12"}]') ;INSERT INTO ps_migration(id, down_migrations) VALUES(13, '[{"sql":"UPDATE ps_stream_subscriptions SET expires_at = expires_at / 1000000, last_synced_at = last_synced_at / 1000000"},{"sql":"ALTER TABLE ps_sync_state RENAME TO ps_sync_state_new"},{"sql":"CREATE TABLE ps_sync_state (\n priority INTEGER NOT NULL PRIMARY KEY,\n last_synced_at TEXT NOT NULL\n) STRICT;"},{"sql":"INSERT INTO ps_sync_state (priority, last_synced_at) SELECT priority, datetime(last_synced_at / 1000000, ''unixepoch'') FROM ps_sync_state_new"},{"sql":"DROP TABLE ps_sync_state_new"},{"sql":"DELETE FROM ps_migration WHERE id >= 13"}]')''', }; - state[14] = '''${state[13]!.trim()} -;INSERT INTO ps_migration(id, down_migrations) VALUES(14, '[{"sql":"INSERT INTO ps_buckets(name, pending_delete, last_op, last_applied_op, target_op)\\nSELECT ''\$local'', 1, seen, applied, target\\n FROM (\\n SELECT\\n IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''last_seen_checkpoint_request_id''), 0) AS seen,\\n IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''last_applied_checkpoint_request_id''), 0) AS applied,\\n (SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''local_target_op'') AS target\\n )\\n WHERE EXISTS (\\n SELECT 1 FROM ps_kv WHERE key = ''local_target_op''\\n )\\nON CONFLICT(name) DO UPDATE SET\\n pending_delete = excluded.pending_delete,\\n last_op = excluded.last_op,\\n last_applied_op = excluded.last_applied_op,\\n target_op = excluded.target_op"},{"sql":"DELETE FROM ps_migration WHERE id >= 14"}]')'''; + state[14] = '''${state[13]!.trim().replaceFirst(' target_op INTEGER NOT NULL DEFAULT 0,\n', '')} +;INSERT INTO ps_migration(id, down_migrations) VALUES(14, '[{"sql":"ALTER TABLE ps_buckets RENAME TO ps_buckets_14"},{"sql":"DROP INDEX ps_buckets_name"},{"sql":"CREATE TABLE ps_buckets(\\n id INTEGER PRIMARY KEY,\\n name TEXT NOT NULL,\\n last_applied_op INTEGER NOT NULL DEFAULT 0,\\n last_op INTEGER NOT NULL DEFAULT 0,\\n target_op INTEGER NOT NULL DEFAULT 0,\\n add_checksum INTEGER NOT NULL DEFAULT 0,\\n op_checksum INTEGER NOT NULL DEFAULT 0,\\n pending_delete INTEGER NOT NULL DEFAULT 0\\n) STRICT"},{"sql":"CREATE UNIQUE INDEX ps_buckets_name ON ps_buckets (name)"},{"sql":"ALTER TABLE ps_buckets ADD COLUMN count_at_last INTEGER NOT NULL DEFAULT 0"},{"sql":"ALTER TABLE ps_buckets ADD COLUMN count_since_last INTEGER NOT NULL DEFAULT 0"},{"sql":"ALTER TABLE ps_buckets ADD COLUMN downloaded_size INTEGER NOT NULL DEFAULT 0"},{"sql":"INSERT INTO ps_buckets(\\n id,\\n name,\\n last_applied_op,\\n last_op,\\n add_checksum,\\n op_checksum,\\n pending_delete,\\n count_at_last,\\n count_since_last,\\n downloaded_size\\n)\\nSELECT\\n id,\\n name,\\n last_applied_op,\\n last_op,\\n add_checksum,\\n op_checksum,\\n pending_delete,\\n count_at_last,\\n count_since_last,\\n downloaded_size\\nFROM ps_buckets_14"},{"sql":"DROP TABLE ps_buckets_14"},{"sql":"INSERT INTO ps_buckets(name, pending_delete, last_op, last_applied_op, target_op)\\nSELECT ''\$local'', 1, seen, applied, target\\n FROM (\\n SELECT\\n IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''last_seen_checkpoint_request_id''), 0) AS seen,\\n IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''last_applied_checkpoint_request_id''), 0) AS applied,\\n (SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''local_target_op'') AS target\\n )\\n WHERE EXISTS (\\n SELECT 1 FROM ps_kv WHERE key = ''local_target_op''\\n )\\nON CONFLICT(name) DO UPDATE SET\\n pending_delete = excluded.pending_delete,\\n last_op = excluded.last_op,\\n last_applied_op = excluded.last_applied_op,\\n target_op = excluded.target_op"},{"sql":"DELETE FROM ps_migration WHERE id >= 14"}]')'''; return state; } @@ -683,7 +683,17 @@ Map _data1() { ('lists', 'l2') ''' }; - data[14] = data[13]!; + data[14] = r''' +;INSERT INTO ps_buckets(id, name, last_applied_op, last_op, add_checksum, op_checksum, pending_delete, count_at_last, count_since_last, downloaded_size) VALUES + (1, 'b1', 0, 0, 0, 120, 0, 0, 0, 0), + (2, 'b2', 0, 0, 1005, 3, 0, 0, 0, 0) +;INSERT INTO ps_oplog(bucket, op_id, row_type, row_id, key, data, hash) VALUES + (1, 1, 'todos', 't1', '', '{}', 100), + (1, 2, 'todos', 't2', '', '{}', 20), + (2, 3, 'lists', 'l1', '', '{}', 3) +;INSERT INTO ps_updated_rows(row_type, row_id) VALUES + ('lists', 'l2') +'''; return data; } diff --git a/docs/schema.md b/docs/schema.md index f7dfbde3..1955bab1 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -15,8 +15,8 @@ A bucket is instantiated for every row returned by a parameter query in a [bucke Clients create entries in `ps_buckets` when receiving a checkpoint message from the sync service, they are also responsible for removing buckets that are no longer relevant to the client. -There is also a special `$local` bucket representing pending -uploads. +Older schema versions also used a special `$local` bucket to represent pending uploads. Current +schema versions keep that local write gate in `ps_kv` instead. We store the following information in `ps_buckets`: @@ -24,12 +24,16 @@ We store the following information in `ps_buckets`: 2. `name`: The name of the bucket as received from the sync service. 3. `last_applied_op`: The last operation id that has been verified and published to views (meaning that it was part of a checkpoint and that we have validated its checksum). -4. `target_op`: Only used for `$local`. TODO: Document further. -5. `add_checksum`: TODO: Document further. -6. `op_checksum`: TODO: Document further. -7. `pending_delete`: TODO: Appears to be unused, document further. -8. `count_at_last`: The amount of operations in the bucket at the last verified checkpoint. -9. `count_since_last`: The amount of operations downloaded since the last verified checkpoint. +4. `add_checksum`: TODO: Document further. +5. `op_checksum`: TODO: Document further. +6. `pending_delete`: TODO: Appears to be unused, document further. +7. `count_at_last`: The amount of operations in the bucket at the last verified checkpoint. +8. `count_since_last`: The amount of operations downloaded since the last verified checkpoint. + +Schema version 14 removes the legacy `target_op` column after migrating `$local.target_op` to +`ps_kv.local_target_op`. This makes older SDKs fail with a hard SQLite error if they try to keep +using the migrated database without downgrading. The down migration restores `target_op` for older +schema versions. ## `ps_crud` diff --git a/docs/sync.md b/docs/sync.md index 74143cab..81fe426d 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -70,9 +70,11 @@ checkpoint ids. Database migration v14 moves legacy `$local` checkpoint state into `ps_kv`: `$local.last_applied_op` becomes `last_applied_checkpoint_request_id`, `$local.last_op` becomes the internal `last_seen_checkpoint_request_id`, a concrete `$local.target_op` advances the request counter, and -`$local.target_op` is stored as `local_target_op`. Downgrading restores a `$local` row only when -`local_target_op` exists, so older SDKs can keep using target-op based blocking without inventing a -synthetic local bucket when there was no local target state. +`$local.target_op` is stored as `local_target_op`. The migration then drops `ps_buckets.target_op` +so older SDKs fail hard if they try to keep using the migrated database directly. Downgrading +restores the column, and restores a `$local` row only when `local_target_op` exists, so older SDKs +can keep using target-op based blocking without inventing a synthetic local bucket when there was no +local target state. If `local_target_op` is absent after migration, there is no local write gate waiting for a checkpoint. In that case, SDKs can start client-created checkpoint requests normally, even when diff --git a/docs/write-checkpoint-requests.md b/docs/write-checkpoint-requests.md index 7edf54c4..02d85cff 100644 --- a/docs/write-checkpoint-requests.md +++ b/docs/write-checkpoint-requests.md @@ -264,6 +264,10 @@ Migration v14 moves the old `$local` bucket state into `ps_kv`: - A concrete `$local.target_op` becomes `last_requested_checkpoint_request_id`. - Any positive `$local.target_op`, including `MAX_OP_ID`, becomes `local_target_op`. +After copying this state, the migration drops `ps_buckets.target_op`. This intentionally makes older +SDKs fail with a hard SQLite error if they try to keep using a migrated database without first +downgrading. + An absent `local_target_op` is safe: there is no local write gate waiting for a checkpoint, so an SDK can start client-created checkpoint requests from `1`. The sync stream will only report that request id after the service has accepted and reached it. @@ -278,7 +282,8 @@ gate too early. In that state, the SDK should create one legacy write checkpoint concrete id with `powersync_control('local_target_op', id)`, and then switch to client-created checkpoint requests. -The down migration rebuilds a `$local` row only when `local_target_op` exists, using: +The down migration restores `ps_buckets.target_op` and rebuilds a `$local` row only when +`local_target_op` exists, using: - `last_seen_checkpoint_request_id` as `$local.last_op` - `last_applied_checkpoint_request_id` as `$local.last_applied_op` From 69bf1d3eb8c217925befd28685ff4078ae63b94f Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 2 Jul 2026 14:13:13 +0200 Subject: [PATCH 07/29] use powersync control response instruction for last applied checkpoint request instead of sync status --- crates/core/src/sync/interface.rs | 2 + crates/core/src/sync/storage_adapter.rs | 4 - crates/core/src/sync/streaming_sync.rs | 6 +- crates/core/src/sync/sync_status.rs | 18 +--- dart/test/goldens/simple_iteration.json | 12 +-- dart/test/goldens/starting_stream.json | 3 +- dart/test/sync_test.dart | 126 +++++++++++++----------- docs/sync.md | 8 +- docs/write-checkpoint-requests.md | 17 ++-- 9 files changed, 97 insertions(+), 99 deletions(-) diff --git a/crates/core/src/sync/interface.rs b/crates/core/src/sync/interface.rs index 57e559e6..a673f353 100644 --- a/crates/core/src/sync/interface.rs +++ b/crates/core/src/sync/interface.rs @@ -151,6 +151,8 @@ pub enum Instruction { }, /// Return a newly allocated checkpoint request id to the SDK. CheckpointRequestId { request_id: i64 }, + /// Notify the SDK that a checkpoint request id has been applied locally. + CheckpointRequestApplied { request_id: i64 }, /// Return the local target op value observed before an optional update. LocalTargetOp { target_op: Option }, // These are defined like this because deserializers in Kotlin can't support either an diff --git a/crates/core/src/sync/storage_adapter.rs b/crates/core/src/sync/storage_adapter.rs index 34f0c3b3..d636f95f 100644 --- a/crates/core/src/sync/storage_adapter.rs +++ b/crates/core/src/sync/storage_adapter.rs @@ -121,9 +121,6 @@ impl StorageAdapter { items }; - let last_applied_checkpoint_request_id = - self.read_i64_kv(LAST_APPLIED_CHECKPOINT_REQUEST_ID_KEY)?; - let mut streams = Vec::new(); self.iterate_local_subscriptions(|sub| { streams.push(ActiveStreamSubscription::from_local(&sub)); @@ -135,7 +132,6 @@ impl StorageAdapter { priority_status: priority_items, downloading: None, streams, - last_applied_checkpoint_request_id, }) } diff --git a/crates/core/src/sync/streaming_sync.rs b/crates/core/src/sync/streaming_sync.rs index 116d92e7..a578dcac 100644 --- a/crates/core/src/sync/streaming_sync.rs +++ b/crates/core/src/sync/streaming_sync.rs @@ -970,15 +970,17 @@ impl StreamingSyncIteration { applied_checkpoint_request_id: Option, ) -> Result<(), PowerSyncError> { if let Some(request_id) = applied_checkpoint_request_id { - // Persisted so it can be restored into sync status when initializing a new client. self.adapter .persist_last_applied_checkpoint_request_id(request_id)?; + event + .instructions + .push(Instruction::CheckpointRequestApplied { request_id }); } event.instructions.push(Instruction::DidCompleteSync {}); self.status.update( - |status| status.applied_checkpoint(timestamp, applied_checkpoint_request_id), + |status| status.applied_checkpoint(timestamp), &mut event.instructions, ); diff --git a/crates/core/src/sync/sync_status.rs b/crates/core/src/sync/sync_status.rs index 7dba73f0..6530c460 100644 --- a/crates/core/src/sync/sync_status.rs +++ b/crates/core/src/sync/sync_status.rs @@ -53,8 +53,6 @@ pub struct DownloadSyncStatus { /// received), information about how far the download has progressed. pub downloading: Option, pub streams: Vec, - /// The latest checkpoint request id whose full checkpoint has been applied locally. - pub last_applied_checkpoint_request_id: Option, } impl DownloadSyncStatus { @@ -119,16 +117,9 @@ impl DownloadSyncStatus { self.debug_assert_priority_status_is_sorted(); } - pub fn applied_checkpoint( - &mut self, - now: TimestampMicros, - applied_checkpoint_request_id: Option, - ) { + pub fn applied_checkpoint(&mut self, now: TimestampMicros) { self.downloading = None; self.priority_status.clear(); - if let Some(request_id) = applied_checkpoint_request_id { - self.last_applied_checkpoint_request_id = Some(request_id); - } self.priority_status.push(SyncPriorityStatus { priority: BucketPriority::SENTINEL, @@ -146,7 +137,6 @@ impl Default for DownloadSyncStatus { downloading: None, priority_status: Vec::new(), streams: Vec::new(), - last_applied_checkpoint_request_id: None, } } } @@ -190,16 +180,12 @@ impl Serialize for DownloadSyncStatus { } } - let mut serializer = serializer.serialize_struct("DownloadSyncStatus", 6)?; + let mut serializer = serializer.serialize_struct("DownloadSyncStatus", 5)?; serializer.serialize_field("connected", &self.connected)?; serializer.serialize_field("connecting", &self.connecting)?; serializer.serialize_field("priority_status", &self.priority_status)?; serializer.serialize_field("downloading", &self.downloading)?; serializer.serialize_field("streams", &SerializeStreamsWithProgress(self))?; - serializer.serialize_field( - "last_applied_checkpoint_request_id", - &self.last_applied_checkpoint_request_id, - )?; serializer.end() } diff --git a/dart/test/goldens/simple_iteration.json b/dart/test/goldens/simple_iteration.json index c6c3ac21..e8eb7ff6 100644 --- a/dart/test/goldens/simple_iteration.json +++ b/dart/test/goldens/simple_iteration.json @@ -10,8 +10,7 @@ "connecting": true, "priority_status": [], "downloading": null, - "streams": [], - "last_applied_checkpoint_request_id": null + "streams": [] } } }, @@ -72,8 +71,7 @@ } } }, - "streams": [], - "last_applied_checkpoint_request_id": null + "streams": [] } } } @@ -123,8 +121,7 @@ } } }, - "streams": [], - "last_applied_checkpoint_request_id": null + "streams": [] } } } @@ -163,8 +160,7 @@ } ], "downloading": null, - "streams": [], - "last_applied_checkpoint_request_id": null + "streams": [] } } } diff --git a/dart/test/goldens/starting_stream.json b/dart/test/goldens/starting_stream.json index bc9b1436..b63f2a03 100644 --- a/dart/test/goldens/starting_stream.json +++ b/dart/test/goldens/starting_stream.json @@ -14,8 +14,7 @@ "connecting": true, "priority_status": [], "downloading": null, - "streams": [], - "last_applied_checkpoint_request_id": null + "streams": [] } } }, diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index 34c1d5a4..06046c3f 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -65,8 +65,7 @@ void _syncTests({ bool establishesSyncStream(List instructions) { return instructions.any((instruction) => - instruction is Map && - instruction.containsKey('EstablishSyncStream')); + instruction is Map && instruction.containsKey('EstablishSyncStream')); } List invokeControl(String operation, Object? data) { @@ -157,6 +156,12 @@ void _syncTests({ return rows.isEmpty ? null : rows.single.columnAt(0); } + Object? lastAppliedCheckpointRequestId() { + final rows = db.select( + "SELECT value FROM ps_kv WHERE key = 'last_applied_checkpoint_request_id'"); + return rows.isEmpty ? null : rows.single.columnAt(0); + } + Object? streamLastCheckpointRequestId(List instructions) { final instruction = instructions.whereType().firstWhere( (instruction) => instruction.containsKey('EstablishSyncStream')); @@ -363,7 +368,15 @@ void _syncTests({ invokeControl('start', null); pushCheckpoint(buckets: priorityBuckets, writeCheckpoint: '1'); - pushCheckpointComplete(); + expect( + pushCheckpointComplete(), + contains( + containsPair( + 'CheckpointRequestApplied', + {'request_id': 1}, + ), + ), + ); controller.elapse(Duration(minutes: 10)); pushCheckpoint(buckets: priorityBuckets); @@ -378,23 +391,20 @@ void _syncTests({ 'UpdateSyncStatus', containsPair( 'status', - allOf( - containsPair('last_applied_checkpoint_request_id', 1), - containsPair( - 'priority_status', - [ - { - 'priority': 2, - 'last_synced_at': timestamp(), - 'has_synced': true - }, - { - 'priority': 2147483647, - 'last_synced_at': timestamp(plusMinutes: -10), - 'has_synced': true - }, - ], - ), + containsPair( + 'priority_status', + [ + { + 'priority': 2, + 'last_synced_at': timestamp(), + 'has_synced': true + }, + { + 'priority': 2147483647, + 'last_synced_at': timestamp(plusMinutes: -10), + 'has_synced': true + }, + ], ), )), ), @@ -404,7 +414,6 @@ void _syncTests({ expect(json.decode(row[0]), { 'connected': false, 'connecting': false, - 'last_applied_checkpoint_request_id': 1, 'priority_status': [ {'priority': 2, 'last_synced_at': timestamp(), 'has_synced': true}, { @@ -448,7 +457,8 @@ void _syncTests({ expect(lastRequestedCheckpointRequestId(), 101); }); - syncTest('requires checkpoint request state before allocating checkpoint ids', (_) { + syncTest('requires checkpoint request state before allocating checkpoint ids', + (_) { invokeControlRaw('start', null); expect( @@ -478,7 +488,8 @@ void _syncTests({ expect(probeLocalTargetOp(), 1); }); - syncTest('does not store non-request target ops as checkpoint request id', (_) { + syncTest('does not store non-request target ops as checkpoint request id', + (_) { expect(probeLocalTargetOp(0), isNull); expect(lastRequestedCheckpointRequestId(), isNull); expect(probeLocalTargetOp(), isNull); @@ -491,13 +502,13 @@ void _syncTests({ syncTest('does not persist placeholder checkpoint request id', (_) { db.execute("insert into items (id, col) values ('local', 'data');"); - invokeControl('start', null); + invokeControlRaw('start', null); expect(lastRequestedCheckpointRequestId(), isNull); }); syncTest( - 'does not mark checkpoint request id as synced for partial checkpoint', + 'does not emit applied checkpoint request id for partial checkpoint', (_) { invokeControl('start', null); @@ -506,55 +517,51 @@ void _syncTests({ expect( instructions, - contains( - containsPair( - 'UpdateSyncStatus', - containsPair( - 'status', - containsPair('last_applied_checkpoint_request_id', null), - ), - ), - ), + isNot(contains(containsPair('CheckpointRequestApplied', anything))), ); + expect(lastAppliedCheckpointRequestId(), isNull); final [row] = db.select('select powersync_offline_sync_status();'); expect( json.decode(row[0]), - containsPair('last_applied_checkpoint_request_id', null), + isNot(containsPair('last_applied_checkpoint_request_id', anything)), ); }, ); - syncTest('keeps synced checkpoint request id across normal checkpoints', (_) { + syncTest('emits applied checkpoint request id for full checkpoint', (_) { invokeControl('start', null); pushCheckpoint(buckets: priorityBuckets, writeCheckpoint: '1'); - pushCheckpointComplete(); - - pushCheckpoint(buckets: priorityBuckets); - final instructions = pushCheckpointComplete(); + final appliedInstructions = pushCheckpointComplete(); expect( - instructions, + appliedInstructions, contains( containsPair( - 'UpdateSyncStatus', - containsPair( - 'status', - containsPair('last_applied_checkpoint_request_id', 1), - ), + 'CheckpointRequestApplied', + {'request_id': 1}, ), ), ); + expect(lastAppliedCheckpointRequestId(), 1); + + pushCheckpoint(buckets: priorityBuckets); + final instructions = pushCheckpointComplete(); + + expect( + instructions, + isNot(contains(containsPair('CheckpointRequestApplied', anything))), + ); final [row] = db.select('select powersync_offline_sync_status();'); expect( json.decode(row[0]), - containsPair('last_applied_checkpoint_request_id', 1), + isNot(containsPair('last_applied_checkpoint_request_id', anything)), ); - expect(db.select(r"SELECT * FROM ps_buckets WHERE name = '$local'"), - isEmpty); + expect( + db.select(r"SELECT * FROM ps_buckets WHERE name = '$local'"), isEmpty); }); test('clearing database clears sync status', () { @@ -591,15 +598,18 @@ void _syncTests({ final request = invokeControl('start', null); expect( request, - contains(containsPair( - 'EstablishSyncStream', - { - // Should request state from before clear - 'request': containsPair('buckets', [ - {'name': 'a', 'after': '1'} - ]), - }, - )), + contains( + containsPair( + 'EstablishSyncStream', + containsPair( + // Should request state from before clear + 'request', + containsPair('buckets', [ + {'name': 'a', 'after': '1'} + ]), + ), + ), + ), ); pushCheckpoint(buckets: [bucketDescription('a', count: 1)]); diff --git a/docs/sync.md b/docs/sync.md index 81fe426d..0dbc5362 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -96,6 +96,7 @@ type Instruction = { LogLine: LogLine } | { EstablishSyncStream: EstablishSyncStream } | { FetchCredentials: FetchCredentials } | { CheckpointRequestId: { request_id: number } } + | { CheckpointRequestApplied: { request_id: number } } | { LocalTargetOp: { target_op: null | number } } // Close a connection previously started after EstablishSyncStream | { CloseSyncStream: { hide_disconnect: boolean } } @@ -128,7 +129,12 @@ interface UpdateSyncStatus { priority_status: [], downloading: null | DownloadProgress, streams: [], - last_applied_checkpoint_request_id: null | number, +} + +// Emitted when a full checkpoint with a write_checkpoint has been applied locally. +// SDKs can use this to resolve pending CheckpointRequest waiters. +interface CheckpointRequestApplied { + request_id: number, } // Instructs SDKs to refresh credentials from the backend connector. diff --git a/docs/write-checkpoint-requests.md b/docs/write-checkpoint-requests.md index 02d85cff..8e89fc42 100644 --- a/docs/write-checkpoint-requests.md +++ b/docs/write-checkpoint-requests.md @@ -211,11 +211,12 @@ on completed_upload: ``` After a full checkpoint applies, core stores the applied checkpoint request id as -`last_applied_checkpoint_request_id` and emits it in sync status. +`last_applied_checkpoint_request_id` and emits a `CheckpointRequestApplied` instruction. ```text after full checkpoint apply: ps_kv['last_applied_checkpoint_request_id'] = checkpoint.write_checkpoint + emit CheckpointRequestApplied { request_id: checkpoint.write_checkpoint } ``` ## Explicit checkpoint requests @@ -225,15 +226,14 @@ the local database has caught up to the service. This creates a checkpoint reque connected sync client and returns a `CheckpointRequest`. This explicit API does not update `local_target_op`: it is a wait marker, not a local upload gate. -The returned object waits until sync status reports `last_applied_checkpoint_request_id >= requestId`. +The returned object waits until core emits `CheckpointRequestApplied` for an id greater than or +equal to the requested id. ```text -isSynced = status.lastAppliedCheckpointRequestId >= requestId - waitForSync() { - for status in syncStatusUpdates { - return when status.lastAppliedCheckpointRequestId >= requestId - throw if status reports a sync error + for instruction in syncInstructions { + return when instruction.CheckpointRequestApplied.request_id >= requestId + throw if sync status reports a sync error } } ``` @@ -253,7 +253,8 @@ request could not be delivered to the service or observed in the sync stream. - `last_seen_checkpoint_request_id`: The latest full checkpoint `write_checkpoint` observed and validated from the sync stream. - `last_applied_checkpoint_request_id`: The latest full checkpoint `write_checkpoint` that has been - applied locally. SDKs expose this in sync status and use it to resolve `CheckpointRequest` waits. + applied locally. Core persists this for migration/downgrade state; SDKs should use + `CheckpointRequestApplied` instructions to resolve `CheckpointRequest` waits. ## Migration from `$local` From 12f58aa25e0c53c4ebdc0f2ba9b93c1d379a4a93 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 2 Jul 2026 18:04:58 +0200 Subject: [PATCH 08/29] remove dead macros. Update migrations to fix potential bugs. update doc entries --- crates/core/src/crud_vtab.rs | 9 +- crates/core/src/macros.rs | 46 --------- crates/core/src/migrations.rs | 35 ++++--- crates/core/src/sync/storage_adapter.rs | 22 ++-- dart/test/crud_test.dart | 10 +- dart/test/migration_test.dart | 57 +++++++++-- dart/test/sync_test.dart | 59 +++++++++-- docs/schema.md | 7 +- docs/sync.md | 56 ++++++---- docs/write-checkpoint-requests.md | 131 ++++++++++++++++-------- 10 files changed, 271 insertions(+), 161 deletions(-) diff --git a/crates/core/src/crud_vtab.rs b/crates/core/src/crud_vtab.rs index e14122a9..36936e3e 100644 --- a/crates/core/src/crud_vtab.rs +++ b/crates/core/src/crud_vtab.rs @@ -247,8 +247,15 @@ impl SimpleCrudTransactionMode { fn record_local_write(&mut self, db: *mut sqlite::sqlite3) -> Result<(), ResultCode> { if !self.had_writes { + // Also clear the seen/applied high-water marks: checkpoint request ids observed before + // this write can't acknowledge it, and they may come from an incompatible id namespace + // (legacy write checkpoints migrated by v14, or state from before a counter restart). + // Keeping them around could open the apply gate for a newly allocated target id that + // compares below a stale seen value. The legacy `$local` bookkeeping had the same + // behavior by resetting the entire row on local writes. db.exec_safe(formatcp!( - "INSERT OR REPLACE INTO ps_kv(key, value) VALUES('local_target_op', {MAX_OP_ID})" + "INSERT OR REPLACE INTO ps_kv(key, value) VALUES('local_target_op', {MAX_OP_ID}); +DELETE FROM ps_kv WHERE key IN ('last_seen_checkpoint_request_id', 'last_applied_checkpoint_request_id')" ))?; self.had_writes = true; } diff --git a/crates/core/src/macros.rs b/crates/core/src/macros.rs index 0bc8dc5a..4e60c953 100644 --- a/crates/core/src/macros.rs +++ b/crates/core/src/macros.rs @@ -43,49 +43,3 @@ macro_rules! create_sqlite_optional_text_fn { } }; } - -#[macro_export] -macro_rules! create_sqlite_int_fn { - ($fn_name:ident, $fn_impl_name:ident, $description:literal) => { - extern "C" fn $fn_name( - ctx: *mut sqlite::context, - argc: c_int, - argv: *mut *mut sqlite::value, - ) { - let args = sqlite::args!(argc, argv); - - let result = $fn_impl_name(ctx, args); - - if let Err(err) = result { - PowerSyncError::from(err).apply_to_ctx($description, ctx); - } else if let Ok(r) = result { - ctx.result_int64(r); - } - } - }; -} - -#[macro_export] -macro_rules! create_sqlite_optional_int_fn { - ($fn_name:ident, $fn_impl_name:ident, $description:literal) => { - extern "C" fn $fn_name( - ctx: *mut sqlite::context, - argc: c_int, - argv: *mut *mut sqlite::value, - ) { - let args = sqlite::args!(argc, argv); - - let result = $fn_impl_name(ctx, args); - - if let Err(err) = result { - PowerSyncError::from(err).apply_to_ctx($description, ctx); - } else if let Ok(r) = result { - if let Some(i) = r { - ctx.result_int64(i); - } else { - ctx.result_null(); - } - } - } - }; -} diff --git a/crates/core/src/migrations.rs b/crates/core/src/migrations.rs index 6d6bbfd4..afc1ae5a 100644 --- a/crates/core/src/migrations.rs +++ b/crates/core/src/migrations.rs @@ -468,18 +468,34 @@ DROP TABLE ps_sync_state_old; // `$local.last_op` represented the latest legacy write checkpoint seen in the sync stream, // so it becomes the last seen checkpoint request. // - // `$local.target_op` can either be a concrete checkpoint request id or a sentinel such as - // i64::MAX while local writes are pending. Store it separately as `local_target_op`, but - // only treat concrete values as requested checkpoint ids. We intentionally don't seed - // `last_requested_checkpoint_request_id` from `$local.last_applied_op` because that is an - // applied value, not necessarily the current requested target. + // `$local.target_op` can either be a concrete legacy write checkpoint id or a sentinel such + // as i64::MAX while local writes are pending. Store it separately as `local_target_op`. + // Seeding `last_requested_checkpoint_request_id` from a concrete target would be possible, + // but should be redundant because SDKs reconcile the request counter with service state on + // connect before advancing it through `next_checkpoint_request_id`. // // An absent local target can safely start client-created checkpoint requests from 1. The // ambiguous case is an existing max-op local target without a concrete requested id: // pending local writes may already be associated with legacy service-created write // checkpoints, so SDKs should bridge once through the legacy endpoint before starting // client-created checkpoint requests. + // + // This migration can also run on a database that was previously on version 14 and then + // downgraded: the down migration rebuilds the `$local` row from ps_kv but keeps the ps_kv + // keys around, and an older SDK may have advanced `$local` since. Clear the keys first so + // the `$local` row is the source of truth and the inserts below can't conflict. + // + // After copying, the `$local` row is deleted: version 14 tracks this state exclusively in + // ps_kv, so ps_buckets only contains real sync buckets. The down migration recreates the + // row from ps_kv when needed. let up = "\ +DELETE FROM ps_kv + WHERE key IN ( + 'last_applied_checkpoint_request_id', + 'last_seen_checkpoint_request_id', + 'local_target_op' + ); + INSERT INTO ps_kv(key, value) SELECT 'last_applied_checkpoint_request_id', last_applied_op FROM ps_buckets @@ -492,19 +508,14 @@ SELECT 'last_seen_checkpoint_request_id', last_op WHERE name = '$local' AND last_op > 0; -INSERT INTO ps_kv(key, value) -SELECT 'last_requested_checkpoint_request_id', target_op - FROM ps_buckets - WHERE name = '$local' - AND target_op > 0 - AND target_op != 9223372036854775807; - INSERT INTO ps_kv(key, value) SELECT 'local_target_op', target_op FROM ps_buckets WHERE name = '$local' AND target_op > 0; +DELETE FROM ps_buckets WHERE name = '$local'; + ALTER TABLE ps_buckets DROP COLUMN target_op; "; local_db.exec_safe(up).into_db_result(local_db)?; diff --git a/crates/core/src/sync/storage_adapter.rs b/crates/core/src/sync/storage_adapter.rs index d636f95f..d76f3efd 100644 --- a/crates/core/src/sync/storage_adapter.rs +++ b/crates/core/src/sync/storage_adapter.rs @@ -523,13 +523,15 @@ WHERE bucket = ?1", /// The target op can also be used internally as a sentinel value such as max op id while local /// writes are pending, so it must not always be interpreted as a checkpoint request id. /// - /// When the target op is a positive, non-sentinel checkpoint request id, it also updates - /// `last_requested_checkpoint_request_id` so clients can migrate from the legacy target-op - /// flow to client-created checkpoint requests. `0` clears the local target, and sentinel - /// values such as max op id must not update the last requested id. + /// This only updates the apply gate. It does not allocate, seed or overwrite + /// `last_requested_checkpoint_request_id`, which is managed by `seed_checkpoint_request_id` and + /// `next_checkpoint_request_id`. /// /// Returns the target op value from before this call. When `target_op` is `None`, this only /// reads the current value. + /// + /// Negative values are rejected when parsing the `powersync_control` payload, before this is + /// called. pub fn probe_local_target_op( &self, target_op: Option, @@ -540,12 +542,6 @@ WHERE bucket = ?1", return Ok(previous_target_op); }; - if target_op < 0 { - return Err(PowerSyncError::argument_error( - "target op must be a non-negative integer", - )); - } - if target_op == 0 { self.delete_kv(LOCAL_TARGET_OP_KEY)?; return Ok(previous_target_op); @@ -553,12 +549,6 @@ WHERE bucket = ?1", self.write_i64_kv(LOCAL_TARGET_OP_KEY, target_op)?; - // Concrete target ops also seed the request counter for clients migrating from legacy - // service-created write checkpoints to client-created checkpoint requests. - if target_op != i64::MAX { - self.write_i64_kv(LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY, target_op)?; - } - Ok(previous_target_op) } diff --git a/dart/test/crud_test.dart b/dart/test/crud_test.dart index 0f561a48..8d13c70a 100644 --- a/dart/test/crud_test.dart +++ b/dart/test/crud_test.dart @@ -250,6 +250,14 @@ void main() { }); test('updates local target op and updated rows', () { + // Stale high-water marks (e.g. migrated legacy write checkpoints) must be cleared by a + // local write, so they can't open the apply gate for a smaller new target id. + db.execute(''' +INSERT INTO ps_kv(key, value) VALUES + ('last_seen_checkpoint_request_id', 6), + ('last_applied_checkpoint_request_id', 5); +'''); + db.execute( 'INSERT INTO powersync_crud (op, id, type, data) VALUES (?, ?, ?, ?)', [ @@ -266,7 +274,7 @@ void main() { isEmpty); expect( db.select( - "SELECT key, value FROM ps_kv WHERE key = 'local_target_op'"), + "SELECT key, value FROM ps_kv WHERE key LIKE '%checkpoint_request_id' OR key = 'local_target_op'"), [ { 'key': 'local_target_op', diff --git a/dart/test/migration_test.dart b/dart/test/migration_test.dart index 771ccd77..514cf4da 100644 --- a/dart/test/migration_test.dart +++ b/dart/test/migration_test.dart @@ -106,16 +106,15 @@ VALUES(1, '$local', 5, 6, 7, 0, 0, 1, 0, 0, 0); db.executeInTx('select powersync_init()'); - expect(db.select('SELECT key, value FROM ps_kv ORDER BY key'), containsAll([ - {'key': 'last_seen_checkpoint_request_id', 'value': 6}, - {'key': 'last_requested_checkpoint_request_id', 'value': 7}, + expect(db.select('SELECT key, value FROM ps_kv ORDER BY key'), [ {'key': 'last_applied_checkpoint_request_id', 'value': 5}, + {'key': 'last_seen_checkpoint_request_id', 'value': 6}, {'key': 'local_target_op', 'value': 7}, - ])); + ]); + expect(db.select(r"SELECT * FROM ps_buckets WHERE name = '$local'"), + isEmpty); expect( - db - .select("PRAGMA table_info('ps_buckets')") - .map((row) => row['name']), + db.select("PRAGMA table_info('ps_buckets')").map((row) => row['name']), isNot(contains('target_op')), ); expect( @@ -138,8 +137,8 @@ VALUES(1, '$local', 5, 6, 9223372036854775807, 0, 0, 1, 0, 0, 0); db.executeInTx('select powersync_init()'); // last_applied_op becomes the applied checkpoint id, but it must not seed the requested - // checkpoint counter. The sentinel target is preserved for blocking, but is not concrete - // enough to become last_requested_checkpoint_request_id. + // checkpoint counter. The sentinel target is preserved for blocking, but target ops no longer + // seed last_requested_checkpoint_request_id. expect(db.select('SELECT key, value FROM ps_kv ORDER BY key'), [ {'key': 'last_applied_checkpoint_request_id', 'value': 5}, {'key': 'last_seen_checkpoint_request_id', 'value': 6}, @@ -157,8 +156,8 @@ VALUES(1, '$local', 0, 0, 9223372036854775807, 0, 0, 1, 0, 0, 0); db.executeInTx('select powersync_init()'); - // The max-op sentinel is valid local target state, but it is not a concrete checkpoint - // request id and must not seed last_requested_checkpoint_request_id. + // The max-op sentinel is valid local target state, but target ops no longer seed + // last_requested_checkpoint_request_id. expect(db.select('SELECT key, value FROM ps_kv ORDER BY key'), [ {'key': 'local_target_op', 'value': 9223372036854775807}, ]); @@ -190,6 +189,42 @@ INSERT INTO ps_kv(key, value) VALUES ); }); + test('re-upgrades after downgrade with checkpoint state', () async { + db.execute(fixtures.finalState); + db.execute(r''' +INSERT INTO ps_kv(key, value) VALUES + ('last_requested_checkpoint_request_id', 7), + ('last_seen_checkpoint_request_id', 6), + ('last_applied_checkpoint_request_id', 5), + ('local_target_op', 7); +'''); + + db.executeInTx('select powersync_test_migration(13)'); + + // Simulate an older SDK advancing the restored $local row while downgraded. + db.execute(r''' +UPDATE ps_buckets + SET last_op = 8, last_applied_op = 8, target_op = 9 + WHERE name = '$local' +'''); + + db.executeInTx('select powersync_init()'); + + // The $local row is the source of truth on re-upgrade; the request counter is unrelated to + // $local and survives the downgrade unchanged. + expect(db.select('SELECT key, value FROM ps_kv ORDER BY key'), [ + {'key': 'last_applied_checkpoint_request_id', 'value': 8}, + {'key': 'last_requested_checkpoint_request_id', 'value': 7}, + {'key': 'last_seen_checkpoint_request_id', 'value': 8}, + {'key': 'local_target_op', 'value': 9}, + ]); + expect(db.select(r"SELECT * FROM ps_buckets WHERE name = '$local'"), + isEmpty); + + final schema = '${getSchema(db)}\n${getMigrations(db)}'; + expect(schema, equals(fixtures.finalState.trim())); + }); + test('does not restore local bucket without local target on downgrade', () async { db.execute(fixtures.finalState); diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index 06046c3f..93a8a4e1 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -474,26 +474,52 @@ void _syncTests({ syncTest('probes and updates local target op without sync iteration', (_) { expect(probeLocalTargetOp(), isNull); expect(probeLocalTargetOp(1), isNull); - expect(lastRequestedCheckpointRequestId(), 1); + expect(lastRequestedCheckpointRequestId(), isNull); expect(probeLocalTargetOp(), 1); expect(probeLocalTargetOp(2), 1); - expect(lastRequestedCheckpointRequestId(), 2); + expect(lastRequestedCheckpointRequestId(), isNull); expect(probeLocalTargetOp(), 2); }); syncTest('accepts text checkpoint request ids for local target op', (_) { expect(probeLocalTargetOp('1'), isNull); - expect(lastRequestedCheckpointRequestId(), 1); + expect(lastRequestedCheckpointRequestId(), isNull); expect(probeLocalTargetOp(), 1); }); - syncTest('does not store non-request target ops as checkpoint request id', - (_) { + syncTest('rejects negative local target ops', (_) { + expect( + () => invokeControlRaw('local_target_op', -1), + throwsA(isSqliteException( + 3091, + contains('local target op must be a non-negative integer'), + )), + ); + }); + + syncTest('local target op does not update checkpoint request id', (_) { + invokeControlRaw('start', null); + invokeControlRaw('seed_checkpoint_request_id', 10); + + expect(lastRequestedCheckpointRequestId(), 10); + expect(probeLocalTargetOp(7), isNull); + expect(probeLocalTargetOp(), 7); + expect(lastRequestedCheckpointRequestId(), 10); + }); + + syncTest('does not store target ops as checkpoint request id', (_) { expect(probeLocalTargetOp(0), isNull); expect(lastRequestedCheckpointRequestId(), isNull); expect(probeLocalTargetOp(), isNull); + expect(probeLocalTargetOp(1), isNull); + expect(lastRequestedCheckpointRequestId(), isNull); + expect(probeLocalTargetOp(), 1); + + expect(probeLocalTargetOp(0), 1); + expect(probeLocalTargetOp(), isNull); + expect(probeLocalTargetOp(9223372036854775807), isNull); expect(lastRequestedCheckpointRequestId(), isNull); expect(probeLocalTargetOp(), 9223372036854775807); @@ -564,6 +590,27 @@ void _syncTests({ db.select(r"SELECT * FROM ps_buckets WHERE name = '$local'"), isEmpty); }); + syncTest('local writes clear checkpoint request high-water marks', (_) { + invokeControl('start', null); + + pushCheckpoint(buckets: priorityBuckets, writeCheckpoint: '5'); + pushCheckpointComplete(); + expect(lastAppliedCheckpointRequestId(), 5); + + // A local write can only be acknowledged by a checkpoint request id observed after it. Stale + // seen/applied values (which may come from another id namespace, like migrated legacy write + // checkpoints) must not remain to open the apply gate for a smaller new target id. + db.execute("insert into items (id, col) values ('local', 'data');"); + + expect(lastAppliedCheckpointRequestId(), isNull); + expect( + db.select( + "SELECT 1 FROM ps_kv WHERE key = 'last_seen_checkpoint_request_id'"), + isEmpty, + ); + expect(probeLocalTargetOp(), 9223372036854775807); + }); + test('clearing database clears sync status', () { invokeControl('start', null); pushCheckpoint(buckets: priorityBuckets); @@ -1030,7 +1077,7 @@ void _syncTests({ db.execute('DELETE FROM ps_crud'); probeLocalTargetOp(1); expect(invokeControl('completed_upload', null), isEmpty); - expect(lastRequestedCheckpointRequestId(), 1); + expect(lastRequestedCheckpointRequestId(), 0); // Sync afterwards containing data and write checkpoint. pushCheckpoint(buckets: priorityBuckets, writeCheckpoint: '1'); diff --git a/docs/schema.md b/docs/schema.md index 1955bab1..4a6462b9 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -31,9 +31,10 @@ a checkpoint and that we have validated its checksum). 8. `count_since_last`: The amount of operations downloaded since the last verified checkpoint. Schema version 14 removes the legacy `target_op` column after migrating `$local.target_op` to -`ps_kv.local_target_op`. This makes older SDKs fail with a hard SQLite error if they try to keep -using the migrated database without downgrading. The down migration restores `target_op` for older -schema versions. +`ps_kv.local_target_op`, and deletes the `$local` row so `ps_buckets` only contains real sync +buckets. This makes older SDKs fail with a hard SQLite error if they try to keep using the migrated +database without downgrading. The down migration restores `target_op` and recreates the `$local` +row from `ps_kv` for older schema versions. ## `ps_crud` diff --git a/docs/sync.md b/docs/sync.md index 0dbc5362..929e1a35 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -42,11 +42,14 @@ The following commands are supported: clears the local target op and returns the previously-observed value in a `LocalTargetOp` result. This command can run outside of a sync iteration and does not affect it. 12. `seed_checkpoint_request_id`: Payload is `null`, an integer, or an integer string. After - receiving `EstablishSyncStream`, SDKs can re-request the client's last checkpoint request state - from the service using the provided hint, then seed core with the actual response for - reconciliation. `NULL` means the service has no record for the client yet; core stores `0` only - when no local seed exists. Integer seeds use `max(local, service)` semantics so the local counter - never moves backwards. + receiving `EstablishSyncStream`, SDKs should reconcile the provided local hint with the service + checkpoint-request state on every connection attempt. This can bump core when the service is + ahead, or restore the service-side value when the service has cleared stale state but core still + has a local hint. Then seed core with the reconciled value. `NULL` means neither side has a + record for the client yet; core stores `0` only when no local seed exists. Integer seeds use + `max(local, service)` semantics so the local counter never moves backwards while either side + still remembers the value. If both the client and service have lost the value, the counter may + restart. When uploads request a write checkpoint, SDKs should call `powersync_control('next_checkpoint_request_id', NULL)` inside a transaction to allocate the id to @@ -62,31 +65,40 @@ target op. The same command is used for compatibility when a new SDK is used wit PowerSync service that does not yet support client-created checkpoint requests; after the service-side write checkpoint request returns a concrete id, call `powersync_control('local_target_op', id)` with that id. Passing `NULL` returns the current target -without changing it, and passing `0` clears the local target. Updating to a positive, non-sentinel -target op also stores it as `last_requested_checkpoint_request_id` to support migrating to -client-created checkpoint requests. Sentinel values such as max op id are not stored as requested -checkpoint ids. +without changing it, and passing `0` clears the local target. This command only updates the apply +gate; it does not allocate, seed, or overwrite `last_requested_checkpoint_request_id`. Database migration v14 moves legacy `$local` checkpoint state into `ps_kv`: `$local.last_applied_op` becomes `last_applied_checkpoint_request_id`, `$local.last_op` becomes the internal -`last_seen_checkpoint_request_id`, a concrete `$local.target_op` advances the request counter, and -`$local.target_op` is stored as `local_target_op`. The migration then drops `ps_buckets.target_op` -so older SDKs fail hard if they try to keep using the migrated database directly. Downgrading -restores the column, and restores a `$local` row only when `local_target_op` exists, so older SDKs -can keep using target-op based blocking without inventing a synthetic local bucket when there was no -local target state. +`last_seen_checkpoint_request_id`, and any positive `$local.target_op` is stored as +`local_target_op`. A concrete `$local.target_op` could be used to seed +`last_requested_checkpoint_request_id`, but it should be redundant because SDKs reconcile the +request counter with the service on connect. The migration then deletes the `$local` row, leaving +only real sync buckets in `ps_buckets`, and drops `ps_buckets.target_op` so +older SDKs fail hard if they try to keep using the migrated database directly. Downgrading restores +the column, and restores a `$local` row only when `local_target_op` exists, so older SDKs can keep +using target-op based blocking without inventing a synthetic local bucket when there was no local +target state. Because the down migration keeps the `ps_kv` keys around, the up migration clears +them before copying, so re-upgrading a downgraded database takes the `$local` row (including any +progress an older SDK made) as the source of truth instead of failing on the existing keys. + +`last_requested_checkpoint_request_id` is internal allocation state used by +`next_checkpoint_request_id` to allocate increasing ids for client-created checkpoint requests. +`last_seen_checkpoint_request_id` and `last_applied_checkpoint_request_id` are high-water marks +that local writes clear, so only checkpoint request ids observed after a write count towards the +apply gate. SDKs should use `CheckpointRequestApplied` instructions for explicit checkpoint +request waits instead of presenting these values as meaningful sync progress. If `local_target_op` is absent after migration, there is no local write gate waiting for a checkpoint. In that case, SDKs can start client-created checkpoint requests normally, even when `last_requested_checkpoint_request_id` is undefined and the first allocated id is `1`. -The ambiguous case is a migrated `local_target_op` of max op id with no -`last_requested_checkpoint_request_id`: local writes are pending, but there is no concrete request -id to wait for yet. The max-op sentinel may also cover earlier pending uploads that were already -associated with legacy service-created write checkpoints, so restarting the client-created counter -from `1` could create a target lower than those existing associations. In that state, create one -old-style write checkpoint first, store the returned concrete id with -`powersync_control('local_target_op', id)`, and then switch to client-created checkpoint requests. +The ambiguous migration case is a migrated `local_target_op` of max op id: local writes are +pending, but there is no concrete request id to wait for yet. The max-op sentinel may also cover +earlier pending uploads that were already associated with legacy service-created write checkpoints. +In that state, create one old-style write checkpoint first, store the returned concrete id with +`powersync_control('local_target_op', id)`, let that gate resolve, and then switch to +client-created checkpoint requests after the request counter has been reconciled on connect. `powersync_control` returns a JSON-encoded array of instructions for the client: diff --git a/docs/write-checkpoint-requests.md b/docs/write-checkpoint-requests.md index 8e89fc42..bc3d34b2 100644 --- a/docs/write-checkpoint-requests.md +++ b/docs/write-checkpoint-requests.md @@ -9,15 +9,22 @@ At a high level: - `local_target_op` replaces `$local.target_op` as the local write apply gate. - `last_seen_checkpoint_request_id` replaces `$local.last_op`. - `last_applied_checkpoint_request_id` replaces `$local.last_applied_op`. -- `last_requested_checkpoint_request_id` tracks the latest concrete checkpoint request id known to - the client. +- `last_requested_checkpoint_request_id` tracks the latest concrete checkpoint request id allocated + by the client, so `next_checkpoint_request_id` can allocate increasing ids for each checkpoint + request. + +These keys are internal SDK/core state, not user-facing sync progress. +`last_requested_checkpoint_request_id` is functional allocation state, while +`last_seen_checkpoint_request_id` and `last_applied_checkpoint_request_id` are mostly diagnostic +high-water marks. Explicit checkpoint waits should follow `CheckpointRequestApplied` instructions. SDKs should not write these keys directly. They update the local target through `powersync_control('local_target_op', value)`, which is the shared helper for both legacy write checkpoints and new client-created checkpoint requests. The `powersync_control('next_checkpoint_request_id', NULL)` command only allocates a checkpoint request id; after the service accepts that request, the SDK uses -`powersync_control('local_target_op', id)` to make the accepted id the local target. +`powersync_control('local_target_op', id)` to make the accepted id the local target for write +checkpoints. For the historic `$local` bucket flow, see `historic-write-checkpoints.md`. @@ -29,17 +36,29 @@ the maximum i64 value. This is the `ps_kv` equivalent of the old `$local.target_ means "there are local writes, but we do not yet know the concrete checkpoint id that will acknowledge them". -The sentinel is stored in `ps_kv`, not in `ps_buckets`: +The sentinel is stored in `ps_kv`, not in `ps_buckets`. The same statement clears the +`last_seen_checkpoint_request_id` and `last_applied_checkpoint_request_id` high-water marks: ```sql INSERT OR REPLACE INTO ps_kv(key, value) VALUES('local_target_op', MAX_OP_ID); + +DELETE FROM ps_kv +WHERE key IN ('last_seen_checkpoint_request_id', 'last_applied_checkpoint_request_id'); ``` +Clearing the high-water marks mirrors how the legacy flow reset the whole `$local` row on local +writes. A checkpoint request id observed before the write cannot acknowledge it, and a stale seen +value may even come from an incompatible id namespace — a legacy write checkpoint migrated by v14, +or state from before a request counter restart. If such a value stayed around, a newly allocated +target id could compare below it and open the apply gate before the service acknowledged the write. +After a local write, only checkpoint request ids observed from that point on count towards the gate. + ## Completing uploaded CRUD -SDK upload code removes uploaded items from `ps_crud`. If the connector supplies a custom write -checkpoint and the queue is empty, that concrete checkpoint becomes the local target immediately. +SDK upload code removes uploaded items from `ps_crud`. If the connector supplies a legacy custom +write checkpoint and the queue is empty, that concrete checkpoint becomes the local target +immediately. Otherwise the target is reset to `MAX_OP_ID`, allowing the sync client to create a standard checkpoint request after the queue drains. @@ -118,12 +137,24 @@ These `powersync_control` commands are the SDK-facing API for the new `ps_kv` ch `powersync_control('start', payload)` begins a sync iteration and emits `EstablishSyncStream` with a `last_checkpoint_request_id` hint. This is core's local `last_requested_checkpoint_request_id` value -before opening the stream, or `NULL` when no local seed exists. On connect, SDKs can use this hint -to re-request the client's last checkpoint request state from the service, then call -`powersync_control('seed_checkpoint_request_id', value)` with the actual response for -reconciliation. `value` may be `NULL` when the service has no record for the client; core stores `0` -only when no local seed exists. Integer seeds use `max(local, service)` semantics so the local -counter never moves backwards. SDKs may also refresh service state when their user/client context +before opening the stream, or `NULL` when no local seed exists. On every connection attempt, SDKs +should reconcile this hint with the service checkpoint-request state before creating new requests. +The reconciliation is bidirectional: if the service still has a higher value, the SDK uses that +response to bump core locally so following requests are accepted; if the service has cleared stale +state but core still has a local hint, the SDK can use the hint to restore the service-side value. +After reconciliation, call `powersync_control('seed_checkpoint_request_id', value)` with the +reconciled value. + +`last_requested_checkpoint_request_id` is a best-effort counter seed, not durable application state. +If either the client or the service still remembers a higher id, the reconciliation response plus +core's `max(local, service)` seeding keeps the local counter moving forward. If the service has +cleared stale state but the client still has a local seed, the local hint can restore the +service-side value and keep the counter from moving backwards. If the client lost local state but +the service still has a record, the service response restores the seed locally. If both sides have +lost the value, it is acceptable for the counter to restart; this can happen after local state is +cleared and stale service state expires, or when multiple user ids share the same client id. After +reconciliation, `value` may be `NULL` when neither side has a record for the client; core stores `0` +only when no local seed exists. SDKs may also refresh service state when their user/client context changes. `powersync_control('next_checkpoint_request_id', NULL)` must be called inside a transaction during @@ -142,9 +173,9 @@ This command only allocates an id. It does not update `local_target_op`. Note on sequences: SQLite does not have standalone sequences. The sequence-like alternatives are either an `AUTOINCREMENT` table backed by SQLite's internal `sqlite_sequence`, or a dedicated single-row counter table like the existing `ps_tx` transaction counter. The checkpoint request -counter currently lives in `ps_kv` because it is also migrated and seeded from legacy/custom -concrete targets via `powersync_control('local_target_op', value)`. If we want stricter structure -later, a dedicated checkpoint-request counter table would be the closest match to a sequence. +counter currently lives in `ps_kv` so it can persist across requests and be reconciled with service +state on connect. If we want stricter structure later, a dedicated checkpoint-request counter table +would be the closest match to a sequence. `powersync_control('local_target_op', op_id)` probes and optionally updates the local target. Like `subscriptions`, this command is handled directly by `powersync_control` and can run outside an @@ -153,9 +184,11 @@ active sync iteration: - `NULL` returns the current `local_target_op` without changing it. - `0` clears `local_target_op`. - A positive value stores `local_target_op`. -- A positive value other than `i64::MAX` also stores `last_requested_checkpoint_request_id`. - Negative values and non-integer inputs are rejected. +This command only updates the apply gate. It does not allocate, seed, or overwrite +`last_requested_checkpoint_request_id`. + The command returns the previous target value in a `LocalTargetOp` result, or `NULL` if there was no target. @@ -168,8 +201,6 @@ if target_op == 0: delete ps_kv['local_target_op'] else: ps_kv['local_target_op'] = target_op - if target_op != MAX_OP_ID: - ps_kv['last_requested_checkpoint_request_id'] = target_op return previous ``` @@ -221,9 +252,9 @@ after full checkpoint apply: ## Explicit checkpoint requests -Swift exposes `PowerSyncDatabaseProtocol.requestCheckpoint()` for callers that want to wait until -the local database has caught up to the service. This creates a checkpoint request id through the -connected sync client and returns a `CheckpointRequest`. +SDKs can expose a `requestCheckpoint()`-style API for callers that want to wait until the local +database has caught up to the service. The SDK creates a checkpoint request id through the connected +sync client and returns a `CheckpointRequest`-style waiter. This explicit API does not update `local_target_op`: it is a wait marker, not a local upload gate. The returned object waits until core emits `CheckpointRequestApplied` for an id greater than or @@ -247,14 +278,18 @@ request could not be delivered to the service or observed in the sync stream. pending, a concrete checkpoint request id after upload completion, or absent when there is no local write gate. - `last_requested_checkpoint_request_id`: The last client-created checkpoint request id allocated - by `powersync_control('next_checkpoint_request_id', NULL)`. - `powersync_control('local_target_op', value)` also writes this key for positive, non-sentinel - targets so migrated or legacy-created concrete targets can seed the client request counter. + by `powersync_control('next_checkpoint_request_id', NULL)`. This is the counter used to allocate + increasing ids for each client-created checkpoint request, including multiple requests in one + client lifetime. The persisted value is also useful for debugging and for seeding the next + connection attempt. SDKs should reconcile it with the service on every connect, and should + tolerate it restarting when both the client and service have lost the previous value. - `last_seen_checkpoint_request_id`: The latest full checkpoint `write_checkpoint` observed and - validated from the sync stream. + validated from the sync stream since the last local write. Local writes clear this key, so only + checkpoint request ids observed after the write can satisfy the apply gate. - `last_applied_checkpoint_request_id`: The latest full checkpoint `write_checkpoint` that has been - applied locally. Core persists this for migration/downgrade state; SDKs should use - `CheckpointRequestApplied` instructions to resolve `CheckpointRequest` waits. + applied locally since the last local write, which clears this key. Core persists this for + migration/downgrade state and debugging; SDKs should use `CheckpointRequestApplied` instructions + to resolve `CheckpointRequest` waits. ## Migration from `$local` @@ -262,26 +297,36 @@ Migration v14 moves the old `$local` bucket state into `ps_kv`: - `$local.last_applied_op` becomes `last_applied_checkpoint_request_id`. - `$local.last_op` becomes `last_seen_checkpoint_request_id`. -- A concrete `$local.target_op` becomes `last_requested_checkpoint_request_id`. - Any positive `$local.target_op`, including `MAX_OP_ID`, becomes `local_target_op`. -After copying this state, the migration drops `ps_buckets.target_op`. This intentionally makes older -SDKs fail with a hard SQLite error if they try to keep using a migrated database without first -downgrading. +A concrete `$local.target_op` could be used to seed `last_requested_checkpoint_request_id`, but it +should be redundant because SDKs reconcile the request counter with service state on connect before +advancing it through `next_checkpoint_request_id`. + +After copying this state, the migration deletes the `$local` row — version 14 tracks this state +exclusively in `ps_kv`, so `ps_buckets` only contains real sync buckets — and drops +`ps_buckets.target_op`. Dropping the column intentionally makes older SDKs fail with a hard SQLite +error if they try to keep using a migrated database without first downgrading. + +The up migration first deletes any existing `last_applied_checkpoint_request_id`, +`last_seen_checkpoint_request_id` and `local_target_op` keys. Those can be present when a database +was previously on version 14 and then downgraded, because the down migration keeps the ps_kv keys +while rebuilding `$local`. Clearing them makes the `$local` row the source of truth on re-upgrade, +picking up any progress an older SDK made while downgraded. `last_requested_checkpoint_request_id` +is unrelated to `$local` and survives a downgrade/upgrade cycle unchanged. An absent `local_target_op` is safe: there is no local write gate waiting for a checkpoint, so an -SDK can start client-created checkpoint requests from `1`. The sync stream will only report that -request id after the service has accepted and reached it. - -The ambiguous case is a migrated `local_target_op` of `MAX_OP_ID` with no -`last_requested_checkpoint_request_id`. That means there is a pending local write gate but no -concrete request id to wait for yet. The `MAX_OP_ID` sentinel only says that local writes dirtied -the gate; it does not prove that no earlier uploads were already associated with legacy -service-created write checkpoints. Those existing checkpoint ids may be higher than a restarted -client counter such as `1`, and using a lower target could let an older seen checkpoint satisfy the -gate too early. In that state, the SDK should create one legacy write checkpoint first, store the -concrete id with `powersync_control('local_target_op', id)`, and then switch to client-created -checkpoint requests. +SDK can seed the request counter on connect and start client-created checkpoint requests normally. +If neither the client nor service has a previous request id, the first allocated id is `1`. The sync +stream will only report that request id after the service has accepted and reached it. + +The ambiguous case is a migrated `local_target_op` of `MAX_OP_ID`. That means there is a pending +local write gate but no concrete request id to wait for yet. The `MAX_OP_ID` sentinel only says that +local writes dirtied the gate; it does not prove that no earlier uploads were already associated +with legacy service-created write checkpoints. In that state, the SDK should create one legacy write +checkpoint first, store the concrete id with `powersync_control('local_target_op', id)`, let that +gate resolve, and then switch to client-created checkpoint requests after the request counter has +been reconciled on connect. The down migration restores `ps_buckets.target_op` and rebuilds a `$local` row only when `local_target_op` exists, using: From a29fe19f56e75261057ac779fba01362c1602393 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 7 Jul 2026 11:12:22 +0200 Subject: [PATCH 09/29] remove dead code --- crates/core/src/view_admin.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/core/src/view_admin.rs b/crates/core/src/view_admin.rs index 6b9496c5..ce20bd0d 100644 --- a/crates/core/src/view_admin.rs +++ b/crates/core/src/view_admin.rs @@ -85,7 +85,6 @@ fn powersync_clear_impl( local_db.exec_safe("DELETE FROM ps_oplog; DELETE FROM ps_buckets")?; } else { trigger_resync(local_db, state)?; - local_db.exec_safe("DELETE FROM ps_buckets WHERE name = '$local'")?; } // language=SQLite From 2527251b237467ad84d1361e0873da9838870647 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 7 Jul 2026 11:26:29 +0200 Subject: [PATCH 10/29] cleanup max values in commands --- crates/core/src/sync/storage_adapter.rs | 26 ++++++++++++------------- crates/core/src/sync/streaming_sync.rs | 4 +++- docs/sync.md | 10 +++++----- docs/write-checkpoint-requests.md | 21 ++++++++++---------- 4 files changed, 32 insertions(+), 29 deletions(-) diff --git a/crates/core/src/sync/storage_adapter.rs b/crates/core/src/sync/storage_adapter.rs index d76f3efd..9081faff 100644 --- a/crates/core/src/sync/storage_adapter.rs +++ b/crates/core/src/sync/storage_adapter.rs @@ -564,6 +564,11 @@ WHERE bucket = ?1", } /// Persists the checkpoint request id that was applied locally. + /// + /// This is always the id from the last applied checkpoint as sent by the service - a plain + /// overwrite with no monotonicity enforced by core. External code owns consistency of these + /// ids; waiters should rely on `CheckpointRequestApplied` instructions rather than comparing + /// this value across reconnects. pub fn persist_last_applied_checkpoint_request_id( &self, request_id: i64, @@ -602,7 +607,11 @@ RETURNING value", self.read_i64_kv(LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY) } - /// Seeds the local checkpoint request counter from service state without moving it backwards. + /// Seeds the local checkpoint request counter from service state. + /// + /// The value is stored verbatim: core does not enforce monotonicity here. SDKs are + /// responsible for reconciling their local hint with the service before seeding, and cannot + /// allocate new checkpoint request ids until that seeding has completed. /// /// A null service value means the service has no record for this client yet. Store zero in /// that case so the first local allocation returns one while still marking the state as seeded. @@ -610,19 +619,10 @@ RETURNING value", &self, request_id: Option, ) -> Result<(), PowerSyncError> { - let stmt = self.db.prepare_v2( - "INSERT INTO ps_kv(key, value) -VALUES(?1, ?2) -ON CONFLICT(key) DO UPDATE SET value = max(CAST(value AS INTEGER), excluded.value)", - )?; - stmt.bind_text( - 1, + self.write_i64_kv( LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY, - sqlite::Destructor::STATIC, - )?; - stmt.bind_int64(2, request_id.unwrap_or(0))?; - stmt.exec()?; - Ok(()) + request_id.unwrap_or(0), + ) } fn read_i64_kv(&self, key: &'static str) -> Result, PowerSyncError> { diff --git a/crates/core/src/sync/streaming_sync.rs b/crates/core/src/sync/streaming_sync.rs index a578dcac..3c44cc0a 100644 --- a/crates/core/src/sync/streaming_sync.rs +++ b/crates/core/src/sync/streaming_sync.rs @@ -418,7 +418,9 @@ impl StreamingSyncIteration { } SyncLocalResult::ChangesApplied { timestamp } => { SyncStateMachineTransition::SyncLocalChangesApplied { - applied_checkpoint_request_id: target.checkpoint.write_checkpoint, + // A checkpoint request is only considered applied once the full + // checkpoint has been applied, not for partial completions. + applied_checkpoint_request_id: None, partial: Some(priority), timestamp, } diff --git a/docs/sync.md b/docs/sync.md index 929e1a35..8d1c2af3 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -45,11 +45,11 @@ The following commands are supported: receiving `EstablishSyncStream`, SDKs should reconcile the provided local hint with the service checkpoint-request state on every connection attempt. This can bump core when the service is ahead, or restore the service-side value when the service has cleared stale state but core still - has a local hint. Then seed core with the reconciled value. `NULL` means neither side has a - record for the client yet; core stores `0` only when no local seed exists. Integer seeds use - `max(local, service)` semantics so the local counter never moves backwards while either side - still remembers the value. If both the client and service have lost the value, the counter may - restart. + has a local hint. Then seed core with the reconciled value. Core stores the seeded value + verbatim and does not enforce monotonicity; SDKs own the reconciliation and must not seed a + stale value. `NULL` means neither side has a record for the client yet; core stores `0` in that + case so the state counts as seeded and the first allocation returns `1`. If both the client and + service have lost the value, the counter may restart. When uploads request a write checkpoint, SDKs should call `powersync_control('next_checkpoint_request_id', NULL)` inside a transaction to allocate the id to diff --git a/docs/write-checkpoint-requests.md b/docs/write-checkpoint-requests.md index bc3d34b2..3a8a7137 100644 --- a/docs/write-checkpoint-requests.md +++ b/docs/write-checkpoint-requests.md @@ -146,16 +146,17 @@ After reconciliation, call `powersync_control('seed_checkpoint_request_id', valu reconciled value. `last_requested_checkpoint_request_id` is a best-effort counter seed, not durable application state. -If either the client or the service still remembers a higher id, the reconciliation response plus -core's `max(local, service)` seeding keeps the local counter moving forward. If the service has -cleared stale state but the client still has a local seed, the local hint can restore the -service-side value and keep the counter from moving backwards. If the client lost local state but -the service still has a record, the service response restores the seed locally. If both sides have -lost the value, it is acceptable for the counter to restart; this can happen after local state is -cleared and stale service state expires, or when multiple user ids share the same client id. After -reconciliation, `value` may be `NULL` when neither side has a record for the client; core stores `0` -only when no local seed exists. SDKs may also refresh service state when their user/client context -changes. +Core stores whatever value is seeded, without enforcing monotonicity: the SDK owns the +reconciliation and is expected to seed the effective state accepted by the service. If either the +client or the service still remembers a higher id, the SDK's reconciliation keeps the local counter +moving forward. If the service has cleared stale state but the client still has a local seed, the +local hint can restore the service-side value and keep the counter from moving backwards. If the +client lost local state but the service still has a record, the service response restores the seed +locally. If both sides have lost the value, it is acceptable for the counter to restart; this can +happen after local state is cleared and stale service state expires, or when multiple user ids share +the same client id. After reconciliation, `value` may be `NULL` when neither side has a record for +the client; core stores `0` in that case so the state counts as seeded and the first allocation +returns `1`. SDKs may also refresh service state when their user/client context changes. `powersync_control('next_checkpoint_request_id', NULL)` must be called inside a transaction during an active sync iteration after `last_requested_checkpoint_request_id` exists locally. It increments From 0709764c64a4cc91528c3d2d98766480df1c674e Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 7 Jul 2026 11:30:40 +0200 Subject: [PATCH 11/29] handle target op instruction consistently --- crates/core/src/sync/interface.rs | 22 ++++++++-------------- crates/core/src/sync/streaming_sync.rs | 4 ++++ dart/test/sync_test.dart | 13 +++++++++++++ 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/crates/core/src/sync/interface.rs b/crates/core/src/sync/interface.rs index a673f353..1b5e8b26 100644 --- a/crates/core/src/sync/interface.rs +++ b/crates/core/src/sync/interface.rs @@ -79,6 +79,10 @@ pub enum SyncControlRequest<'a> { StopSyncStream, /// The client requests a new checkpoint request id. NextCheckpointRequestId, + /// The client probes and optionally updates the local target op. + /// + /// This can run outside of a sync iteration and does not affect it. + ProbeLocalTargetOp { target_op: Option }, /// The client is forwading a sync event to the core extension. SyncEvent(SyncEvent<'a>), } @@ -253,23 +257,13 @@ pub fn register(db: *mut sqlite::sqlite3, state: Rc) -> Result<() }), "stop" => SyncControlRequest::StopSyncStream, "next_checkpoint_request_id" => SyncControlRequest::NextCheckpointRequestId, - "local_target_op" => { - let target_op = parse_optional_i64_payload( + "local_target_op" => SyncControlRequest::ProbeLocalTargetOp { + target_op: parse_optional_i64_payload( *payload, "local target op", "local target op must be an integer, integer string, or null", - )?; - let adapter = state.storage_adapter(db)?; - let target_op = adapter.probe_local_target_op(target_op)?; - let formatted = - serde_json::to_string(&alloc::vec![Instruction::LocalTargetOp { - target_op - },]) - .map_err(PowerSyncError::internal)?; - ctx.result_text_transient(&formatted); - ctx.result_subtype(SUBTYPE_JSON); - return Ok(()); - } + )?, + }, "line_text" => SyncControlRequest::SyncEvent(SyncEvent::TextLine { data: if payload.value_type() == ColumnType::Text { payload.text() diff --git a/crates/core/src/sync/streaming_sync.rs b/crates/core/src/sync/streaming_sync.rs index 3c44cc0a..bd91eef9 100644 --- a/crates/core/src/sync/streaming_sync.rs +++ b/crates/core/src/sync/streaming_sync.rs @@ -133,6 +133,10 @@ impl SyncClient { let request_id = self.adapter.next_checkpoint_request_id()?; Ok(alloc::vec![Instruction::CheckpointRequestId { request_id }]) } + SyncControlRequest::ProbeLocalTargetOp { target_op } => { + let target_op = self.adapter.probe_local_target_op(target_op)?; + Ok(alloc::vec![Instruction::LocalTargetOp { target_op }]) + } SyncControlRequest::StopSyncStream => self.state.tear_down(), } } diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index 93a8a4e1..80d59208 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -457,6 +457,19 @@ void _syncTests({ expect(lastRequestedCheckpointRequestId(), 101); }); + syncTest('stores seeded checkpoint request ids verbatim', (_) { + invokeControlRaw('start', null); + invokeControlRaw('seed_checkpoint_request_id', 41); + expect(lastRequestedCheckpointRequestId(), 41); + + // Core does not enforce monotonicity when seeding. SDKs own reconciliation and seed the + // effective state accepted by the service, which may be below the local counter (e.g. after + // switching users). + invokeControlRaw('seed_checkpoint_request_id', 5); + expect(lastRequestedCheckpointRequestId(), 5); + expect(nextCheckpointRequestId(), 6); + }); + syncTest('requires checkpoint request state before allocating checkpoint ids', (_) { invokeControlRaw('start', null); From 39045f8b1ff14b0ab4737aa9bca627cf9250ace7 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 7 Jul 2026 11:33:00 +0200 Subject: [PATCH 12/29] cleanup errors --- crates/core/src/sync/interface.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/core/src/sync/interface.rs b/crates/core/src/sync/interface.rs index 1b5e8b26..e019f6da 100644 --- a/crates/core/src/sync/interface.rs +++ b/crates/core/src/sync/interface.rs @@ -395,9 +395,10 @@ fn parse_optional_i64_payload( let value = match payload.value_type() { ColumnType::Null => return Ok(None), ColumnType::Integer => payload.int64(), - ColumnType::Text => payload.text().parse::().map_err(|_| { - PowerSyncError::argument_error(format!("{name} must be an integer string")) - })?, + ColumnType::Text => payload + .text() + .parse::() + .map_err(|_| PowerSyncError::argument_error(type_error))?, _ => return Err(PowerSyncError::argument_error(type_error)), }; From c7254a49485bf130748a225b19fe8ec446bfab79 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 7 Jul 2026 12:51:17 +0200 Subject: [PATCH 13/29] update docs --- docs/write-checkpoint-requests.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/write-checkpoint-requests.md b/docs/write-checkpoint-requests.md index 3a8a7137..8b52752a 100644 --- a/docs/write-checkpoint-requests.md +++ b/docs/write-checkpoint-requests.md @@ -273,6 +273,12 @@ waitForSync() { The public database method requires an active or connecting sync client, because a disconnected request could not be delivered to the service or observed in the sync stream. +Waiters do not need durable applied state across reconnects or app restarts. The connect-time +counter reconciliation doubles as a re-request: the SDK posts the effective checkpoint request id +to the service on every connection attempt, so the next checkpoint carries a `write_checkpoint` +greater than or equal to any previously requested id and core emits a fresh +`CheckpointRequestApplied` instruction that resolves outstanding waits. + ## `ps_kv` checkpoint state - `local_target_op`: The current apply gate. It is either `MAX_OP_ID` while local writes are From 46cc29726b5a22bdb06836032456ed20582549bc Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 7 Jul 2026 13:31:04 +0200 Subject: [PATCH 14/29] docs updates --- crates/core/src/crud_vtab.rs | 9 +++---- crates/core/src/sync/interface.rs | 7 +++-- dart/test/crud_test.dart | 4 +-- dart/test/sync_test.dart | 4 +-- docs/schema.md | 7 +++-- docs/sync.md | 25 ++++++++++------- docs/write-checkpoint-requests.md | 45 +++++++++++++++++++++++++++---- 7 files changed, 74 insertions(+), 27 deletions(-) diff --git a/crates/core/src/crud_vtab.rs b/crates/core/src/crud_vtab.rs index 36936e3e..f0be98bc 100644 --- a/crates/core/src/crud_vtab.rs +++ b/crates/core/src/crud_vtab.rs @@ -248,11 +248,10 @@ impl SimpleCrudTransactionMode { fn record_local_write(&mut self, db: *mut sqlite::sqlite3) -> Result<(), ResultCode> { if !self.had_writes { // Also clear the seen/applied high-water marks: checkpoint request ids observed before - // this write can't acknowledge it, and they may come from an incompatible id namespace - // (legacy write checkpoints migrated by v14, or state from before a counter restart). - // Keeping them around could open the apply gate for a newly allocated target id that - // compares below a stale seen value. The legacy `$local` bookkeeping had the same - // behavior by resetting the entire row on local writes. + // this write can't acknowledge it, and stale values may predate a request counter + // restart. Keeping them around could open the apply gate for a newly allocated target + // id that compares below a stale seen value. The legacy `$local` bookkeeping had the + // same behavior by resetting the entire row on local writes. db.exec_safe(formatcp!( "INSERT OR REPLACE INTO ps_kv(key, value) VALUES('local_target_op', {MAX_OP_ID}); DELETE FROM ps_kv WHERE key IN ('last_seen_checkpoint_request_id', 'last_applied_checkpoint_request_id')" diff --git a/crates/core/src/sync/interface.rs b/crates/core/src/sync/interface.rs index e019f6da..94b40fad 100644 --- a/crates/core/src/sync/interface.rs +++ b/crates/core/src/sync/interface.rs @@ -143,8 +143,11 @@ pub enum Instruction { request: StreamingSyncRequest, /// The latest checkpoint request id known locally before opening this stream. /// - /// SDKs can use a missing value as a cue to fetch checkpoint request state from the service - /// and report it back with `seed_checkpoint_request_id`. + /// This is simply the client's current counter state. SDKs use it on every connection + /// attempt to re-affirm checkpoint request state with the service, which may have deleted + /// its record. The re-affirmation works bidirectionally: it can restore the service-side + /// value from this hint or bump the local counter from the service's response, which is + /// reported back with `seed_checkpoint_request_id`. last_checkpoint_request_id: Option, }, FetchCredentials { diff --git a/dart/test/crud_test.dart b/dart/test/crud_test.dart index 8d13c70a..3745f0fa 100644 --- a/dart/test/crud_test.dart +++ b/dart/test/crud_test.dart @@ -250,8 +250,8 @@ void main() { }); test('updates local target op and updated rows', () { - // Stale high-water marks (e.g. migrated legacy write checkpoints) must be cleared by a - // local write, so they can't open the apply gate for a smaller new target id. + // Stale high-water marks (e.g. from before a request counter restart) must be cleared + // by a local write, so they can't open the apply gate for a smaller new target id. db.execute(''' INSERT INTO ps_kv(key, value) VALUES ('last_seen_checkpoint_request_id', 6), diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index 80d59208..7178937c 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -611,8 +611,8 @@ void _syncTests({ expect(lastAppliedCheckpointRequestId(), 5); // A local write can only be acknowledged by a checkpoint request id observed after it. Stale - // seen/applied values (which may come from another id namespace, like migrated legacy write - // checkpoints) must not remain to open the apply gate for a smaller new target id. + // seen/applied values (which may predate a request counter restart) must not remain to open + // the apply gate for a smaller new target id. db.execute("insert into items (id, col) values ('local', 'data');"); expect(lastAppliedCheckpointRequestId(), isNull); diff --git a/docs/schema.md b/docs/schema.md index 4a6462b9..35b4fb5a 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -33,8 +33,11 @@ a checkpoint and that we have validated its checksum). Schema version 14 removes the legacy `target_op` column after migrating `$local.target_op` to `ps_kv.local_target_op`, and deletes the `$local` row so `ps_buckets` only contains real sync buckets. This makes older SDKs fail with a hard SQLite error if they try to keep using the migrated -database without downgrading. The down migration restores `target_op` and recreates the `$local` -row from `ps_kv` for older schema versions. +database without downgrading. That failure is deliberate — including for multi-process deployments +where processes with mixed SDK versions share one database — because an older SDK silently +maintaining `$local` state the new implementation no longer reads would be worse than a loud error. +The down migration restores `target_op` and recreates the `$local` row from `ps_kv` for older +schema versions. ## `ps_crud` diff --git a/docs/sync.md b/docs/sync.md index 8d1c2af3..300a2e77 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -47,9 +47,14 @@ The following commands are supported: ahead, or restore the service-side value when the service has cleared stale state but core still has a local hint. Then seed core with the reconciled value. Core stores the seeded value verbatim and does not enforce monotonicity; SDKs own the reconciliation and must not seed a - stale value. `NULL` means neither side has a record for the client yet; core stores `0` in that - case so the state counts as seeded and the first allocation returns `1`. If both the client and - service have lost the value, the counter may restart. + stale value. A `NULL` payload is accepted for completeness (core stores `0`, marking the state + as seeded so the first allocation returns `1`), but SDKs should not need it in practice: + posting a checkpoint request with an id of at least `1` during reconciliation and seeding the + service's response covers the no-record case and doubles as a probe of the service's + checkpoint-request support. Blindly forwarding a raw `NULL` service response while core holds a + counter would reset it, since the store is verbatim (see + `docs/write-checkpoint-requests.md`). If both the client and service have lost the value, the + counter may restart. When uploads request a write checkpoint, SDKs should call `powersync_control('next_checkpoint_request_id', NULL)` inside a transaction to allocate the id to @@ -78,7 +83,9 @@ only real sync buckets in `ps_buckets`, and drops `ps_buckets.target_op` so older SDKs fail hard if they try to keep using the migrated database directly. Downgrading restores the column, and restores a `$local` row only when `local_target_op` exists, so older SDKs can keep using target-op based blocking without inventing a synthetic local bucket when there was no local -target state. Because the down migration keeps the `ps_kv` keys around, the up migration clears +target state. A restored concrete target remains satisfiable after a downgrade because checkpoint +request ids and legacy write checkpoint ids share one namespace: the service reports accepted +checkpoint request ids as the `write_checkpoint` values older-protocol clients observe. Because the down migration keeps the `ps_kv` keys around, the up migration clears them before copying, so re-upgrading a downgraded database takes the `$local` row (including any progress an older SDK made) as the source of truth instead of failing on the existing keys. @@ -124,11 +131,11 @@ interface LogLine { } // Instructs client SDKs to open a connection to the sync service. -// last_checkpoint_request_id is core's local counter value before this stream request. On connect, -// SDKs can use it to re-request this client's last checkpoint request state from the service, then -// call powersync_control('seed_checkpoint_request_id', value) with the actual response for -// reconciliation. `value` may be null when the service has no checkpoint request state for this -// client. +// last_checkpoint_request_id is the client's current counter state before this stream request. +// On every connect, SDKs use it to re-affirm checkpoint request state with the service (which may +// have deleted its record). The re-affirmation is bidirectional: the hint can restore the +// service-side value, or the service's response can bump the local counter via +// powersync_control('seed_checkpoint_request_id', response). interface EstablishSyncStream { request: any // The JSON-encoded StreamingSyncRequest to send to the sync service last_checkpoint_request_id: null | number diff --git a/docs/write-checkpoint-requests.md b/docs/write-checkpoint-requests.md index 8b52752a..d5e64b52 100644 --- a/docs/write-checkpoint-requests.md +++ b/docs/write-checkpoint-requests.md @@ -49,8 +49,7 @@ WHERE key IN ('last_seen_checkpoint_request_id', 'last_applied_checkpoint_reques Clearing the high-water marks mirrors how the legacy flow reset the whole `$local` row on local writes. A checkpoint request id observed before the write cannot acknowledge it, and a stale seen -value may even come from an incompatible id namespace — a legacy write checkpoint migrated by v14, -or state from before a request counter restart. If such a value stayed around, a newly allocated +value may even predate a request counter restart. If such a value stayed around, a newly allocated target id could compare below it and open the apply gate before the service acknowledged the write. After a local write, only checkpoint request ids observed from that point on count towards the gate. @@ -154,9 +153,16 @@ local hint can restore the service-side value and keep the counter from moving b client lost local state but the service still has a record, the service response restores the seed locally. If both sides have lost the value, it is acceptable for the counter to restart; this can happen after local state is cleared and stale service state expires, or when multiple user ids share -the same client id. After reconciliation, `value` may be `NULL` when neither side has a record for -the client; core stores `0` in that case so the state counts as seeded and the first allocation -returns `1`. SDKs may also refresh service state when their user/client context changes. +the same client id. SDKs may also refresh service state when their user/client context changes. + +Because seeds are stored verbatim, seeding `NULL` (or a low id) while core holds a higher counter +resets that counter, and previously allocated ids would be handed out again. SDKs must therefore +never forward a raw service response without reconciling: the recommended pattern is to always post +an id of at least `1` (the maximum of the local hint and any concrete local target) during +reconciliation and seed core with the service's response to that request. In practice SDKs never +need to seed `NULL` at all — when there is no local record, posting a checkpoint request with id +`1` works and doubles as a probe of the service's checkpoint-request support. Core still accepts a +`NULL` seed for completeness. `powersync_control('next_checkpoint_request_id', NULL)` must be called inside a transaction during an active sync iteration after `last_requested_checkpoint_request_id` exists locally. It increments @@ -171,6 +177,14 @@ RETURNING value; This command only allocates an id. It does not update `local_target_op`. +Calling it without an active iteration or before seeding raises a state error. This is a normal +part of the connection lifecycle (for example a `requestCheckpoint` call racing a stream restart), +not a programming error — SDKs should surface it as a retryable condition. + +The increment participates in the caller's transaction. If the transaction rolls back after the id +was already posted to the service, the same id is allocated and posted again on retry; this is safe +because the service treats the latest posted id as the effective request state. + Note on sequences: SQLite does not have standalone sequences. The sequence-like alternatives are either an `AUTOINCREMENT` table backed by SQLite's internal `sqlite_sequence`, or a dedicated single-row counter table like the existing `ps_tx` transaction counter. The checkpoint request @@ -298,6 +312,12 @@ greater than or equal to any previously requested id and core emits a fresh migration/downgrade state and debugging; SDKs should use `CheckpointRequestApplied` instructions to resolve `CheckpointRequest` waits. +`powersync_clear` deletes all of these keys in both clear modes (it removes every `ps_kv` entry +except `client_id`). This is deliberate and mirrors the legacy behavior of deleting the `$local` +row: pending CRUD is wiped in the same operation, so no apply gate is needed, and the request +counter is restored by the connect-time reconciliation described above. If the service has also +lost its record, the counter restarting is acceptable. + ## Migration from `$local` Migration v14 moves the old `$local` bucket state into `ps_kv`: @@ -344,3 +364,18 @@ The down migration restores `ps_buckets.target_op` and rebuilds a `$local` row o This keeps older SDKs able to use the historic target-op gate after a downgrade without inventing a synthetic `$local` bucket when there was no local target state. + +Two properties make the restored gate safe rather than a potential stall: + +- **Shared id namespace.** Client-created checkpoint request ids and legacy write checkpoint ids + are one namespace, compatible in both directions — including values migrated from the historic + service-generated write checkpoint scheme. The service reports the checkpoint request ids it + accepted as the `write_checkpoint` values older-protocol clients observe, so a restored concrete + `$local.target_op` is satisfiable by the next write checkpoint the downgraded SDK sees; it does + not wait on an incomparable id sequence. +- **Downgrade fidelity.** `last_seen_checkpoint_request_id` and + `last_applied_checkpoint_request_id` only advance on full checkpoint completions — but so did the + legacy `$local.last_op`/`last_applied_op` bookkeeping (partial priority applies never updated + `$local`, which is not a real bucket). The rebuilt `$local` row therefore matches exactly what a + legacy client would have recorded at the same point in the stream; the down migration cannot lag + behind legacy behavior. From caf5370b2e57bd2eb1c886920d7cec6ca102920c Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 7 Jul 2026 13:42:14 +0200 Subject: [PATCH 15/29] cleanup Result types and DB operations. --- crates/core/src/sync/streaming_sync.rs | 83 ++++++++++++++------------ 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/crates/core/src/sync/streaming_sync.rs b/crates/core/src/sync/streaming_sync.rs index bd91eef9..031272f7 100644 --- a/crates/core/src/sync/streaming_sync.rs +++ b/crates/core/src/sync/streaming_sync.rs @@ -95,24 +95,21 @@ impl SyncClient { SyncControlRequest::SyncEvent(sync_event) => { let mut active = ActiveEvent::new(sync_event); - let run_result = { - let ClientState::IterationActive(handle) = &mut self.state else { - return Err(PowerSyncError::state_error("No iteration is active")); - }; - - handle.run(&mut active) + let ClientState::IterationActive(handle) = &mut self.state else { + return Err(PowerSyncError::state_error("No iteration is active")); }; - match run_result { + match handle.run(&mut active) { Err(e) => { self.state = ClientState::Idle; return Err(e); } - Ok(true) => { - self.state = ClientState::Idle; + Ok(done) => { + if done { + self.state = ClientState::Idle; + } } - Ok(false) => {} - } + }; if let Some(recoverable) = active.recoverable_error.take() { Err(recoverable) @@ -220,18 +217,18 @@ impl SyncIterationHandle { }; let mut context = Context::from_waker(&waker); - let done = if let Poll::Ready(result) = self.future.poll(&mut context) { - let close = result?; + Ok( + if let Poll::Ready(result) = self.future.poll(&mut context) { + let close = result?; - active - .instructions - .push(Instruction::CloseSyncStream(close)); - true - } else { - false - }; - - Ok(done) + active + .instructions + .push(Instruction::CloseSyncStream(close)); + true + } else { + false + }, + ) } } @@ -384,6 +381,13 @@ impl StreamingSyncIteration { }); event.instructions.push(Instruction::FlushFileSystem {}); + // Persist here so that all database writes happen while preparing the + // transition, keeping apply_transition infallible. + if let Some(request_id) = target.write_checkpoint { + self.adapter + .persist_last_applied_checkpoint_request_id(request_id)?; + } + SyncStateMachineTransition::SyncLocalChangesApplied { applied_checkpoint_request_id: target.write_checkpoint, partial: None, @@ -482,7 +486,7 @@ impl StreamingSyncIteration { target: &mut SyncTarget, event: &mut ActiveEvent, transition: SyncStateMachineTransition, - ) -> Result, PowerSyncError> { + ) -> Option { match transition { SyncStateMachineTransition::StartTrackingCheckpoint { progress, @@ -516,7 +520,7 @@ impl StreamingSyncIteration { diagnostics.handle_data_line(line, &*status, &mut event.instructions); } } - SyncStateMachineTransition::CloseIteration(close) => return Ok(Some(close)), + SyncStateMachineTransition::CloseIteration(close) => return Some(close), SyncStateMachineTransition::SyncLocalFailedDueToPendingCrud { validated_but_not_applied, } => { @@ -535,17 +539,13 @@ impl StreamingSyncIteration { &mut event.instructions, ); } else { - self.handle_checkpoint_applied( - event, - timestamp, - applied_checkpoint_request_id, - )?; + self.handle_checkpoint_applied(event, timestamp, applied_checkpoint_request_id); } } SyncStateMachineTransition::Empty => {} }; - Ok(None) + None } /// Handles a single sync line. @@ -559,7 +559,7 @@ impl StreamingSyncIteration { line: &SyncLineWithSource, ) -> Result, PowerSyncError> { let transition = self.prepare_handling_sync_line(target, event, line)?; - self.apply_transition(target, event, transition) + Ok(self.apply_transition(target, event, transition)) } /// Runs a full sync iteration, returning nothing when it completes regularly or an error when @@ -684,7 +684,11 @@ impl StreamingSyncIteration { line: "Applied pending checkpoint after completed upload".into(), }); - self.handle_checkpoint_applied(event, timestamp, checkpoint.write_checkpoint)?; + if let Some(request_id) = checkpoint.write_checkpoint { + self.adapter + .persist_last_applied_checkpoint_request_id(request_id)?; + } + self.handle_checkpoint_applied(event, timestamp, checkpoint.write_checkpoint); } _ => { event.instructions.push(Instruction::LogLine { @@ -923,11 +927,11 @@ impl StreamingSyncIteration { /// subscriptions, used to associate [BucketSubscriptionReason::DerivedFromExplicitSubscription]. async fn prepare_request(&mut self) -> Result { let event = Self::receive_event().await; - if !matches!(&event.event, SyncEvent::Initialize) { + let SyncEvent::Initialize = event.event else { return Err(PowerSyncError::argument_error( "first event must initialize", )); - } + }; let offline_state = self.adapter.offline_sync_state()?; self.status.update( @@ -969,15 +973,18 @@ impl StreamingSyncIteration { }) } + /// Emits the instructions and status update for a fully applied checkpoint. + /// + /// The applied checkpoint request id must already have been persisted by the caller: this + /// runs while applying a state transition, which must stay infallible (see + /// [SyncStateMachineTransition]). fn handle_checkpoint_applied( &mut self, event: &mut ActiveEvent, timestamp: TimestampMicros, applied_checkpoint_request_id: Option, - ) -> Result<(), PowerSyncError> { + ) { if let Some(request_id) = applied_checkpoint_request_id { - self.adapter - .persist_last_applied_checkpoint_request_id(request_id)?; event .instructions .push(Instruction::CheckpointRequestApplied { request_id }); @@ -989,8 +996,6 @@ impl StreamingSyncIteration { |status| status.applied_checkpoint(timestamp), &mut event.instructions, ); - - Ok(()) } } From c86ef3b1394d876de16f3d6e7841eea561e27584 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 7 Jul 2026 14:34:35 +0200 Subject: [PATCH 16/29] AI comments --- crates/core/src/migrations.rs | 3 +++ crates/core/src/sync/storage_adapter.rs | 3 ++- crates/core/src/view_admin.rs | 4 +++- dart/test/sync_test.dart | 14 ++++++++++++++ 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/crates/core/src/migrations.rs b/crates/core/src/migrations.rs index afc1ae5a..7139b5ca 100644 --- a/crates/core/src/migrations.rs +++ b/crates/core/src/migrations.rs @@ -488,6 +488,9 @@ DROP TABLE ps_sync_state_old; // After copying, the `$local` row is deleted: version 14 tracks this state exclusively in // ps_kv, so ps_buckets only contains real sync buckets. The down migration recreates the // row from ps_kv when needed. + // + // DROP COLUMN requires SQLite 3.35+; the extension already refuses to load below + // MIN_SQLITE_VERSION_NUMBER (3.44), so this is safe in the up path. let up = "\ DELETE FROM ps_kv WHERE key IN ( diff --git a/crates/core/src/sync/storage_adapter.rs b/crates/core/src/sync/storage_adapter.rs index 9081faff..857488e2 100644 --- a/crates/core/src/sync/storage_adapter.rs +++ b/crates/core/src/sync/storage_adapter.rs @@ -528,7 +528,8 @@ WHERE bucket = ?1", /// `next_checkpoint_request_id`. /// /// Returns the target op value from before this call. When `target_op` is `None`, this only - /// reads the current value. + /// reads the current value. A `target_op` of zero clears the stored target, removing the apply + /// gate entirely; any other value overwrites it. /// /// Negative values are rejected when parsing the `powersync_control` payload, before this is /// called. diff --git a/crates/core/src/view_admin.rs b/crates/core/src/view_admin.rs index ce20bd0d..df6156e7 100644 --- a/crates/core/src/view_admin.rs +++ b/crates/core/src/view_admin.rs @@ -163,7 +163,9 @@ fn trigger_resync(db: *mut sqlite::sqlite3, state: &DatabaseState) -> Result<(), } } - db.exec_safe("UPDATE ps_buckets SET last_applied_op = 0 WHERE name != '$local'") + // Since migration v14, ps_buckets only contains real sync buckets: the synthetic `$local` + // bucket has moved to ps_kv, so no filter is needed here. + db.exec_safe("UPDATE ps_buckets SET last_applied_op = 0") .into_db_result(db)?; Ok(Default::default()) } diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index 7178937c..fea0a419 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -468,6 +468,20 @@ void _syncTests({ invokeControlRaw('seed_checkpoint_request_id', 5); expect(lastRequestedCheckpointRequestId(), 5); expect(nextCheckpointRequestId(), 6); + + // A NULL seed is stored as 0, restarting the counter. Forwarding a raw NULL service response + // while a counter exists resets it - SDKs must reconcile before seeding. + invokeControlRaw('seed_checkpoint_request_id', null); + expect(lastRequestedCheckpointRequestId(), 0); + expect(nextCheckpointRequestId(), 1); + }); + + syncTest('accepts text checkpoint request ids when seeding', (_) { + invokeControlRaw('start', null); + invokeControlRaw('seed_checkpoint_request_id', '41'); + + expect(lastRequestedCheckpointRequestId(), 41); + expect(nextCheckpointRequestId(), 42); }); syncTest('requires checkpoint request state before allocating checkpoint ids', From 5c339ff7b186aa325cb3bce76e5afddf8141544a Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 8 Jul 2026 09:27:01 +0200 Subject: [PATCH 17/29] merge DidCompleteSync and CheckpointRequestApplied --- crates/core/src/sync/interface.rs | 8 +++++--- crates/core/src/sync/storage_adapter.rs | 4 ++-- crates/core/src/sync/streaming_sync.rs | 10 +++------- dart/test/sync_test.dart | 13 +++++++------ docs/sync.md | 14 ++++++-------- docs/write-checkpoint-requests.md | 19 ++++++++++--------- 6 files changed, 33 insertions(+), 35 deletions(-) diff --git a/crates/core/src/sync/interface.rs b/crates/core/src/sync/interface.rs index 94b40fad..0b74a3e3 100644 --- a/crates/core/src/sync/interface.rs +++ b/crates/core/src/sync/interface.rs @@ -158,8 +158,6 @@ pub enum Instruction { }, /// Return a newly allocated checkpoint request id to the SDK. CheckpointRequestId { request_id: i64 }, - /// Notify the SDK that a checkpoint request id has been applied locally. - CheckpointRequestApplied { request_id: i64 }, /// Return the local target op value observed before an optional update. LocalTargetOp { target_op: Option }, // These are defined like this because deserializers in Kotlin can't support either an @@ -169,7 +167,11 @@ pub enum Instruction { /// Flush the file-system if it's non-durable (only applicable to the Dart SDK). FlushFileSystem {}, /// Notify that a sync has been completed, prompting client SDKs to clear earlier errors. - DidCompleteSync {}, + DidCompleteSync { + /// The checkpoint request id applied by this completed sync, if the checkpoint had one. + #[serde(skip_serializing_if = "Option::is_none")] + applied_checkpoint_request_id: Option, + }, /// Handle a diagnostic event. /// diff --git a/crates/core/src/sync/storage_adapter.rs b/crates/core/src/sync/storage_adapter.rs index 857488e2..a41a0e1e 100644 --- a/crates/core/src/sync/storage_adapter.rs +++ b/crates/core/src/sync/storage_adapter.rs @@ -568,8 +568,8 @@ WHERE bucket = ?1", /// /// This is always the id from the last applied checkpoint as sent by the service - a plain /// overwrite with no monotonicity enforced by core. External code owns consistency of these - /// ids; waiters should rely on `CheckpointRequestApplied` instructions rather than comparing - /// this value across reconnects. + /// ids; waiters should rely on `DidCompleteSync.applied_checkpoint_request_id` rather than + /// comparing this value across reconnects. pub fn persist_last_applied_checkpoint_request_id( &self, request_id: i64, diff --git a/crates/core/src/sync/streaming_sync.rs b/crates/core/src/sync/streaming_sync.rs index 031272f7..d88ef413 100644 --- a/crates/core/src/sync/streaming_sync.rs +++ b/crates/core/src/sync/streaming_sync.rs @@ -984,13 +984,9 @@ impl StreamingSyncIteration { timestamp: TimestampMicros, applied_checkpoint_request_id: Option, ) { - if let Some(request_id) = applied_checkpoint_request_id { - event - .instructions - .push(Instruction::CheckpointRequestApplied { request_id }); - } - - event.instructions.push(Instruction::DidCompleteSync {}); + event.instructions.push(Instruction::DidCompleteSync { + applied_checkpoint_request_id, + }); self.status.update( |status| status.applied_checkpoint(timestamp), diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index fea0a419..b44ed58b 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -372,8 +372,8 @@ void _syncTests({ pushCheckpointComplete(), contains( containsPair( - 'CheckpointRequestApplied', - {'request_id': 1}, + 'DidCompleteSync', + {'applied_checkpoint_request_id': 1}, ), ), ); @@ -570,7 +570,8 @@ void _syncTests({ expect( instructions, - isNot(contains(containsPair('CheckpointRequestApplied', anything))), + isNot(contains(containsPair('DidCompleteSync', + containsPair('applied_checkpoint_request_id', anything)))), ); expect(lastAppliedCheckpointRequestId(), isNull); @@ -592,8 +593,8 @@ void _syncTests({ appliedInstructions, contains( containsPair( - 'CheckpointRequestApplied', - {'request_id': 1}, + 'DidCompleteSync', + {'applied_checkpoint_request_id': 1}, ), ), ); @@ -604,7 +605,7 @@ void _syncTests({ expect( instructions, - isNot(contains(containsPair('CheckpointRequestApplied', anything))), + contains(containsPair('DidCompleteSync', {})), ); final [row] = db.select('select powersync_offline_sync_status();'); diff --git a/docs/sync.md b/docs/sync.md index 300a2e77..efbf633f 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -93,7 +93,7 @@ progress an older SDK made) as the source of truth instead of failing on the exi `next_checkpoint_request_id` to allocate increasing ids for client-created checkpoint requests. `last_seen_checkpoint_request_id` and `last_applied_checkpoint_request_id` are high-water marks that local writes clear, so only checkpoint request ids observed after a write count towards the -apply gate. SDKs should use `CheckpointRequestApplied` instructions for explicit checkpoint +apply gate. SDKs should use `DidCompleteSync.applied_checkpoint_request_id` for explicit checkpoint request waits instead of presenting these values as meaningful sync progress. If `local_target_op` is absent after migration, there is no local write gate waiting for a @@ -115,15 +115,15 @@ type Instruction = { LogLine: LogLine } | { EstablishSyncStream: EstablishSyncStream } | { FetchCredentials: FetchCredentials } | { CheckpointRequestId: { request_id: number } } - | { CheckpointRequestApplied: { request_id: number } } | { LocalTargetOp: { target_op: null | number } } // Close a connection previously started after EstablishSyncStream | { CloseSyncStream: { hide_disconnect: boolean } } // For the Dart web client, flush the (otherwise non-durable) file system. | { FlushFileSystem: {} } // Notify clients that a checkpoint was completed. Clients can clear the - // download error state in response to this. - | { DidCompleteSync: {} } + // download error state in response to this. If a full checkpoint with a + // write_checkpoint was applied, applied_checkpoint_request_id is set. + | { DidCompleteSync: DidCompleteSync } interface LogLine { severity: 'DEBUG' | 'INFO' | 'WARNING', @@ -150,10 +150,8 @@ interface UpdateSyncStatus { streams: [], } -// Emitted when a full checkpoint with a write_checkpoint has been applied locally. -// SDKs can use this to resolve pending CheckpointRequest waiters. -interface CheckpointRequestApplied { - request_id: number, +interface DidCompleteSync { + applied_checkpoint_request_id?: number, } // Instructs SDKs to refresh credentials from the backend connector. diff --git a/docs/write-checkpoint-requests.md b/docs/write-checkpoint-requests.md index d5e64b52..b4911373 100644 --- a/docs/write-checkpoint-requests.md +++ b/docs/write-checkpoint-requests.md @@ -16,7 +16,8 @@ At a high level: These keys are internal SDK/core state, not user-facing sync progress. `last_requested_checkpoint_request_id` is functional allocation state, while `last_seen_checkpoint_request_id` and `last_applied_checkpoint_request_id` are mostly diagnostic -high-water marks. Explicit checkpoint waits should follow `CheckpointRequestApplied` instructions. +high-water marks. Explicit checkpoint waits should follow +`DidCompleteSync.applied_checkpoint_request_id`. SDKs should not write these keys directly. They update the local target through `powersync_control('local_target_op', value)`, which is the shared helper for both legacy write @@ -257,12 +258,12 @@ on completed_upload: ``` After a full checkpoint applies, core stores the applied checkpoint request id as -`last_applied_checkpoint_request_id` and emits a `CheckpointRequestApplied` instruction. +`last_applied_checkpoint_request_id` and emits it on the `DidCompleteSync` instruction. ```text after full checkpoint apply: ps_kv['last_applied_checkpoint_request_id'] = checkpoint.write_checkpoint - emit CheckpointRequestApplied { request_id: checkpoint.write_checkpoint } + emit DidCompleteSync { applied_checkpoint_request_id: checkpoint.write_checkpoint } ``` ## Explicit checkpoint requests @@ -272,13 +273,13 @@ database has caught up to the service. The SDK creates a checkpoint request id t sync client and returns a `CheckpointRequest`-style waiter. This explicit API does not update `local_target_op`: it is a wait marker, not a local upload gate. -The returned object waits until core emits `CheckpointRequestApplied` for an id greater than or -equal to the requested id. +The returned object waits until core emits `DidCompleteSync` with +`applied_checkpoint_request_id` greater than or equal to the requested id. ```text waitForSync() { for instruction in syncInstructions { - return when instruction.CheckpointRequestApplied.request_id >= requestId + return when instruction.DidCompleteSync.applied_checkpoint_request_id >= requestId throw if sync status reports a sync error } } @@ -291,7 +292,7 @@ Waiters do not need durable applied state across reconnects or app restarts. The counter reconciliation doubles as a re-request: the SDK posts the effective checkpoint request id to the service on every connection attempt, so the next checkpoint carries a `write_checkpoint` greater than or equal to any previously requested id and core emits a fresh -`CheckpointRequestApplied` instruction that resolves outstanding waits. +`DidCompleteSync.applied_checkpoint_request_id` that resolves outstanding waits. ## `ps_kv` checkpoint state @@ -309,8 +310,8 @@ greater than or equal to any previously requested id and core emits a fresh checkpoint request ids observed after the write can satisfy the apply gate. - `last_applied_checkpoint_request_id`: The latest full checkpoint `write_checkpoint` that has been applied locally since the last local write, which clears this key. Core persists this for - migration/downgrade state and debugging; SDKs should use `CheckpointRequestApplied` instructions - to resolve `CheckpointRequest` waits. + migration/downgrade state and debugging; SDKs should use + `DidCompleteSync.applied_checkpoint_request_id` to resolve `CheckpointRequest` waits. `powersync_clear` deletes all of these keys in both clear modes (it removes every `ps_kv` entry except `client_id`). This is deliberate and mirrors the legacy behavior of deleting the `$local` From b6ab6d853a9103f7abd2abfcec7aeeaf2f91cc0b Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 8 Jul 2026 09:40:03 +0200 Subject: [PATCH 18/29] cleanup comments for --- crates/core/src/crud_vtab.rs | 7 +++---- crates/core/src/schema/raw_table.rs | 3 +-- crates/core/src/sync/interface.rs | 2 ++ crates/core/src/sync/storage_adapter.rs | 10 +++++----- crates/core/src/view_admin.rs | 2 -- dart/test/sync_test.dart | 3 +-- 6 files changed, 12 insertions(+), 15 deletions(-) diff --git a/crates/core/src/crud_vtab.rs b/crates/core/src/crud_vtab.rs index f0be98bc..7d2acfe4 100644 --- a/crates/core/src/crud_vtab.rs +++ b/crates/core/src/crud_vtab.rs @@ -29,7 +29,7 @@ const SIMPLE_NAME: &CStr = c"powersync_crud"; // ps_crud(tx_id, data). // The second form (without the trailing underscore) takes the data to insert as individual // components and constructs the data to insert into `ps_crud` internally. It will also update -// `ps_updated_rows` and the `$local` bucket. +// `ps_updated_rows` and the local write apply gate. // // Using a virtual table like this allows us to hook into xBegin, xCommit and xRollback to automatically // increment transaction ids. These are only called when powersync_crud_ is used as part of a transaction, @@ -166,7 +166,7 @@ impl VirtualTable { stmt.bind_text(2, &serialized, sqlite::Destructor::STATIC)?; stmt.exec()?; - // However, we also set ps_updated_rows and update the $local bucket + // However, we also set ps_updated_rows and mark the local write state. let set_updated_rows = simple.set_updated_rows_statement(db)?; set_updated_rows.bind_text(1, row_type, sqlite::Destructor::STATIC)?; set_updated_rows.bind_text(2, id, sqlite::Destructor::STATIC)?; @@ -250,8 +250,7 @@ impl SimpleCrudTransactionMode { // Also clear the seen/applied high-water marks: checkpoint request ids observed before // this write can't acknowledge it, and stale values may predate a request counter // restart. Keeping them around could open the apply gate for a newly allocated target - // id that compares below a stale seen value. The legacy `$local` bookkeeping had the - // same behavior by resetting the entire row on local writes. + // id that compares below a stale seen value. db.exec_safe(formatcp!( "INSERT OR REPLACE INTO ps_kv(key, value) VALUES('local_target_op', {MAX_OP_ID}); DELETE FROM ps_kv WHERE key IN ('last_seen_checkpoint_request_id', 'last_applied_checkpoint_request_id')" diff --git a/crates/core/src/schema/raw_table.rs b/crates/core/src/schema/raw_table.rs index ee06501b..23144ca7 100644 --- a/crates/core/src/schema/raw_table.rs +++ b/crates/core/src/schema/raw_table.rs @@ -267,8 +267,7 @@ pub fn generate_raw_table_trigger( // Prevent illegal writes to a table marked as insert-only by raising errors here. buffer.push_str("SELECT RAISE(FAIL, 'Unexpected update on insert-only table');\n"); } else { - // Write directly to powersync_crud_ to skip writing the $local bucket for insert-only - // tables. + // Insert-only tables use manual CRUD writes so they don't block incoming data. let fragment = table_columns_to_json_object("NEW", &as_schema_table)?; buffer.powersync_crud_manual_put(&table.name, &fragment); } diff --git a/crates/core/src/sync/interface.rs b/crates/core/src/sync/interface.rs index 0b74a3e3..88f6b358 100644 --- a/crates/core/src/sync/interface.rs +++ b/crates/core/src/sync/interface.rs @@ -400,6 +400,8 @@ fn parse_optional_i64_payload( let value = match payload.value_type() { ColumnType::Null => return Ok(None), ColumnType::Integer => payload.int64(), + // Allow decimal strings as a fallback for JavaScript SQLite drivers that can't bind + // 64-bit integers as BigInt without losing precision through Number. ColumnType::Text => payload .text() .parse::() diff --git a/crates/core/src/sync/storage_adapter.rs b/crates/core/src/sync/storage_adapter.rs index a41a0e1e..ea43a981 100644 --- a/crates/core/src/sync/storage_adapter.rs +++ b/crates/core/src/sync/storage_adapter.rs @@ -77,9 +77,10 @@ impl StorageAdapter { pub fn collect_bucket_requests(&self) -> Result, PowerSyncError> { // language=SQLite - let statement = self.db.prepare_v2( - "SELECT name, last_op FROM ps_buckets WHERE pending_delete = 0 AND name != '$local'", - ).into_db_result(self.db)?; + let statement = self + .db + .prepare_v2("SELECT name, last_op FROM ps_buckets WHERE pending_delete = 0") + .into_db_result(self.db)?; let mut requests = Vec::::new(); @@ -555,8 +556,7 @@ WHERE bucket = ?1", /// Persists the checkpoint request id observed in a complete sync checkpoint. /// - /// This replaces the legacy `$local.last_op` bookkeeping used to decide whether downloaded - /// data can be applied after local uploads complete. + /// This is used to decide whether downloaded data can be applied after local uploads complete. pub fn persist_last_seen_checkpoint_request_id( &self, request_id: i64, diff --git a/crates/core/src/view_admin.rs b/crates/core/src/view_admin.rs index df6156e7..3ef012b6 100644 --- a/crates/core/src/view_admin.rs +++ b/crates/core/src/view_admin.rs @@ -163,8 +163,6 @@ fn trigger_resync(db: *mut sqlite::sqlite3, state: &DatabaseState) -> Result<(), } } - // Since migration v14, ps_buckets only contains real sync buckets: the synthetic `$local` - // bucket has moved to ps_kv, so no filter is needed here. db.exec_safe("UPDATE ps_buckets SET last_applied_op = 0") .into_db_result(db)?; Ok(Default::default()) diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index b44ed58b..5bb1ae08 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -784,7 +784,7 @@ void _syncTests({ }); test('deletes old buckets', () { - for (final name in ['one', 'two', 'three', r'$local']) { + for (final name in ['one', 'two', 'three']) { db.execute('INSERT INTO ps_buckets (name) VALUES (?)', [name]); } @@ -816,7 +816,6 @@ void _syncTests({ // Should delete the old buckets two and three expect(db.select('select name from ps_buckets order by id'), [ {'name': 'one'}, - {'name': r'$local'} ]); }); From 7481d3edf0a16851b1ba51ea8479f6f2db7ca667 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 8 Jul 2026 09:46:41 +0200 Subject: [PATCH 19/29] prevent seeding checkpoint request sequence with null --- crates/core/src/sync/interface.rs | 24 ++++++++++-- crates/core/src/sync/storage_adapter.rs | 13 +------ dart/test/error_test.dart | 2 +- dart/test/goldens/simple_iteration.json | 2 +- dart/test/goldens/starting_stream.json | 2 +- dart/test/migration_test.dart | 2 +- dart/test/sync_local_performance_test.dart | 2 +- dart/test/sync_stream_test.dart | 5 +-- dart/test/sync_test.dart | 44 ++++++++++++++++------ docs/sync.md | 16 +++----- docs/write-checkpoint-requests.md | 15 ++++---- 11 files changed, 75 insertions(+), 52 deletions(-) diff --git a/crates/core/src/sync/interface.rs b/crates/core/src/sync/interface.rs index 88f6b358..0be322e8 100644 --- a/crates/core/src/sync/interface.rs +++ b/crates/core/src/sync/interface.rs @@ -98,7 +98,7 @@ pub enum SyncEvent<'a> { DidRefreshToken, /// Seeds the checkpoint request counter from service state. SeedCheckpointRequestId { - request_id: Option, + request_id: i64, }, /// Notifies the sync client that the current CRUD upload (for which the client SDK is /// responsible) has finished. @@ -290,10 +290,10 @@ pub fn register(db: *mut sqlite::sqlite3, state: Rc) -> Result<() "refreshed_token" => SyncControlRequest::SyncEvent(SyncEvent::DidRefreshToken), "seed_checkpoint_request_id" => { SyncControlRequest::SyncEvent(SyncEvent::SeedCheckpointRequestId { - request_id: parse_optional_i64_payload( + request_id: parse_positive_i64_payload( *payload, "checkpoint request id", - "checkpoint request id must be an integer, integer string, or null", + "checkpoint request id must be an integer or integer string", )?, }) } @@ -417,3 +417,21 @@ fn parse_optional_i64_payload( Ok(Some(value)) } + +fn parse_positive_i64_payload( + payload: *mut sqlite::value, + name: &'static str, + type_error: &'static str, +) -> Result { + let Some(value) = parse_optional_i64_payload(payload, name, type_error)? else { + return Err(PowerSyncError::argument_error(type_error)); + }; + + if value == 0 { + return Err(PowerSyncError::argument_error(format!( + "{name} must be a positive integer" + ))); + } + + Ok(value) +} diff --git a/crates/core/src/sync/storage_adapter.rs b/crates/core/src/sync/storage_adapter.rs index ea43a981..efbaead0 100644 --- a/crates/core/src/sync/storage_adapter.rs +++ b/crates/core/src/sync/storage_adapter.rs @@ -613,17 +613,8 @@ RETURNING value", /// The value is stored verbatim: core does not enforce monotonicity here. SDKs are /// responsible for reconciling their local hint with the service before seeding, and cannot /// allocate new checkpoint request ids until that seeding has completed. - /// - /// A null service value means the service has no record for this client yet. Store zero in - /// that case so the first local allocation returns one while still marking the state as seeded. - pub fn seed_checkpoint_request_id( - &self, - request_id: Option, - ) -> Result<(), PowerSyncError> { - self.write_i64_kv( - LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY, - request_id.unwrap_or(0), - ) + pub fn seed_checkpoint_request_id(&self, request_id: i64) -> Result<(), PowerSyncError> { + self.write_i64_kv(LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY, request_id) } fn read_i64_kv(&self, key: &'static str) -> Result, PowerSyncError> { diff --git a/dart/test/error_test.dart b/dart/test/error_test.dart index 305a91f4..7ac7c800 100644 --- a/dart/test/error_test.dart +++ b/dart/test/error_test.dart @@ -78,7 +78,7 @@ void main() { final control = db.prepare(stmt); control.execute(['start', null]); - control.execute(['seed_checkpoint_request_id', null]); + control.execute(['seed_checkpoint_request_id', 1]); expect( () => control.execute(['line_text', 'invalid sync line']), throwsA(isSqliteException( diff --git a/dart/test/goldens/simple_iteration.json b/dart/test/goldens/simple_iteration.json index e8eb7ff6..4df0ff79 100644 --- a/dart/test/goldens/simple_iteration.json +++ b/dart/test/goldens/simple_iteration.json @@ -35,7 +35,7 @@ }, { "operation": "seed_checkpoint_request_id", - "data": null, + "data": 1, "output": [] }, { diff --git a/dart/test/goldens/starting_stream.json b/dart/test/goldens/starting_stream.json index b63f2a03..0c26397b 100644 --- a/dart/test/goldens/starting_stream.json +++ b/dart/test/goldens/starting_stream.json @@ -41,7 +41,7 @@ }, { "operation": "seed_checkpoint_request_id", - "data": null, + "data": 1, "output": [] } ] diff --git a/dart/test/migration_test.dart b/dart/test/migration_test.dart index 514cf4da..a65c72c3 100644 --- a/dart/test/migration_test.dart +++ b/dart/test/migration_test.dart @@ -347,7 +347,7 @@ UPDATE ps_buckets db.execute('begin'); control('start'); - control('seed_checkpoint_request_id'); + control('seed_checkpoint_request_id', 1); control( 'line_text', json.encode(checkpoint(lastOpId: 3, buckets: [ diff --git a/dart/test/sync_local_performance_test.dart b/dart/test/sync_local_performance_test.dart index 81546158..ed5d5b5b 100644 --- a/dart/test/sync_local_performance_test.dart +++ b/dart/test/sync_local_performance_test.dart @@ -100,7 +100,7 @@ COMMIT; // Start a fake sync client to apply the changes we've already written to // ps_oplog control('start'); - control('seed_checkpoint_request_id'); + control('seed_checkpoint_request_id', 1); final lastOpid = db.select('select max(op_id) from ps_oplog').single.columnAt(0) as int; final allBuckets = db diff --git a/dart/test/sync_stream_test.dart b/dart/test/sync_stream_test.dart index 21c02561..0e86f488 100644 --- a/dart/test/sync_stream_test.dart +++ b/dart/test/sync_stream_test.dart @@ -62,8 +62,7 @@ void main() { bool establishesSyncStream(List instructions) { return instructions.any((instruction) => - instruction is Map && - instruction.containsKey('EstablishSyncStream')); + instruction is Map && instruction.containsKey('EstablishSyncStream')); } List control(String operation, Object? data) { @@ -71,7 +70,7 @@ void main() { if (operation == 'start' && establishesSyncStream(result)) { return [ ...result, - ...controlRaw('seed_checkpoint_request_id', null), + ...controlRaw('seed_checkpoint_request_id', 1), ]; } return result; diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index 5bb1ae08..5f0ece47 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -79,8 +79,8 @@ void _syncTests({ if (operation == 'start' && establishesSyncStream(result)) { final seedResult = matcher.enabled - ? matcher.invoke('seed_checkpoint_request_id', null) - : invokeControlRaw('seed_checkpoint_request_id', null); + ? matcher.invoke('seed_checkpoint_request_id', 1) + : invokeControlRaw('seed_checkpoint_request_id', 1); return [...result, ...seedResult]; } @@ -233,7 +233,7 @@ void _syncTests({ 'app_metadata': {'key1': 'value1', 'key2': 'value2'} }), ), - ...invokeControlRaw('seed_checkpoint_request_id', null), + ...invokeControlRaw('seed_checkpoint_request_id', 1), ]; expect( @@ -430,11 +430,11 @@ void _syncTests({ syncTest('allocates requested checkpoint request ids', (_) { invokeControl('start', null); - expect(nextCheckpointRequestId(), 1); - expect(lastRequestedCheckpointRequestId(), 1); - expect(nextCheckpointRequestId(), 2); expect(lastRequestedCheckpointRequestId(), 2); + + expect(nextCheckpointRequestId(), 3); + expect(lastRequestedCheckpointRequestId(), 3); }); syncTest('seeds requested checkpoint request ids from service state', (_) { @@ -468,12 +468,32 @@ void _syncTests({ invokeControlRaw('seed_checkpoint_request_id', 5); expect(lastRequestedCheckpointRequestId(), 5); expect(nextCheckpointRequestId(), 6); + }); - // A NULL seed is stored as 0, restarting the counter. Forwarding a raw NULL service response - // while a counter exists resets it - SDKs must reconcile before seeding. - invokeControlRaw('seed_checkpoint_request_id', null); - expect(lastRequestedCheckpointRequestId(), 0); - expect(nextCheckpointRequestId(), 1); + syncTest('rejects absent checkpoint request ids when seeding', (_) { + invokeControlRaw('start', null); + + expect( + () => invokeControlRaw('seed_checkpoint_request_id', null), + throwsA(isSqliteException( + 3091, + contains('checkpoint request id must be an integer or integer string'), + )), + ); + expect(lastRequestedCheckpointRequestId(), isNull); + }); + + syncTest('rejects zero checkpoint request ids when seeding', (_) { + invokeControlRaw('start', null); + + expect( + () => invokeControlRaw('seed_checkpoint_request_id', 0), + throwsA(isSqliteException( + 3091, + contains('checkpoint request id must be a positive integer'), + )), + ); + expect(lastRequestedCheckpointRequestId(), isNull); }); syncTest('accepts text checkpoint request ids when seeding', (_) { @@ -1104,7 +1124,7 @@ void _syncTests({ db.execute('DELETE FROM ps_crud'); probeLocalTargetOp(1); expect(invokeControl('completed_upload', null), isEmpty); - expect(lastRequestedCheckpointRequestId(), 0); + expect(lastRequestedCheckpointRequestId(), 1); // Sync afterwards containing data and write checkpoint. pushCheckpoint(buckets: priorityBuckets, writeCheckpoint: '1'); diff --git a/docs/sync.md b/docs/sync.md index efbf633f..bab34a51 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -41,20 +41,16 @@ The following commands are supported: 11. `local_target_op`: Payload is `null`, an integer, or an integer string. Probes, updates or clears the local target op and returns the previously-observed value in a `LocalTargetOp` result. This command can run outside of a sync iteration and does not affect it. -12. `seed_checkpoint_request_id`: Payload is `null`, an integer, or an integer string. After - receiving `EstablishSyncStream`, SDKs should reconcile the provided local hint with the service +12. `seed_checkpoint_request_id`: Payload is a positive integer or integer string. After receiving + `EstablishSyncStream`, SDKs should reconcile the provided local hint with the service checkpoint-request state on every connection attempt. This can bump core when the service is ahead, or restore the service-side value when the service has cleared stale state but core still has a local hint. Then seed core with the reconciled value. Core stores the seeded value verbatim and does not enforce monotonicity; SDKs own the reconciliation and must not seed a - stale value. A `NULL` payload is accepted for completeness (core stores `0`, marking the state - as seeded so the first allocation returns `1`), but SDKs should not need it in practice: - posting a checkpoint request with an id of at least `1` during reconciliation and seeding the - service's response covers the no-record case and doubles as a probe of the service's - checkpoint-request support. Blindly forwarding a raw `NULL` service response while core holds a - counter would reset it, since the store is verbatim (see - `docs/write-checkpoint-requests.md`). If both the client and service have lost the value, the - counter may restart. + stale value. Posting a checkpoint request with an id of at least `1` during reconciliation and + seeding the service's response covers the no-record case and doubles as a probe of the service's + checkpoint-request support. If both the client and service have lost the value, the counter may + restart. When uploads request a write checkpoint, SDKs should call `powersync_control('next_checkpoint_request_id', NULL)` inside a transaction to allocate the id to diff --git a/docs/write-checkpoint-requests.md b/docs/write-checkpoint-requests.md index b4911373..37a2a161 100644 --- a/docs/write-checkpoint-requests.md +++ b/docs/write-checkpoint-requests.md @@ -156,14 +156,13 @@ locally. If both sides have lost the value, it is acceptable for the counter to happen after local state is cleared and stale service state expires, or when multiple user ids share the same client id. SDKs may also refresh service state when their user/client context changes. -Because seeds are stored verbatim, seeding `NULL` (or a low id) while core holds a higher counter -resets that counter, and previously allocated ids would be handed out again. SDKs must therefore -never forward a raw service response without reconciling: the recommended pattern is to always post -an id of at least `1` (the maximum of the local hint and any concrete local target) during -reconciliation and seed core with the service's response to that request. In practice SDKs never -need to seed `NULL` at all — when there is no local record, posting a checkpoint request with id -`1` works and doubles as a probe of the service's checkpoint-request support. Core still accepts a -`NULL` seed for completeness. +Because seeds are stored verbatim, seeding a low id while core holds a higher counter resets that +counter, and previously allocated ids would be handed out again. SDKs must therefore never forward +a raw service response without reconciling: the recommended pattern is to always post an id of at +least `1` (the maximum of the local hint and any concrete local target) during reconciliation and +seed core with the service's response to that request. When there is no local record, posting a +checkpoint request with id `1` works and doubles as a probe of the service's checkpoint-request +support. Core rejects `NULL` seeds. `powersync_control('next_checkpoint_request_id', NULL)` must be called inside a transaction during an active sync iteration after `last_requested_checkpoint_request_id` exists locally. It increments From 510bf4d6d227d30b584975c8ca6327731da77acf Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 8 Jul 2026 09:53:39 +0200 Subject: [PATCH 20/29] directly return responses for next_checkpoint_request_id and local_target_op --- crates/core/src/sync/interface.rs | 54 ++++++++++++++++++-------- crates/core/src/sync/streaming_sync.rs | 17 -------- dart/test/sync_test.dart | 28 ++++++++++--- docs/sync.md | 14 +++---- docs/write-checkpoint-requests.md | 8 ++-- 5 files changed, 70 insertions(+), 51 deletions(-) diff --git a/crates/core/src/sync/interface.rs b/crates/core/src/sync/interface.rs index 0be322e8..c0ad6fad 100644 --- a/crates/core/src/sync/interface.rs +++ b/crates/core/src/sync/interface.rs @@ -77,12 +77,6 @@ pub enum SyncControlRequest<'a> { StartSyncStream(StartSyncStream), /// The client requests to stop the current sync iteration. StopSyncStream, - /// The client requests a new checkpoint request id. - NextCheckpointRequestId, - /// The client probes and optionally updates the local target op. - /// - /// This can run outside of a sync iteration and does not affect it. - ProbeLocalTargetOp { target_op: Option }, /// The client is forwading a sync event to the core extension. SyncEvent(SyncEvent<'a>), } @@ -156,12 +150,6 @@ pub enum Instruction { /// If false, this is a pre-fetch. did_expire: bool, }, - /// Return a newly allocated checkpoint request id to the SDK. - CheckpointRequestId { request_id: i64 }, - /// Return the local target op value observed before an optional update. - LocalTargetOp { target_op: Option }, - // These are defined like this because deserializers in Kotlin can't support either an - // object or a literal value /// Close the websocket / HTTP stream to the sync service. CloseSyncStream(CloseSyncStream), /// Flush the file-system if it's non-durable (only applicable to the Dart SDK). @@ -261,14 +249,46 @@ pub fn register(db: *mut sqlite::sqlite3, state: Rc) -> Result<() } }), "stop" => SyncControlRequest::StopSyncStream, - "next_checkpoint_request_id" => SyncControlRequest::NextCheckpointRequestId, - "local_target_op" => SyncControlRequest::ProbeLocalTargetOp { - target_op: parse_optional_i64_payload( + "next_checkpoint_request_id" => { + let has_sync_iteration = { + let client = state.sync_client.borrow(); + client + .as_ref() + .map(|client| client.has_sync_iteration()) + .unwrap_or(false) + }; + + if !has_sync_iteration { + return Err(PowerSyncError::state_error("No iteration is active")); + } + + let adapter = state.storage_adapter(db)?; + if !adapter.has_checkpoint_request_id()? { + return Err(PowerSyncError::state_error( + "Checkpoint request state has not been seeded", + )); + } + + let request_id = adapter.next_checkpoint_request_id()?; + ctx.result_int64(request_id); + return Ok(()); + } + "local_target_op" => { + let target_op = parse_optional_i64_payload( *payload, "local target op", "local target op must be an integer, integer string, or null", - )?, - }, + )?; + let adapter = state.storage_adapter(db)?; + let previous_target_op = adapter.probe_local_target_op(target_op)?; + + match previous_target_op { + Some(target_op) => ctx.result_int64(target_op), + None => ctx.result_null(), + } + + return Ok(()); + } "line_text" => SyncControlRequest::SyncEvent(SyncEvent::TextLine { data: if payload.value_type() == ColumnType::Text { payload.text() diff --git a/crates/core/src/sync/streaming_sync.rs b/crates/core/src/sync/streaming_sync.rs index d88ef413..31f3b48b 100644 --- a/crates/core/src/sync/streaming_sync.rs +++ b/crates/core/src/sync/streaming_sync.rs @@ -117,23 +117,6 @@ impl SyncClient { Ok(active.instructions) } } - SyncControlRequest::NextCheckpointRequestId => { - if !self.has_sync_iteration() { - return Err(PowerSyncError::state_error("No iteration is active")); - } - if !self.adapter.has_checkpoint_request_id()? { - return Err(PowerSyncError::state_error( - "Checkpoint request state has not been seeded", - )); - } - - let request_id = self.adapter.next_checkpoint_request_id()?; - Ok(alloc::vec![Instruction::CheckpointRequestId { request_id }]) - } - SyncControlRequest::ProbeLocalTargetOp { target_op } => { - let target_op = self.adapter.probe_local_target_op(target_op)?; - Ok(alloc::vec![Instruction::LocalTargetOp { target_op }]) - } SyncControlRequest::StopSyncStream => self.state.tear_down(), } } diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index 5f0ece47..8ba085db 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -63,6 +63,26 @@ void _syncTests({ return jsonDecode(row.columnAt(0)); } + Object? invokeControlScalar(String operation, Object? data) { + db.execute('begin'); + ResultSet result; + + try { + result = db.select('SELECT powersync_control(?, ?)', [operation, data]); + + const statement = 'SELECT * FROM sqlite_stmt WHERE busy AND sql != ?;'; + final busy = db.select(statement, [statement]); + expect(busy, isEmpty); + } catch (e) { + db.execute('rollback'); + rethrow; + } + + db.execute('commit'); + final [row] = result; + return row.columnAt(0); + } + bool establishesSyncStream(List instructions) { return instructions.any((instruction) => instruction is Map && instruction.containsKey('EstablishSyncStream')); @@ -170,15 +190,11 @@ void _syncTests({ } int nextCheckpointRequestId() { - final [instruction] = invokeControl('next_checkpoint_request_id', null); - final data = (instruction as Map)['CheckpointRequestId'] as Map; - return data['request_id'] as int; + return invokeControlScalar('next_checkpoint_request_id', null) as int; } Object? probeLocalTargetOp([Object? opId]) { - final [instruction] = invokeControl('local_target_op', opId); - final data = (instruction as Map)['LocalTargetOp'] as Map; - return data['target_op']; + return invokeControlScalar('local_target_op', opId); } ResultSet fetchRows() { diff --git a/docs/sync.md b/docs/sync.md index bab34a51..135af531 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -36,11 +36,12 @@ The following commands are supported: 9. `subscriptions`: Store a new sync stream subscription in the database or remove it. This command can run outside of a sync iteration and does not affect it. 10. `next_checkpoint_request_id`: No payload. During an active sync iteration after checkpoint - request state exists locally, allocates and returns the next checkpoint request id in a - `CheckpointRequestId` instruction. + request state exists locally, allocates and returns the next checkpoint request id as an + integer result. 11. `local_target_op`: Payload is `null`, an integer, or an integer string. Probes, updates or - clears the local target op and returns the previously-observed value in a `LocalTargetOp` - result. This command can run outside of a sync iteration and does not affect it. + clears the local target op and returns the previously-observed value as an integer result, or + SQL `NULL` if there was no target. This command can run outside of a sync iteration and does not + affect it. 12. `seed_checkpoint_request_id`: Payload is a positive integer or integer string. After receiving `EstablishSyncStream`, SDKs should reconcile the provided local hint with the service checkpoint-request state on every connection attempt. This can bump core when the service is @@ -103,15 +104,14 @@ In that state, create one old-style write checkpoint first, store the returned c `powersync_control('local_target_op', id)`, let that gate resolve, and then switch to client-created checkpoint requests after the request counter has been reconciled on connect. -`powersync_control` returns a JSON-encoded array of instructions for the client: +Most `powersync_control` commands return a JSON-encoded array of instructions for the client. +`next_checkpoint_request_id` and `local_target_op` return scalar values directly. ```typescript type Instruction = { LogLine: LogLine } | { UpdateSyncStatus: UpdateSyncStatus } | { EstablishSyncStream: EstablishSyncStream } | { FetchCredentials: FetchCredentials } - | { CheckpointRequestId: { request_id: number } } - | { LocalTargetOp: { target_op: null | number } } // Close a connection previously started after EstablishSyncStream | { CloseSyncStream: { hide_disconnect: boolean } } // For the Dart web client, flush the (otherwise non-durable) file system. diff --git a/docs/write-checkpoint-requests.md b/docs/write-checkpoint-requests.md index 37a2a161..a0aace17 100644 --- a/docs/write-checkpoint-requests.md +++ b/docs/write-checkpoint-requests.md @@ -92,7 +92,7 @@ The SDK implementation: ```text let previousTarget = transaction { - powersync_control('local_target_op', NULL).LocalTargetOp.target_op + powersync_control('local_target_op', NULL) } if previousTarget == MAX_OP_ID { @@ -116,7 +116,7 @@ the request. Only then does the upload path store that id as `local_target_op` w ```text let requestId = transaction { - powersync_control('next_checkpoint_request_id', NULL).CheckpointRequestId.request_id + powersync_control('next_checkpoint_request_id', NULL) } POST /sync/checkpoint-request { @@ -166,7 +166,7 @@ support. Core rejects `NULL` seeds. `powersync_control('next_checkpoint_request_id', NULL)` must be called inside a transaction during an active sync iteration after `last_requested_checkpoint_request_id` exists locally. It increments -and returns `last_requested_checkpoint_request_id` in a `CheckpointRequestId` instruction. +and returns `last_requested_checkpoint_request_id` as an integer result. ```sql INSERT INTO ps_kv(key, value) @@ -204,7 +204,7 @@ active sync iteration: This command only updates the apply gate. It does not allocate, seed, or overwrite `last_requested_checkpoint_request_id`. -The command returns the previous target value in a `LocalTargetOp` result, or `NULL` if there was no +The command returns the previous target value as an integer result, or SQL `NULL` if there was no target. ```text From de130f4814397331868fe4af94060c1a95342d9e Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 8 Jul 2026 10:18:54 +0200 Subject: [PATCH 21/29] update docs for consice explainations. Add reconcilliation flow details. --- docs/sync.md | 81 ++---- docs/write-checkpoint-requests.md | 431 +++++++----------------------- 2 files changed, 119 insertions(+), 393 deletions(-) diff --git a/docs/sync.md b/docs/sync.md index 135af531..7ac1721a 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -43,66 +43,27 @@ The following commands are supported: SQL `NULL` if there was no target. This command can run outside of a sync iteration and does not affect it. 12. `seed_checkpoint_request_id`: Payload is a positive integer or integer string. After receiving - `EstablishSyncStream`, SDKs should reconcile the provided local hint with the service - checkpoint-request state on every connection attempt. This can bump core when the service is - ahead, or restore the service-side value when the service has cleared stale state but core still - has a local hint. Then seed core with the reconciled value. Core stores the seeded value - verbatim and does not enforce monotonicity; SDKs own the reconciliation and must not seed a - stale value. Posting a checkpoint request with an id of at least `1` during reconciliation and - seeding the service's response covers the no-record case and doubles as a probe of the service's - checkpoint-request support. If both the client and service have lost the value, the counter may - restart. - -When uploads request a write checkpoint, SDKs should call -`powersync_control('next_checkpoint_request_id', NULL)` inside a transaction to allocate the id to -pass to the request-checkpoint API. In checkpoint-request mode, the SDK should first allocate the id, -then post that id to the service, and then call `powersync_control('local_target_op', id)` with the -same id once the service accepts the request. This sets the local target op to the request op, -replacing the pending-write sentinel with the concrete checkpoint request id that the sync stream -can satisfy. `next_checkpoint_request_id` only advances the request counter; it does not update the -local target op used to block applying downloaded rows. - -`powersync_control('local_target_op', op_id)` probes and optionally updates the internal local -target op. The same command is used for compatibility when a new SDK is used with an older -PowerSync service that does not yet support client-created checkpoint requests; after the -service-side write checkpoint request returns a concrete id, call -`powersync_control('local_target_op', id)` with that id. Passing `NULL` returns the current target -without changing it, and passing `0` clears the local target. This command only updates the apply -gate; it does not allocate, seed, or overwrite `last_requested_checkpoint_request_id`. - -Database migration v14 moves legacy `$local` checkpoint state into `ps_kv`: `$local.last_applied_op` -becomes `last_applied_checkpoint_request_id`, `$local.last_op` becomes the internal -`last_seen_checkpoint_request_id`, and any positive `$local.target_op` is stored as -`local_target_op`. A concrete `$local.target_op` could be used to seed -`last_requested_checkpoint_request_id`, but it should be redundant because SDKs reconcile the -request counter with the service on connect. The migration then deletes the `$local` row, leaving -only real sync buckets in `ps_buckets`, and drops `ps_buckets.target_op` so -older SDKs fail hard if they try to keep using the migrated database directly. Downgrading restores -the column, and restores a `$local` row only when `local_target_op` exists, so older SDKs can keep -using target-op based blocking without inventing a synthetic local bucket when there was no local -target state. A restored concrete target remains satisfiable after a downgrade because checkpoint -request ids and legacy write checkpoint ids share one namespace: the service reports accepted -checkpoint request ids as the `write_checkpoint` values older-protocol clients observe. Because the down migration keeps the `ps_kv` keys around, the up migration clears -them before copying, so re-upgrading a downgraded database takes the `$local` row (including any -progress an older SDK made) as the source of truth instead of failing on the existing keys. - -`last_requested_checkpoint_request_id` is internal allocation state used by -`next_checkpoint_request_id` to allocate increasing ids for client-created checkpoint requests. -`last_seen_checkpoint_request_id` and `last_applied_checkpoint_request_id` are high-water marks -that local writes clear, so only checkpoint request ids observed after a write count towards the -apply gate. SDKs should use `DidCompleteSync.applied_checkpoint_request_id` for explicit checkpoint -request waits instead of presenting these values as meaningful sync progress. - -If `local_target_op` is absent after migration, there is no local write gate waiting for a -checkpoint. In that case, SDKs can start client-created checkpoint requests normally, even when -`last_requested_checkpoint_request_id` is undefined and the first allocated id is `1`. - -The ambiguous migration case is a migrated `local_target_op` of max op id: local writes are -pending, but there is no concrete request id to wait for yet. The max-op sentinel may also cover -earlier pending uploads that were already associated with legacy service-created write checkpoints. -In that state, create one old-style write checkpoint first, store the returned concrete id with -`powersync_control('local_target_op', id)`, let that gate resolve, and then switch to -client-created checkpoint requests after the request counter has been reconciled on connect. + `EstablishSyncStream`, SDKs should reconcile the local hint with service-side + checkpoint-request state, then seed core with the accepted positive id. + +## Checkpoint Request Expectations + +Checkpoint request state exists to protect local writes and to support explicit "wait until synced" +requests. The detailed state model lives in `write-checkpoint-requests.md`; this section summarizes +what SDKs need to do. + +- On every connection, reconcile `EstablishSyncStream.last_checkpoint_request_id` with the service. + Post at least `1` when there is no known id, then call + `powersync_control('seed_checkpoint_request_id', acceptedId)`. + The service returns the maximum of client and service-side state, so this hydrates a client that + lost its local value and recreates service-side state when the service lost its record. +- Wait for seeding to complete before creating checkpoint requests. For an upload write checkpoint, + call `powersync_control('next_checkpoint_request_id', NULL)` in a transaction, post the returned + id to the service, then store the accepted id with `powersync_control('local_target_op', id)`. +- `local_target_op` is the apply gate for local writes. `next_checkpoint_request_id` only allocates + ids; it does not update that gate. +- Resolve explicit checkpoint waiters from `DidCompleteSync.applied_checkpoint_request_id`, not from + `ps_kv`. Most `powersync_control` commands return a JSON-encoded array of instructions for the client. `next_checkpoint_request_id` and `local_target_op` return scalar values directly. diff --git a/docs/write-checkpoint-requests.md b/docs/write-checkpoint-requests.md index a0aace17..06f1279d 100644 --- a/docs/write-checkpoint-requests.md +++ b/docs/write-checkpoint-requests.md @@ -1,122 +1,81 @@ # Write Checkpoint State in `ps_kv` -The new write checkpoint logic moves the historic `$local` bucket bookkeeping into `ps_kv`. -`ps_buckets` now tracks real sync buckets, while local upload gating and checkpoint-request -progress are represented as key/value state. - -At a high level: - -- `local_target_op` replaces `$local.target_op` as the local write apply gate. -- `last_seen_checkpoint_request_id` replaces `$local.last_op`. -- `last_applied_checkpoint_request_id` replaces `$local.last_applied_op`. -- `last_requested_checkpoint_request_id` tracks the latest concrete checkpoint request id allocated - by the client, so `next_checkpoint_request_id` can allocate increasing ids for each checkpoint - request. - -These keys are internal SDK/core state, not user-facing sync progress. -`last_requested_checkpoint_request_id` is functional allocation state, while -`last_seen_checkpoint_request_id` and `last_applied_checkpoint_request_id` are mostly diagnostic -high-water marks. Explicit checkpoint waits should follow -`DidCompleteSync.applied_checkpoint_request_id`. - -SDKs should not write these keys directly. They update the local target through -`powersync_control('local_target_op', value)`, which is the shared helper for both legacy write -checkpoints and new client-created checkpoint requests. The -`powersync_control('next_checkpoint_request_id', NULL)` command only allocates a checkpoint request -id; after the service accepts that request, the SDK uses -`powersync_control('local_target_op', id)` to make the accepted id the local target for write -checkpoints. - -For the historic `$local` bucket flow, see `historic-write-checkpoints.md`. - -## Local writes - -A client write to a synced table/view records an entry in `ps_crud`. For simple CRUD triggers, the -same transaction also records the affected row in `ps_updated_rows` and sets `local_target_op` to -the maximum i64 value. This is the `ps_kv` equivalent of the old `$local.target_op` sentinel: it -means "there are local writes, but we do not yet know the concrete checkpoint id that will -acknowledge them". - -The sentinel is stored in `ps_kv`, not in `ps_buckets`. The same statement clears the -`last_seen_checkpoint_request_id` and `last_applied_checkpoint_request_id` high-water marks: +This document describes the checkpoint state used to keep downloaded rows behind local writes until +the service has observed those writes, while also supporting explicit "wait until synced" +checkpoint requests. -```sql -INSERT OR REPLACE INTO ps_kv(key, value) -VALUES('local_target_op', MAX_OP_ID); +The state is internal core/SDK bookkeeping. Apps should not read it as user-facing sync progress. -DELETE FROM ps_kv -WHERE key IN ('last_seen_checkpoint_request_id', 'last_applied_checkpoint_request_id'); -``` +## State Model -Clearing the high-water marks mirrors how the legacy flow reset the whole `$local` row on local -writes. A checkpoint request id observed before the write cannot acknowledge it, and a stale seen -value may even predate a request counter restart. If such a value stayed around, a newly allocated -target id could compare below it and open the apply gate before the service acknowledged the write. -After a local write, only checkpoint request ids observed from that point on count towards the gate. +| Key | Purpose | Who updates it | +| --- | --- | --- | +| `local_target_op` | Apply gate for local writes. Downloaded full checkpoints can apply only after the stream has seen this id. `MAX_OP_ID` means "local writes exist, but no concrete checkpoint id is known yet". | Core CRUD triggers set the sentinel; SDK upload code stores the accepted concrete id through `powersync_control('local_target_op', id)`. | +| `last_requested_checkpoint_request_id` | Allocation counter for client-created checkpoint requests. | SDKs seed it after connection reconciliation, then core increments it through `powersync_control('next_checkpoint_request_id', NULL)`. | +| `last_seen_checkpoint_request_id` | Latest full checkpoint `write_checkpoint` observed in the stream since the last local write. | Core updates it when a full checkpoint validates. Local writes clear it. | +| `last_applied_checkpoint_request_id` | Latest full checkpoint `write_checkpoint` applied locally since the last local write. | Core updates it after a full checkpoint applies. Local writes clear it. | -## Completing uploaded CRUD +Why four keys? -SDK upload code removes uploaded items from `ps_crud`. If the connector supplies a legacy custom -write checkpoint and the queue is empty, that concrete checkpoint becomes the local target -immediately. -Otherwise the target is reset to `MAX_OP_ID`, allowing the sync client to create a standard -checkpoint request after the queue drains. +- `local_target_op` is the gate that protects local writes. +- `last_requested_checkpoint_request_id` is the counter used to create new requests. +- `last_seen_checkpoint_request_id` answers "has the stream reached the gate yet?" +- `last_applied_checkpoint_request_id` is persisted for diagnostics. SDK waiters should use + `DidCompleteSync.applied_checkpoint_request_id` instead. -```text -transaction { - deleteUploadedCrud(upTo: lastUploadedId) +## SDK Expectations - if let customCheckpoint, crudQueueIsEmpty { - powersync_control('local_target_op', customCheckpoint) - } else { - powersync_control('local_target_op', MAX_OP_ID) - } -} -``` +SDKs should use `powersync_control` as the public API for this state: -## Updating the local target +1. On each sync connection, read `EstablishSyncStream.last_checkpoint_request_id`. +2. Reconcile that local hint with service-side checkpoint-request state before allocating new ids. +3. Seed core with the reconciled positive id by calling + `powersync_control('seed_checkpoint_request_id', id)`. +4. Wait for seeding to complete before creating checkpoint requests. +5. When upload code needs a write checkpoint, allocate an id inside a transaction with + `powersync_control('next_checkpoint_request_id', NULL)`. +6. Post that id to the service checkpoint-request endpoint. +7. After the service accepts it, store it as the local write gate with + `powersync_control('local_target_op', id)`. +8. Resolve explicit waiters from `DidCompleteSync.applied_checkpoint_request_id`, not from `ps_kv`. -Once uploads are complete, the sync client updates the local target through -`powersync_control('local_target_op', value)`. It only does this when the current target is still -`MAX_OP_ID`, which avoids overwriting a custom checkpoint that was already stored by -`complete(writeCheckpoint:)`. +`seed_checkpoint_request_id` stores the id verbatim and does not enforce monotonicity. SDKs own +reconciliation and must not seed stale service state. The recommended reconciliation pattern is: -The SDK implementation: +```text +effectiveId = max(localHint ?? 0, concreteLocalTarget ?? 0, 1) +acceptedId = postCheckpointRequestStateToService(effectiveId) +powersync_control('seed_checkpoint_request_id', acceptedId) +``` -1. Probes the current target with `powersync_control('local_target_op', NULL)`. -2. Reads `sqlite_sequence.seq` for `ps_crud`. -3. Gets a concrete checkpoint id from either the new or legacy service API. -4. Re-enters a write transaction. -5. Verifies that `ps_crud` is still empty and that its sequence did not change. -6. Stores the concrete target with `powersync_control('local_target_op', opId)`. +Posting at least `1` covers the no-record case and probes whether the service supports checkpoint +requests. Core rejects `NULL` and `0` seeds. -```text -let previousTarget = transaction { - powersync_control('local_target_op', NULL) -} +The service returns the maximum of the client-provided id and its service-side record. If the client +lost local state, for example after `disconnectAndClear`, this response hydrates core's counter. If +the service lost its record, posting the local hint recreates the service-side state. -if previousTarget == MAX_OP_ID { - let seqBefore = psCrudSequence() - let checkpointId = await createOrFetchCheckpointId() +`powersync_control('next_checkpoint_request_id', NULL)` requires an active sync iteration and a +seeded request counter. SDKs should wait for the connection reconciliation and seed step before +creating checkpoint requests. - transaction { - guard ps_crud.isEmpty && psCrudSequence() == seqBefore else { - return - } +## Local Write Gate - powersync_control('local_target_op', checkpointId) - } -} +A local write records CRUD and sets: + +```sql +ps_kv['local_target_op'] = MAX_OP_ID ``` -In checkpoint-request mode, `getWriteCheckpoint()` calls `requestCheckpoint()`. That allocates an -id locally, sends it to `/sync/checkpoint-request`, and returns the same id once the service accepts -the request. Only then does the upload path store that id as `local_target_op` with -`powersync_control('local_target_op', id)`. +It also clears `last_seen_checkpoint_request_id` and `last_applied_checkpoint_request_id`, because +checkpoint ids observed before the write cannot acknowledge it. + +After uploaded CRUD is accepted by the backend, SDK code replaces `MAX_OP_ID` with a concrete +checkpoint id: ```text -let requestId = transaction { - powersync_control('next_checkpoint_request_id', NULL) +transaction { + requestId = powersync_control('next_checkpoint_request_id', NULL) } POST /sync/checkpoint-request { @@ -124,258 +83,64 @@ POST /sync/checkpoint-request { checkpoint_request_id: requestId } -return requestId -``` - -The legacy fallback still calls `/write-checkpoint2.json`; the returned write checkpoint is stored -through the same `powersync_control('local_target_op', opId)` helper. This keeps SDK target updates -consistent across both protocols. - -## Sync control commands - -These `powersync_control` commands are the SDK-facing API for the new `ps_kv` checkpoint state. - -`powersync_control('start', payload)` begins a sync iteration and emits `EstablishSyncStream` with a -`last_checkpoint_request_id` hint. This is core's local `last_requested_checkpoint_request_id` value -before opening the stream, or `NULL` when no local seed exists. On every connection attempt, SDKs -should reconcile this hint with the service checkpoint-request state before creating new requests. -The reconciliation is bidirectional: if the service still has a higher value, the SDK uses that -response to bump core locally so following requests are accepted; if the service has cleared stale -state but core still has a local hint, the SDK can use the hint to restore the service-side value. -After reconciliation, call `powersync_control('seed_checkpoint_request_id', value)` with the -reconciled value. - -`last_requested_checkpoint_request_id` is a best-effort counter seed, not durable application state. -Core stores whatever value is seeded, without enforcing monotonicity: the SDK owns the -reconciliation and is expected to seed the effective state accepted by the service. If either the -client or the service still remembers a higher id, the SDK's reconciliation keeps the local counter -moving forward. If the service has cleared stale state but the client still has a local seed, the -local hint can restore the service-side value and keep the counter from moving backwards. If the -client lost local state but the service still has a record, the service response restores the seed -locally. If both sides have lost the value, it is acceptable for the counter to restart; this can -happen after local state is cleared and stale service state expires, or when multiple user ids share -the same client id. SDKs may also refresh service state when their user/client context changes. - -Because seeds are stored verbatim, seeding a low id while core holds a higher counter resets that -counter, and previously allocated ids would be handed out again. SDKs must therefore never forward -a raw service response without reconciling: the recommended pattern is to always post an id of at -least `1` (the maximum of the local hint and any concrete local target) during reconciliation and -seed core with the service's response to that request. When there is no local record, posting a -checkpoint request with id `1` works and doubles as a probe of the service's checkpoint-request -support. Core rejects `NULL` seeds. - -`powersync_control('next_checkpoint_request_id', NULL)` must be called inside a transaction during -an active sync iteration after `last_requested_checkpoint_request_id` exists locally. It increments -and returns `last_requested_checkpoint_request_id` as an integer result. - -```sql -INSERT INTO ps_kv(key, value) -VALUES('last_requested_checkpoint_request_id', 1) -ON CONFLICT(key) DO UPDATE SET value = CAST(value AS INTEGER) + 1 -RETURNING value; +transaction { + previousTarget = powersync_control('local_target_op', NULL) + if previousTarget == MAX_OP_ID && ps_crud is still empty { + powersync_control('local_target_op', requestId) + } +} ``` -This command only allocates an id. It does not update `local_target_op`. - -Calling it without an active iteration or before seeding raises a state error. This is a normal -part of the connection lifecycle (for example a `requestCheckpoint` call racing a stream restart), -not a programming error — SDKs should surface it as a retryable condition. - -The increment participates in the caller's transaction. If the transaction rolls back after the id -was already posted to the service, the same id is allocated and posted again on retry; this is safe -because the service treats the latest posted id as the effective request state. - -Note on sequences: SQLite does not have standalone sequences. The sequence-like alternatives are -either an `AUTOINCREMENT` table backed by SQLite's internal `sqlite_sequence`, or a dedicated -single-row counter table like the existing `ps_tx` transaction counter. The checkpoint request -counter currently lives in `ps_kv` so it can persist across requests and be reconciled with service -state on connect. If we want stricter structure later, a dedicated checkpoint-request counter table -would be the closest match to a sequence. - -`powersync_control('local_target_op', op_id)` probes and optionally updates the local target. Like -`subscriptions`, this command is handled directly by `powersync_control` and can run outside an -active sync iteration: - -- `NULL` returns the current `local_target_op` without changing it. -- `0` clears `local_target_op`. -- A positive value stores `local_target_op`. -- Negative values and non-integer inputs are rejected. +`local_target_op` is intentionally separate from `last_requested_checkpoint_request_id`: allocating +a checkpoint request id does not mean it should block or unblock local writes. -This command only updates the apply gate. It does not allocate, seed, or overwrite -`last_requested_checkpoint_request_id`. +## Applying Downloaded Checkpoints -The command returns the previous target value as an integer result, or SQL `NULL` if there was no -target. +The service reports accepted checkpoint request ids as `checkpoint.write_checkpoint`. For full +checkpoints, core stores that value as `last_seen_checkpoint_request_id`. +The gate uses "seen" rather than "applied" because this check runs before the current checkpoint +can be applied. -```text -previous = ps_kv['local_target_op'] +Full checkpoints and non-priority-0 partial checkpoints can publish only when: -if target_op == NULL: - return previous -if target_op == 0: - delete ps_kv['local_target_op'] -else: - ps_kv['local_target_op'] = target_op +- `ps_crud` is empty, and +- `local_target_op` is absent or less than or equal to `last_seen_checkpoint_request_id`. -return previous -``` +Priority 0 partial syncs may publish while uploads are outstanding. -## Applying downloaded checkpoints +If a full checkpoint validates but cannot apply because local CRUD is pending, core keeps it as a +pending checkpoint. When the SDK later sends `completed_upload`, core retries it unless the +pending checkpoint is older than the current `local_target_op`. -The sync stream reports the checkpoint request id in `checkpoint.write_checkpoint`. After a full -checkpoint validates, core persists it as `last_seen_checkpoint_request_id`. +After a full checkpoint applies, core emits: ```text -on full checkpoint with write_checkpoint: - ps_kv['last_seen_checkpoint_request_id'] = checkpoint.write_checkpoint +DidCompleteSync { applied_checkpoint_request_id?: checkpoint.write_checkpoint } ``` -Before publishing downloaded rows, `sync_local` checks the local gate. Full checkpoints and -non-priority-0 partial checkpoints can only apply when: +SDKs should resolve `requestCheckpoint()` / `waitForSync()` waiters when this value is greater than +or equal to the requested id. -- `local_target_op` is absent, or it is less than or equal to `last_seen_checkpoint_request_id`. -- `ps_crud` is empty. +## Control Commands -Priority 0 partial syncs are the exception: they may publish while uploads are outstanding. +`powersync_control('seed_checkpoint_request_id', id)` -```sql -SELECT 1 -FROM ps_kv AS target -LEFT JOIN ps_kv AS seen ON seen.key = 'last_seen_checkpoint_request_id' -WHERE target.key = 'local_target_op' - AND CAST(target.value AS INTEGER) > COALESCE(CAST(seen.value AS INTEGER), 0); -``` +- Payload: positive integer or integer string. +- Stores the reconciled checkpoint-request counter seed. +- Must be called after connection reconciliation and before allocating new ids. -If a full checkpoint validated but cannot apply because local CRUD is pending, the state machine -keeps it as `validated_but_not_applied`. When the SDK later sends `completed_upload`, core retries -that checkpoint unless its `write_checkpoint` is older than the current `local_target_op`. +`powersync_control('next_checkpoint_request_id', NULL)` -```text -on completed_upload: - if pending_checkpoint.write_checkpoint >= local_target_op: - retry applying pending_checkpoint -``` +- Returns: next checkpoint request id as a SQLite integer. +- Requires an active sync iteration and a seeded request counter. +- SDKs should call this only after the connection reconciliation and seed step has completed. +- Participates in the caller's transaction. If the transaction rolls back after the id was posted + to the service, retrying posts the same id again, which is safe because the service treats the + latest posted id as effective state. -After a full checkpoint applies, core stores the applied checkpoint request id as -`last_applied_checkpoint_request_id` and emits it on the `DidCompleteSync` instruction. - -```text -after full checkpoint apply: - ps_kv['last_applied_checkpoint_request_id'] = checkpoint.write_checkpoint - emit DidCompleteSync { applied_checkpoint_request_id: checkpoint.write_checkpoint } -``` - -## Explicit checkpoint requests - -SDKs can expose a `requestCheckpoint()`-style API for callers that want to wait until the local -database has caught up to the service. The SDK creates a checkpoint request id through the connected -sync client and returns a `CheckpointRequest`-style waiter. - -This explicit API does not update `local_target_op`: it is a wait marker, not a local upload gate. -The returned object waits until core emits `DidCompleteSync` with -`applied_checkpoint_request_id` greater than or equal to the requested id. - -```text -waitForSync() { - for instruction in syncInstructions { - return when instruction.DidCompleteSync.applied_checkpoint_request_id >= requestId - throw if sync status reports a sync error - } -} -``` +`powersync_control('local_target_op', value)` -The public database method requires an active or connecting sync client, because a disconnected -request could not be delivered to the service or observed in the sync stream. - -Waiters do not need durable applied state across reconnects or app restarts. The connect-time -counter reconciliation doubles as a re-request: the SDK posts the effective checkpoint request id -to the service on every connection attempt, so the next checkpoint carries a `write_checkpoint` -greater than or equal to any previously requested id and core emits a fresh -`DidCompleteSync.applied_checkpoint_request_id` that resolves outstanding waits. - -## `ps_kv` checkpoint state - -- `local_target_op`: The current apply gate. It is either `MAX_OP_ID` while local writes are - pending, a concrete checkpoint request id after upload completion, or absent when there is no - local write gate. -- `last_requested_checkpoint_request_id`: The last client-created checkpoint request id allocated - by `powersync_control('next_checkpoint_request_id', NULL)`. This is the counter used to allocate - increasing ids for each client-created checkpoint request, including multiple requests in one - client lifetime. The persisted value is also useful for debugging and for seeding the next - connection attempt. SDKs should reconcile it with the service on every connect, and should - tolerate it restarting when both the client and service have lost the previous value. -- `last_seen_checkpoint_request_id`: The latest full checkpoint `write_checkpoint` observed and - validated from the sync stream since the last local write. Local writes clear this key, so only - checkpoint request ids observed after the write can satisfy the apply gate. -- `last_applied_checkpoint_request_id`: The latest full checkpoint `write_checkpoint` that has been - applied locally since the last local write, which clears this key. Core persists this for - migration/downgrade state and debugging; SDKs should use - `DidCompleteSync.applied_checkpoint_request_id` to resolve `CheckpointRequest` waits. - -`powersync_clear` deletes all of these keys in both clear modes (it removes every `ps_kv` entry -except `client_id`). This is deliberate and mirrors the legacy behavior of deleting the `$local` -row: pending CRUD is wiped in the same operation, so no apply gate is needed, and the request -counter is restored by the connect-time reconciliation described above. If the service has also -lost its record, the counter restarting is acceptable. - -## Migration from `$local` - -Migration v14 moves the old `$local` bucket state into `ps_kv`: - -- `$local.last_applied_op` becomes `last_applied_checkpoint_request_id`. -- `$local.last_op` becomes `last_seen_checkpoint_request_id`. -- Any positive `$local.target_op`, including `MAX_OP_ID`, becomes `local_target_op`. - -A concrete `$local.target_op` could be used to seed `last_requested_checkpoint_request_id`, but it -should be redundant because SDKs reconcile the request counter with service state on connect before -advancing it through `next_checkpoint_request_id`. - -After copying this state, the migration deletes the `$local` row — version 14 tracks this state -exclusively in `ps_kv`, so `ps_buckets` only contains real sync buckets — and drops -`ps_buckets.target_op`. Dropping the column intentionally makes older SDKs fail with a hard SQLite -error if they try to keep using a migrated database without first downgrading. - -The up migration first deletes any existing `last_applied_checkpoint_request_id`, -`last_seen_checkpoint_request_id` and `local_target_op` keys. Those can be present when a database -was previously on version 14 and then downgraded, because the down migration keeps the ps_kv keys -while rebuilding `$local`. Clearing them makes the `$local` row the source of truth on re-upgrade, -picking up any progress an older SDK made while downgraded. `last_requested_checkpoint_request_id` -is unrelated to `$local` and survives a downgrade/upgrade cycle unchanged. - -An absent `local_target_op` is safe: there is no local write gate waiting for a checkpoint, so an -SDK can seed the request counter on connect and start client-created checkpoint requests normally. -If neither the client nor service has a previous request id, the first allocated id is `1`. The sync -stream will only report that request id after the service has accepted and reached it. - -The ambiguous case is a migrated `local_target_op` of `MAX_OP_ID`. That means there is a pending -local write gate but no concrete request id to wait for yet. The `MAX_OP_ID` sentinel only says that -local writes dirtied the gate; it does not prove that no earlier uploads were already associated -with legacy service-created write checkpoints. In that state, the SDK should create one legacy write -checkpoint first, store the concrete id with `powersync_control('local_target_op', id)`, let that -gate resolve, and then switch to client-created checkpoint requests after the request counter has -been reconciled on connect. - -The down migration restores `ps_buckets.target_op` and rebuilds a `$local` row only when -`local_target_op` exists, using: - -- `last_seen_checkpoint_request_id` as `$local.last_op` -- `last_applied_checkpoint_request_id` as `$local.last_applied_op` -- `local_target_op` as `$local.target_op` - -This keeps older SDKs able to use the historic target-op gate after a downgrade without inventing a -synthetic `$local` bucket when there was no local target state. - -Two properties make the restored gate safe rather than a potential stall: - -- **Shared id namespace.** Client-created checkpoint request ids and legacy write checkpoint ids - are one namespace, compatible in both directions — including values migrated from the historic - service-generated write checkpoint scheme. The service reports the checkpoint request ids it - accepted as the `write_checkpoint` values older-protocol clients observe, so a restored concrete - `$local.target_op` is satisfiable by the next write checkpoint the downgraded SDK sees; it does - not wait on an incomparable id sequence. -- **Downgrade fidelity.** `last_seen_checkpoint_request_id` and - `last_applied_checkpoint_request_id` only advance on full checkpoint completions — but so did the - legacy `$local.last_op`/`last_applied_op` bookkeeping (partial priority applies never updated - `$local`, which is not a real bucket). The rebuilt `$local` row therefore matches exactly what a - legacy client would have recorded at the same point in the stream; the down migration cannot lag - behind legacy behavior. +- Payload `NULL`: return current target without changing it. +- Payload `0`: clear the target. +- Positive payload: store a concrete target. +- Returns the previous target as a SQLite integer, or SQL `NULL` if absent. From d40c8da371ab6d25be91420958de588c92efd253 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 8 Jul 2026 10:31:45 +0200 Subject: [PATCH 22/29] add internal_applied_checkpoint_request_id to sync status --- crates/core/src/sync/storage_adapter.rs | 2 ++ crates/core/src/sync/streaming_sync.rs | 2 +- crates/core/src/sync/sync_status.rs | 21 ++++++++++++-- dart/test/sync_test.dart | 37 +++++++++++++++++++++++++ docs/sync.md | 7 +++-- docs/write-checkpoint-requests.md | 5 ++++ 6 files changed, 69 insertions(+), 5 deletions(-) diff --git a/crates/core/src/sync/storage_adapter.rs b/crates/core/src/sync/storage_adapter.rs index efbaead0..bf93308a 100644 --- a/crates/core/src/sync/storage_adapter.rs +++ b/crates/core/src/sync/storage_adapter.rs @@ -133,6 +133,8 @@ impl StorageAdapter { priority_status: priority_items, downloading: None, streams, + // Checkpoint requests should not be made or compared while offline. + internal_applied_checkpoint_request_id: None, }) } diff --git a/crates/core/src/sync/streaming_sync.rs b/crates/core/src/sync/streaming_sync.rs index 31f3b48b..2c5a6eed 100644 --- a/crates/core/src/sync/streaming_sync.rs +++ b/crates/core/src/sync/streaming_sync.rs @@ -972,7 +972,7 @@ impl StreamingSyncIteration { }); self.status.update( - |status| status.applied_checkpoint(timestamp), + |status| status.applied_checkpoint(timestamp, applied_checkpoint_request_id), &mut event.instructions, ); } diff --git a/crates/core/src/sync/sync_status.rs b/crates/core/src/sync/sync_status.rs index 6530c460..aabaad09 100644 --- a/crates/core/src/sync/sync_status.rs +++ b/crates/core/src/sync/sync_status.rs @@ -53,6 +53,11 @@ pub struct DownloadSyncStatus { /// received), information about how far the download has progressed. pub downloading: Option, pub streams: Vec, + /// Runtime-only request id from the most recent full checkpoint apply. + /// + /// This is exposed in sync status for SDK internals, but it is not persisted and should not be + /// treated as user-facing download progress. + pub internal_applied_checkpoint_request_id: Option, } impl DownloadSyncStatus { @@ -73,6 +78,7 @@ impl DownloadSyncStatus { self.connected = false; self.downloading = None; self.connecting = true; + self.internal_applied_checkpoint_request_id = None; self.debug_assert_priority_status_is_sorted(); } @@ -117,7 +123,11 @@ impl DownloadSyncStatus { self.debug_assert_priority_status_is_sorted(); } - pub fn applied_checkpoint(&mut self, now: TimestampMicros) { + pub fn applied_checkpoint( + &mut self, + now: TimestampMicros, + applied_checkpoint_request_id: Option, + ) { self.downloading = None; self.priority_status.clear(); @@ -126,6 +136,8 @@ impl DownloadSyncStatus { last_synced_at: Some(now), has_synced: Some(true), }); + + self.internal_applied_checkpoint_request_id = applied_checkpoint_request_id; } } @@ -137,6 +149,7 @@ impl Default for DownloadSyncStatus { downloading: None, priority_status: Vec::new(), streams: Vec::new(), + internal_applied_checkpoint_request_id: None, } } } @@ -180,12 +193,16 @@ impl Serialize for DownloadSyncStatus { } } - let mut serializer = serializer.serialize_struct("DownloadSyncStatus", 5)?; + let field_count = 5 + usize::from(self.internal_applied_checkpoint_request_id.is_some()); + let mut serializer = serializer.serialize_struct("DownloadSyncStatus", field_count)?; serializer.serialize_field("connected", &self.connected)?; serializer.serialize_field("connecting", &self.connecting)?; serializer.serialize_field("priority_status", &self.priority_status)?; serializer.serialize_field("downloading", &self.downloading)?; serializer.serialize_field("streams", &SerializeStreamsWithProgress(self))?; + if let Some(request_id) = self.internal_applied_checkpoint_request_id { + serializer.serialize_field("internal_applied_checkpoint_request_id", &request_id)?; + } serializer.end() } diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index 8ba085db..e8b97134 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -400,6 +400,15 @@ void _syncTests({ invokeControl('stop', null); final instructions = invokeControl('start', null); + expect( + instructions, + isNot(contains(containsPair( + 'UpdateSyncStatus', + containsPair( + 'status', + containsPair( + 'internal_applied_checkpoint_request_id', anything))))), + ); expect( instructions, contains( @@ -609,6 +618,15 @@ void _syncTests({ isNot(contains(containsPair('DidCompleteSync', containsPair('applied_checkpoint_request_id', anything)))), ); + expect( + instructions, + isNot(contains(containsPair( + 'UpdateSyncStatus', + containsPair( + 'status', + containsPair( + 'internal_applied_checkpoint_request_id', anything))))), + ); expect(lastAppliedCheckpointRequestId(), isNull); final [row] = db.select('select powersync_offline_sync_status();'); @@ -634,6 +652,16 @@ void _syncTests({ ), ), ); + expect( + appliedInstructions, + contains(containsPair( + 'UpdateSyncStatus', + containsPair( + 'status', + containsPair('internal_applied_checkpoint_request_id', 1), + ), + )), + ); expect(lastAppliedCheckpointRequestId(), 1); pushCheckpoint(buckets: priorityBuckets); @@ -643,6 +671,15 @@ void _syncTests({ instructions, contains(containsPair('DidCompleteSync', {})), ); + expect( + instructions, + isNot(contains(containsPair( + 'UpdateSyncStatus', + containsPair( + 'status', + containsPair( + 'internal_applied_checkpoint_request_id', anything))))), + ); final [row] = db.select('select powersync_offline_sync_status();'); expect( diff --git a/docs/sync.md b/docs/sync.md index 7ac1721a..6b502d95 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -62,8 +62,10 @@ what SDKs need to do. id to the service, then store the accepted id with `powersync_control('local_target_op', id)`. - `local_target_op` is the apply gate for local writes. `next_checkpoint_request_id` only allocates ids; it does not update that gate. -- Resolve explicit checkpoint waiters from `DidCompleteSync.applied_checkpoint_request_id`, not from - `ps_kv`. +- Resolve explicit checkpoint waiters from `DidCompleteSync.applied_checkpoint_request_id`. SDKs + that drive waiters from status snapshots can also watch + `UpdateSyncStatus.status.internal_applied_checkpoint_request_id`. Treat that status field as + runtime-only SDK state, not persisted checkpoint state or app-visible progress. Most `powersync_control` commands return a JSON-encoded array of instructions for the client. `next_checkpoint_request_id` and `local_target_op` return scalar values directly. @@ -105,6 +107,7 @@ interface UpdateSyncStatus { priority_status: [], downloading: null | DownloadProgress, streams: [], + internal_applied_checkpoint_request_id?: number, } interface DidCompleteSync { diff --git a/docs/write-checkpoint-requests.md b/docs/write-checkpoint-requests.md index 06f1279d..a1f3639b 100644 --- a/docs/write-checkpoint-requests.md +++ b/docs/write-checkpoint-requests.md @@ -121,6 +121,11 @@ DidCompleteSync { applied_checkpoint_request_id?: checkpoint.write_checkpoint } SDKs should resolve `requestCheckpoint()` / `waitForSync()` waiters when this value is greater than or equal to the requested id. +Core also includes the same applied request id in +`UpdateSyncStatus.status.internal_applied_checkpoint_request_id` for the status update emitted with +the completed checkpoint. SDKs may use that for status-stream waiters, but should treat it as +internal, runtime-only state rather than app-visible progress or persisted checkpoint state. + ## Control Commands `powersync_control('seed_checkpoint_request_id', id)` From 6aca9923031dd92bf55d5c32c157aacc9dda4026 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Wed, 8 Jul 2026 15:25:56 +0200 Subject: [PATCH 23/29] expose internal_last_applied_checkpoint_request_id in the sync status. --- crates/core/src/sync/storage_adapter.rs | 2 +- crates/core/src/sync/streaming_sync.rs | 7 ++++ crates/core/src/sync/sync_status.rs | 19 +++++----- dart/test/sync_test.dart | 46 +++++++++++++++++++++---- docs/sync.md | 4 +-- docs/write-checkpoint-requests.md | 10 +++--- 6 files changed, 67 insertions(+), 21 deletions(-) diff --git a/crates/core/src/sync/storage_adapter.rs b/crates/core/src/sync/storage_adapter.rs index bf93308a..1335390e 100644 --- a/crates/core/src/sync/storage_adapter.rs +++ b/crates/core/src/sync/storage_adapter.rs @@ -134,7 +134,7 @@ impl StorageAdapter { downloading: None, streams, // Checkpoint requests should not be made or compared while offline. - internal_applied_checkpoint_request_id: None, + internal_last_applied_checkpoint_request_id: None, }) } diff --git a/crates/core/src/sync/streaming_sync.rs b/crates/core/src/sync/streaming_sync.rs index 2c5a6eed..a75a6cd8 100644 --- a/crates/core/src/sync/streaming_sync.rs +++ b/crates/core/src/sync/streaming_sync.rs @@ -967,6 +967,13 @@ impl StreamingSyncIteration { timestamp: TimestampMicros, applied_checkpoint_request_id: Option, ) { + if let Some(request_id) = applied_checkpoint_request_id { + event.instructions.push(Instruction::LogLine { + severity: LogSeverity::DEBUG, + line: format!("Applied checkpoint request id {request_id}").into(), + }); + } + event.instructions.push(Instruction::DidCompleteSync { applied_checkpoint_request_id, }); diff --git a/crates/core/src/sync/sync_status.rs b/crates/core/src/sync/sync_status.rs index aabaad09..04568e47 100644 --- a/crates/core/src/sync/sync_status.rs +++ b/crates/core/src/sync/sync_status.rs @@ -53,11 +53,11 @@ pub struct DownloadSyncStatus { /// received), information about how far the download has progressed. pub downloading: Option, pub streams: Vec, - /// Runtime-only request id from the most recent full checkpoint apply. + /// Runtime-only request id from the most recent applied checkpoint request. /// /// This is exposed in sync status for SDK internals, but it is not persisted and should not be /// treated as user-facing download progress. - pub internal_applied_checkpoint_request_id: Option, + pub internal_last_applied_checkpoint_request_id: Option, } impl DownloadSyncStatus { @@ -72,13 +72,14 @@ impl DownloadSyncStatus { self.connected = false; self.connecting = false; self.downloading = None; + self.internal_last_applied_checkpoint_request_id = None; } pub fn start_connecting(&mut self) { self.connected = false; self.downloading = None; self.connecting = true; - self.internal_applied_checkpoint_request_id = None; + self.internal_last_applied_checkpoint_request_id = None; self.debug_assert_priority_status_is_sorted(); } @@ -137,7 +138,7 @@ impl DownloadSyncStatus { has_synced: Some(true), }); - self.internal_applied_checkpoint_request_id = applied_checkpoint_request_id; + self.internal_last_applied_checkpoint_request_id = applied_checkpoint_request_id; } } @@ -149,7 +150,7 @@ impl Default for DownloadSyncStatus { downloading: None, priority_status: Vec::new(), streams: Vec::new(), - internal_applied_checkpoint_request_id: None, + internal_last_applied_checkpoint_request_id: None, } } } @@ -193,15 +194,17 @@ impl Serialize for DownloadSyncStatus { } } - let field_count = 5 + usize::from(self.internal_applied_checkpoint_request_id.is_some()); + let field_count = + 5 + usize::from(self.internal_last_applied_checkpoint_request_id.is_some()); let mut serializer = serializer.serialize_struct("DownloadSyncStatus", field_count)?; serializer.serialize_field("connected", &self.connected)?; serializer.serialize_field("connecting", &self.connecting)?; serializer.serialize_field("priority_status", &self.priority_status)?; serializer.serialize_field("downloading", &self.downloading)?; serializer.serialize_field("streams", &SerializeStreamsWithProgress(self))?; - if let Some(request_id) = self.internal_applied_checkpoint_request_id { - serializer.serialize_field("internal_applied_checkpoint_request_id", &request_id)?; + if let Some(request_id) = self.internal_last_applied_checkpoint_request_id { + serializer + .serialize_field("internal_last_applied_checkpoint_request_id", &request_id)?; } serializer.end() diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index e8b97134..5a5e434e 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -407,7 +407,7 @@ void _syncTests({ containsPair( 'status', containsPair( - 'internal_applied_checkpoint_request_id', anything))))), + 'internal_last_applied_checkpoint_request_id', anything))))), ); expect( instructions, @@ -624,8 +624,8 @@ void _syncTests({ 'UpdateSyncStatus', containsPair( 'status', - containsPair( - 'internal_applied_checkpoint_request_id', anything))))), + containsPair('internal_last_applied_checkpoint_request_id', + anything))))), ); expect(lastAppliedCheckpointRequestId(), isNull); @@ -652,13 +652,18 @@ void _syncTests({ ), ), ); + expect( + appliedInstructions, + contains(containsPair('LogLine', + {'severity': 'DEBUG', 'line': 'Applied checkpoint request id 1'})), + ); expect( appliedInstructions, contains(containsPair( 'UpdateSyncStatus', containsPair( 'status', - containsPair('internal_applied_checkpoint_request_id', 1), + containsPair('internal_last_applied_checkpoint_request_id', 1), ), )), ); @@ -678,7 +683,7 @@ void _syncTests({ containsPair( 'status', containsPair( - 'internal_applied_checkpoint_request_id', anything))))), + 'internal_last_applied_checkpoint_request_id', anything))))), ); final [row] = db.select('select powersync_offline_sync_status();'); @@ -691,6 +696,29 @@ void _syncTests({ db.select(r"SELECT * FROM ps_buckets WHERE name = '$local'"), isEmpty); }); + syncTest('disconnect clears applied checkpoint request id from status', (_) { + invokeControl('start', null); + + pushCheckpoint(buckets: priorityBuckets, writeCheckpoint: '1'); + pushCheckpointComplete(); + + final instructions = invokeControl('stop', null); + expect( + instructions, + contains(containsPair( + 'UpdateSyncStatus', + containsPair( + 'status', + allOf( + containsPair('connected', false), + isNot(containsPair( + 'internal_last_applied_checkpoint_request_id', anything)), + ), + ), + )), + ); + }); + syncTest('local writes clear checkpoint request high-water marks', (_) { invokeControl('start', null); @@ -1160,7 +1188,13 @@ void _syncTests({ // Now complete the upload process. probeLocalTargetOp(1); - invokeControl('completed_upload', null); + final uploadCompleteInstructions = + invokeControl('completed_upload', null); + expect( + uploadCompleteInstructions, + contains(containsPair('LogLine', + {'severity': 'DEBUG', 'line': 'Applied checkpoint request id 1'})), + ); // This should apply the pending write checkpoint. expect(fetchRows(), [ diff --git a/docs/sync.md b/docs/sync.md index 6b502d95..9c2e231c 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -64,7 +64,7 @@ what SDKs need to do. ids; it does not update that gate. - Resolve explicit checkpoint waiters from `DidCompleteSync.applied_checkpoint_request_id`. SDKs that drive waiters from status snapshots can also watch - `UpdateSyncStatus.status.internal_applied_checkpoint_request_id`. Treat that status field as + `UpdateSyncStatus.status.internal_last_applied_checkpoint_request_id`. Treat that status field as runtime-only SDK state, not persisted checkpoint state or app-visible progress. Most `powersync_control` commands return a JSON-encoded array of instructions for the client. @@ -107,7 +107,7 @@ interface UpdateSyncStatus { priority_status: [], downloading: null | DownloadProgress, streams: [], - internal_applied_checkpoint_request_id?: number, + internal_last_applied_checkpoint_request_id?: number, } interface DidCompleteSync { diff --git a/docs/write-checkpoint-requests.md b/docs/write-checkpoint-requests.md index a1f3639b..bd18fb94 100644 --- a/docs/write-checkpoint-requests.md +++ b/docs/write-checkpoint-requests.md @@ -121,10 +121,12 @@ DidCompleteSync { applied_checkpoint_request_id?: checkpoint.write_checkpoint } SDKs should resolve `requestCheckpoint()` / `waitForSync()` waiters when this value is greater than or equal to the requested id. -Core also includes the same applied request id in -`UpdateSyncStatus.status.internal_applied_checkpoint_request_id` for the status update emitted with -the completed checkpoint. SDKs may use that for status-stream waiters, but should treat it as -internal, runtime-only state rather than app-visible progress or persisted checkpoint state. +Core also includes the applied request id in +`UpdateSyncStatus.status.internal_last_applied_checkpoint_request_id` when a status update follows a +checkpoint apply with a `write_checkpoint`. Later status updates without an applied request id, +including disconnect updates, clear the field. SDKs may use this for status-stream waiters, but +should treat it as internal, runtime-only state rather than app-visible progress or persisted +checkpoint state. ## Control Commands From a6d73faded23c97bff4a2ef879b5fbe50e97c11f Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 9 Jul 2026 08:37:26 +0200 Subject: [PATCH 24/29] expose current checkpoint request id for retries --- crates/core/src/sync/interface.rs | 8 ++++++++ dart/test/sync_test.dart | 29 +++++++++++++++++++++++++++++ docs/sync.md | 13 ++++++++++--- docs/write-checkpoint-requests.md | 17 +++++++++++++++++ 4 files changed, 64 insertions(+), 3 deletions(-) diff --git a/crates/core/src/sync/interface.rs b/crates/core/src/sync/interface.rs index c0ad6fad..17d387a7 100644 --- a/crates/core/src/sync/interface.rs +++ b/crates/core/src/sync/interface.rs @@ -289,6 +289,14 @@ pub fn register(db: *mut sqlite::sqlite3, state: Rc) -> Result<() return Ok(()); } + "current_checkpoint_request_id" => { + let adapter = state.storage_adapter(db)?; + match adapter.last_checkpoint_request_id()? { + Some(request_id) => ctx.result_int64(request_id), + None => ctx.result_null(), + } + return Ok(()); + } "line_text" => SyncControlRequest::SyncEvent(SyncEvent::TextLine { data: if payload.value_type() == ColumnType::Text { payload.text() diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index 5a5e434e..b00d94b7 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -197,6 +197,10 @@ void _syncTests({ return invokeControlScalar('local_target_op', opId); } + Object? currentCheckpointRequestId() { + return invokeControlScalar('current_checkpoint_request_id', null); + } + ResultSet fetchRows() { return db.select('select * from items'); } @@ -462,6 +466,31 @@ void _syncTests({ expect(lastRequestedCheckpointRequestId(), 3); }); + syncTest('reports current checkpoint request id without incrementing', (_) { + expect(currentCheckpointRequestId(), isNull); + + invokeControlRaw('start', null); + invokeControlRaw('seed_checkpoint_request_id', 1); + expect(currentCheckpointRequestId(), 1); + + expect(currentCheckpointRequestId(), 1); + expect(nextCheckpointRequestId(), 2); + expect(currentCheckpointRequestId(), 2); + expect(lastRequestedCheckpointRequestId(), 2); + + pushCheckpoint(buckets: priorityBuckets, writeCheckpoint: '2'); + pushCheckpointComplete(); + expect(lastAppliedCheckpointRequestId(), 2); + expect(currentCheckpointRequestId(), 2); + + db.execute("insert into items (id, col) values ('local', 'data');"); + expect(lastAppliedCheckpointRequestId(), isNull); + expect(currentCheckpointRequestId(), 2); + + expect(nextCheckpointRequestId(), 3); + expect(currentCheckpointRequestId(), 3); + }); + syncTest('seeds requested checkpoint request ids from service state', (_) { final startInstructions = invokeControlRaw('start', null); expect(streamLastCheckpointRequestId(startInstructions), isNull); diff --git a/docs/sync.md b/docs/sync.md index 9c2e231c..406b291d 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -38,11 +38,14 @@ The following commands are supported: 10. `next_checkpoint_request_id`: No payload. During an active sync iteration after checkpoint request state exists locally, allocates and returns the next checkpoint request id as an integer result. -11. `local_target_op`: Payload is `null`, an integer, or an integer string. Probes, updates or +11. `current_checkpoint_request_id`: No payload. Returns the current checkpoint request sequence + value as an integer result, or SQL `NULL` if absent. This command does not allocate a new id and + can run outside a sync iteration. +12. `local_target_op`: Payload is `null`, an integer, or an integer string. Probes, updates or clears the local target op and returns the previously-observed value as an integer result, or SQL `NULL` if there was no target. This command can run outside of a sync iteration and does not affect it. -12. `seed_checkpoint_request_id`: Payload is a positive integer or integer string. After receiving +13. `seed_checkpoint_request_id`: Payload is a positive integer or integer string. After receiving `EstablishSyncStream`, SDKs should reconcile the local hint with service-side checkpoint-request state, then seed core with the accepted positive id. @@ -62,13 +65,17 @@ what SDKs need to do. id to the service, then store the accepted id with `powersync_control('local_target_op', id)`. - `local_target_op` is the apply gate for local writes. `next_checkpoint_request_id` only allocates ids; it does not update that gate. +- To retry a checkpoint request without incrementing the counter, read + `powersync_control('current_checkpoint_request_id', NULL)` and repost that id when the SDK's + runtime last-applied checkpoint request id is absent or lower. - Resolve explicit checkpoint waiters from `DidCompleteSync.applied_checkpoint_request_id`. SDKs that drive waiters from status snapshots can also watch `UpdateSyncStatus.status.internal_last_applied_checkpoint_request_id`. Treat that status field as runtime-only SDK state, not persisted checkpoint state or app-visible progress. Most `powersync_control` commands return a JSON-encoded array of instructions for the client. -`next_checkpoint_request_id` and `local_target_op` return scalar values directly. +`next_checkpoint_request_id`, `current_checkpoint_request_id` and `local_target_op` return values +directly. ```typescript type Instruction = { LogLine: LogLine } diff --git a/docs/write-checkpoint-requests.md b/docs/write-checkpoint-requests.md index bd18fb94..372c880b 100644 --- a/docs/write-checkpoint-requests.md +++ b/docs/write-checkpoint-requests.md @@ -59,6 +59,16 @@ the service lost its record, posting the local hint recreates the service-side s seeded request counter. SDKs should wait for the connection reconciliation and seed step before creating checkpoint requests. +To retry an existing checkpoint request without advancing the counter, SDKs can read: + +```text +powersync_control('current_checkpoint_request_id', NULL) +``` + +Core returns the current sequence value as a SQLite integer, or SQL `NULL` when the counter has not +been seeded. SDKs can compare this with their runtime last-applied checkpoint request id and repost +the current id when the applied id is absent or lower. + ## Local Write Gate A local write records CRUD and sets: @@ -145,6 +155,13 @@ checkpoint state. to the service, retrying posts the same id again, which is safe because the service treats the latest posted id as effective state. +`powersync_control('current_checkpoint_request_id', NULL)` + +- Returns: current checkpoint request sequence value as a SQLite integer, or SQL `NULL` if absent. +- Does not allocate a new id. +- SDKs should compare this with their runtime last-applied checkpoint request id to decide whether + to repost the current id. + `powersync_control('local_target_op', value)` - Payload `NULL`: return current target without changing it. From 7a9de7212970389c36a3771ed14d8ee2bd9c5a1c Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 9 Jul 2026 10:04:19 +0200 Subject: [PATCH 25/29] rename to target_checkpoint_request_id --- crates/core/src/crud_vtab.rs | 2 +- crates/core/src/migrations.rs | 12 ++--- crates/core/src/sync/interface.rs | 14 +++--- crates/core/src/sync/storage_adapter.rs | 46 ++++++++---------- crates/core/src/sync/streaming_sync.rs | 2 +- crates/core/src/sync/sync_local.rs | 2 +- dart/test/crud_test.dart | 6 +-- dart/test/migration_test.dart | 12 ++--- dart/test/sync_test.dart | 62 ++++++++++++------------- dart/test/utils/migration_fixtures.dart | 2 +- docs/schema.md | 2 +- docs/sync.md | 18 +++---- docs/write-checkpoint-requests.md | 41 ++++++++-------- 13 files changed, 110 insertions(+), 111 deletions(-) diff --git a/crates/core/src/crud_vtab.rs b/crates/core/src/crud_vtab.rs index 7d2acfe4..f33508cd 100644 --- a/crates/core/src/crud_vtab.rs +++ b/crates/core/src/crud_vtab.rs @@ -252,7 +252,7 @@ impl SimpleCrudTransactionMode { // restart. Keeping them around could open the apply gate for a newly allocated target // id that compares below a stale seen value. db.exec_safe(formatcp!( - "INSERT OR REPLACE INTO ps_kv(key, value) VALUES('local_target_op', {MAX_OP_ID}); + "INSERT OR REPLACE INTO ps_kv(key, value) VALUES('target_checkpoint_request_id', {MAX_OP_ID}); DELETE FROM ps_kv WHERE key IN ('last_seen_checkpoint_request_id', 'last_applied_checkpoint_request_id')" ))?; self.had_writes = true; diff --git a/crates/core/src/migrations.rs b/crates/core/src/migrations.rs index 7139b5ca..87c56cb9 100644 --- a/crates/core/src/migrations.rs +++ b/crates/core/src/migrations.rs @@ -469,7 +469,7 @@ DROP TABLE ps_sync_state_old; // so it becomes the last seen checkpoint request. // // `$local.target_op` can either be a concrete legacy write checkpoint id or a sentinel such - // as i64::MAX while local writes are pending. Store it separately as `local_target_op`. + // as i64::MAX while local writes are pending. Store it separately as `target_checkpoint_request_id`. // Seeding `last_requested_checkpoint_request_id` from a concrete target would be possible, // but should be redundant because SDKs reconcile the request counter with service state on // connect before advancing it through `next_checkpoint_request_id`. @@ -496,7 +496,7 @@ DELETE FROM ps_kv WHERE key IN ( 'last_applied_checkpoint_request_id', 'last_seen_checkpoint_request_id', - 'local_target_op' + 'target_checkpoint_request_id' ); INSERT INTO ps_kv(key, value) @@ -512,7 +512,7 @@ SELECT 'last_seen_checkpoint_request_id', last_op AND last_op > 0; INSERT INTO ps_kv(key, value) -SELECT 'local_target_op', target_op +SELECT 'target_checkpoint_request_id', target_op FROM ps_buckets WHERE name = '$local' AND target_op > 0; @@ -531,7 +531,7 @@ ALTER TABLE ps_buckets DROP COLUMN target_op; // `$local.pending_delete = 1` marked this as a synthetic local-only bucket instead of a // normal service bucket. Restore each old progress column from its matching ps_kv key. // The 0 defaults cover a local target that exists before any checkpoint has been seen or - // applied. If `local_target_op` is absent, don't create a `$local` row: the old + // applied. If `target_checkpoint_request_id` is absent, don't create a `$local` row: the old // implementation also didn't have a `$local` bucket unless there was local target state to // track. const DOWN_STATEMENTS: &[&str] = &[ @@ -582,10 +582,10 @@ SELECT '$local', 1, seen, applied, target SELECT IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = 'last_seen_checkpoint_request_id'), 0) AS seen, IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = 'last_applied_checkpoint_request_id'), 0) AS applied, - (SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = 'local_target_op') AS target + (SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = 'target_checkpoint_request_id') AS target ) WHERE EXISTS ( - SELECT 1 FROM ps_kv WHERE key = 'local_target_op' + SELECT 1 FROM ps_kv WHERE key = 'target_checkpoint_request_id' ) ON CONFLICT(name) DO UPDATE SET pending_delete = excluded.pending_delete, diff --git a/crates/core/src/sync/interface.rs b/crates/core/src/sync/interface.rs index 17d387a7..be21d84e 100644 --- a/crates/core/src/sync/interface.rs +++ b/crates/core/src/sync/interface.rs @@ -273,17 +273,17 @@ pub fn register(db: *mut sqlite::sqlite3, state: Rc) -> Result<() ctx.result_int64(request_id); return Ok(()); } - "local_target_op" => { - let target_op = parse_optional_i64_payload( + "target_checkpoint_request_id" => { + let target = parse_optional_i64_payload( *payload, - "local target op", - "local target op must be an integer, integer string, or null", + "target checkpoint request id", + "target checkpoint request id must be an integer, integer string, or null", )?; let adapter = state.storage_adapter(db)?; - let previous_target_op = adapter.probe_local_target_op(target_op)?; + let previous_target = adapter.probe_target_checkpoint_request_id(target)?; - match previous_target_op { - Some(target_op) => ctx.result_int64(target_op), + match previous_target { + Some(target) => ctx.result_int64(target), None => ctx.result_null(), } diff --git a/crates/core/src/sync/storage_adapter.rs b/crates/core/src/sync/storage_adapter.rs index 1335390e..f0267783 100644 --- a/crates/core/src/sync/storage_adapter.rs +++ b/crates/core/src/sync/storage_adapter.rs @@ -30,10 +30,10 @@ const LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY: &str = "last_requested_checkpoin const LAST_SEEN_CHECKPOINT_REQUEST_ID_KEY: &str = "last_seen_checkpoint_request_id"; const LAST_APPLIED_CHECKPOINT_REQUEST_ID_KEY: &str = "last_applied_checkpoint_request_id"; -// Tracks the local target used to block applying downloaded rows while local writes are -// outstanding. When present, this is normally either the max-op sentinel for pending local writes or -// a concrete checkpoint request id also stored in LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY. -const LOCAL_TARGET_OP_KEY: &str = "local_target_op"; +// Tracks the target used to block applying downloaded rows while local writes are outstanding. +// When present, this is normally either the max-op sentinel for pending local writes or a concrete +// checkpoint request id also stored in LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY. +const TARGET_CHECKPOINT_REQUEST_ID_KEY: &str = "target_checkpoint_request_id"; /// An adapter for storing sync state. /// @@ -509,13 +509,11 @@ WHERE bucket = ?1", Ok(()) } - pub fn local_state(&self) -> Result, PowerSyncError> { - Ok(self - .read_i64_kv(LOCAL_TARGET_OP_KEY)? - .map(|target_op| LocalState { target_op })) + pub fn target_checkpoint_request_id(&self) -> Result, PowerSyncError> { + self.read_i64_kv(TARGET_CHECKPOINT_REQUEST_ID_KEY) } - /// Probes and optionally updates the local target op used to block applying downloaded rows + /// Probes and optionally updates the target checkpoint request id used to block applying downloaded rows /// while local writes are outstanding. /// /// In the write-checkpoint flow, callers allocate a checkpoint request id, post it to the @@ -523,37 +521,37 @@ WHERE bucket = ?1", /// once the request succeeds. This is also used for older services where the SDK cannot create /// checkpoint requests explicitly. /// - /// The target op can also be used internally as a sentinel value such as max op id while local + /// The target can also be used internally as a sentinel value such as max op id while local /// writes are pending, so it must not always be interpreted as a checkpoint request id. /// /// This only updates the apply gate. It does not allocate, seed or overwrite /// `last_requested_checkpoint_request_id`, which is managed by `seed_checkpoint_request_id` and /// `next_checkpoint_request_id`. /// - /// Returns the target op value from before this call. When `target_op` is `None`, this only - /// reads the current value. A `target_op` of zero clears the stored target, removing the apply + /// Returns the target value from before this call. When `target` is `None`, this only + /// reads the current value. A `target` of zero clears the stored target, removing the apply /// gate entirely; any other value overwrites it. /// /// Negative values are rejected when parsing the `powersync_control` payload, before this is /// called. - pub fn probe_local_target_op( + pub fn probe_target_checkpoint_request_id( &self, - target_op: Option, + target: Option, ) -> Result, PowerSyncError> { - let previous_target_op = self.local_state()?.map(|state| state.target_op); + let previous_target = self.target_checkpoint_request_id()?; - let Some(target_op) = target_op else { - return Ok(previous_target_op); + let Some(target) = target else { + return Ok(previous_target); }; - if target_op == 0 { - self.delete_kv(LOCAL_TARGET_OP_KEY)?; - return Ok(previous_target_op); + if target == 0 { + self.delete_kv(TARGET_CHECKPOINT_REQUEST_ID_KEY)?; + return Ok(previous_target); } - self.write_i64_kv(LOCAL_TARGET_OP_KEY, target_op)?; + self.write_i64_kv(TARGET_CHECKPOINT_REQUEST_ID_KEY, target)?; - Ok(previous_target_op) + Ok(previous_target) } /// Persists the checkpoint request id observed in a complete sync checkpoint. @@ -653,10 +651,6 @@ ON CONFLICT(key) DO UPDATE SET value = excluded.value", } } -pub struct LocalState { - pub target_op: i64, -} - pub struct BucketInfo { pub id: i64, pub last_applied_op: i64, diff --git a/crates/core/src/sync/streaming_sync.rs b/crates/core/src/sync/streaming_sync.rs index a75a6cd8..208553a7 100644 --- a/crates/core/src/sync/streaming_sync.rs +++ b/crates/core/src/sync/streaming_sync.rs @@ -651,7 +651,7 @@ impl StreamingSyncIteration { return Ok(()); }; - let target_write = self.adapter.local_state()?.map(|e| e.target_op); + let target_write = self.adapter.target_checkpoint_request_id()?; if checkpoint.write_checkpoint < target_write { // Note: None < Some(x). The pending checkpoint does not contain the write // checkpoint created during the upload, so we don't have to try applying it, it's diff --git a/crates/core/src/sync/sync_local.rs b/crates/core/src/sync/sync_local.rs index aef8f8d4..9555d2aa 100644 --- a/crates/core/src/sync/sync_local.rs +++ b/crates/core/src/sync/sync_local.rs @@ -70,7 +70,7 @@ impl<'a> SyncOperation<'a> { "SELECT 1 FROM ps_kv AS target LEFT JOIN ps_kv AS seen ON seen.key = 'last_seen_checkpoint_request_id' -WHERE target.key = 'local_target_op' +WHERE target.key = 'target_checkpoint_request_id' AND CAST(target.value AS INTEGER) > COALESCE(CAST(seen.value AS INTEGER), 0)", )?; diff --git a/dart/test/crud_test.dart b/dart/test/crud_test.dart index 3745f0fa..74ca5743 100644 --- a/dart/test/crud_test.dart +++ b/dart/test/crud_test.dart @@ -249,7 +249,7 @@ void main() { }); }); - test('updates local target op and updated rows', () { + test('updates target checkpoint request id and updated rows', () { // Stale high-water marks (e.g. from before a request counter restart) must be cleared // by a local write, so they can't open the apply gate for a smaller new target id. db.execute(''' @@ -274,10 +274,10 @@ INSERT INTO ps_kv(key, value) VALUES isEmpty); expect( db.select( - "SELECT key, value FROM ps_kv WHERE key LIKE '%checkpoint_request_id' OR key = 'local_target_op'"), + "SELECT key, value FROM ps_kv WHERE key LIKE '%checkpoint_request_id'"), [ { - 'key': 'local_target_op', + 'key': 'target_checkpoint_request_id', 'value': 9223372036854775807, } ]); diff --git a/dart/test/migration_test.dart b/dart/test/migration_test.dart index a65c72c3..ef9387cc 100644 --- a/dart/test/migration_test.dart +++ b/dart/test/migration_test.dart @@ -109,7 +109,7 @@ VALUES(1, '$local', 5, 6, 7, 0, 0, 1, 0, 0, 0); expect(db.select('SELECT key, value FROM ps_kv ORDER BY key'), [ {'key': 'last_applied_checkpoint_request_id', 'value': 5}, {'key': 'last_seen_checkpoint_request_id', 'value': 6}, - {'key': 'local_target_op', 'value': 7}, + {'key': 'target_checkpoint_request_id', 'value': 7}, ]); expect(db.select(r"SELECT * FROM ps_buckets WHERE name = '$local'"), isEmpty); @@ -142,7 +142,7 @@ VALUES(1, '$local', 5, 6, 9223372036854775807, 0, 0, 1, 0, 0, 0); expect(db.select('SELECT key, value FROM ps_kv ORDER BY key'), [ {'key': 'last_applied_checkpoint_request_id', 'value': 5}, {'key': 'last_seen_checkpoint_request_id', 'value': 6}, - {'key': 'local_target_op', 'value': 9223372036854775807}, + {'key': 'target_checkpoint_request_id', 'value': 9223372036854775807}, ]); }); @@ -159,7 +159,7 @@ VALUES(1, '$local', 0, 0, 9223372036854775807, 0, 0, 1, 0, 0, 0); // The max-op sentinel is valid local target state, but target ops no longer seed // last_requested_checkpoint_request_id. expect(db.select('SELECT key, value FROM ps_kv ORDER BY key'), [ - {'key': 'local_target_op', 'value': 9223372036854775807}, + {'key': 'target_checkpoint_request_id', 'value': 9223372036854775807}, ]); }); @@ -170,7 +170,7 @@ INSERT INTO ps_kv(key, value) VALUES ('last_requested_checkpoint_request_id', 7), ('last_seen_checkpoint_request_id', 6), ('last_applied_checkpoint_request_id', 5), - ('local_target_op', 7); + ('target_checkpoint_request_id', 7); '''); db.executeInTx('select powersync_test_migration(13)'); @@ -196,7 +196,7 @@ INSERT INTO ps_kv(key, value) VALUES ('last_requested_checkpoint_request_id', 7), ('last_seen_checkpoint_request_id', 6), ('last_applied_checkpoint_request_id', 5), - ('local_target_op', 7); + ('target_checkpoint_request_id', 7); '''); db.executeInTx('select powersync_test_migration(13)'); @@ -216,7 +216,7 @@ UPDATE ps_buckets {'key': 'last_applied_checkpoint_request_id', 'value': 8}, {'key': 'last_requested_checkpoint_request_id', 'value': 7}, {'key': 'last_seen_checkpoint_request_id', 'value': 8}, - {'key': 'local_target_op', 'value': 9}, + {'key': 'target_checkpoint_request_id', 'value': 9}, ]); expect(db.select(r"SELECT * FROM ps_buckets WHERE name = '$local'"), isEmpty); diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index b00d94b7..8ee4dfe9 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -193,8 +193,8 @@ void _syncTests({ return invokeControlScalar('next_checkpoint_request_id', null) as int; } - Object? probeLocalTargetOp([Object? opId]) { - return invokeControlScalar('local_target_op', opId); + Object? probeTargetCheckpointRequestId([Object? opId]) { + return invokeControlScalar('target_checkpoint_request_id', opId); } Object? currentCheckpointRequestId() { @@ -569,61 +569,61 @@ void _syncTests({ contains('Checkpoint request state has not been seeded'), )), ); - expect(probeLocalTargetOp(), isNull); + expect(probeTargetCheckpointRequestId(), isNull); }); - syncTest('probes and updates local target op without sync iteration', (_) { - expect(probeLocalTargetOp(), isNull); - expect(probeLocalTargetOp(1), isNull); + syncTest('probes and updates target checkpoint request id without sync iteration', (_) { + expect(probeTargetCheckpointRequestId(), isNull); + expect(probeTargetCheckpointRequestId(1), isNull); expect(lastRequestedCheckpointRequestId(), isNull); - expect(probeLocalTargetOp(), 1); + expect(probeTargetCheckpointRequestId(), 1); - expect(probeLocalTargetOp(2), 1); + expect(probeTargetCheckpointRequestId(2), 1); expect(lastRequestedCheckpointRequestId(), isNull); - expect(probeLocalTargetOp(), 2); + expect(probeTargetCheckpointRequestId(), 2); }); - syncTest('accepts text checkpoint request ids for local target op', (_) { - expect(probeLocalTargetOp('1'), isNull); + syncTest('accepts text checkpoint request ids for target checkpoint request id', (_) { + expect(probeTargetCheckpointRequestId('1'), isNull); expect(lastRequestedCheckpointRequestId(), isNull); - expect(probeLocalTargetOp(), 1); + expect(probeTargetCheckpointRequestId(), 1); }); - syncTest('rejects negative local target ops', (_) { + syncTest('rejects negative target checkpoint request ids', (_) { expect( - () => invokeControlRaw('local_target_op', -1), + () => invokeControlRaw('target_checkpoint_request_id', -1), throwsA(isSqliteException( 3091, - contains('local target op must be a non-negative integer'), + contains('target checkpoint request id must be a non-negative integer'), )), ); }); - syncTest('local target op does not update checkpoint request id', (_) { + syncTest('target checkpoint request id does not update checkpoint request id', (_) { invokeControlRaw('start', null); invokeControlRaw('seed_checkpoint_request_id', 10); expect(lastRequestedCheckpointRequestId(), 10); - expect(probeLocalTargetOp(7), isNull); - expect(probeLocalTargetOp(), 7); + expect(probeTargetCheckpointRequestId(7), isNull); + expect(probeTargetCheckpointRequestId(), 7); expect(lastRequestedCheckpointRequestId(), 10); }); syncTest('does not store target ops as checkpoint request id', (_) { - expect(probeLocalTargetOp(0), isNull); + expect(probeTargetCheckpointRequestId(0), isNull); expect(lastRequestedCheckpointRequestId(), isNull); - expect(probeLocalTargetOp(), isNull); + expect(probeTargetCheckpointRequestId(), isNull); - expect(probeLocalTargetOp(1), isNull); + expect(probeTargetCheckpointRequestId(1), isNull); expect(lastRequestedCheckpointRequestId(), isNull); - expect(probeLocalTargetOp(), 1); + expect(probeTargetCheckpointRequestId(), 1); - expect(probeLocalTargetOp(0), 1); - expect(probeLocalTargetOp(), isNull); + expect(probeTargetCheckpointRequestId(0), 1); + expect(probeTargetCheckpointRequestId(), isNull); - expect(probeLocalTargetOp(9223372036854775807), isNull); + expect(probeTargetCheckpointRequestId(9223372036854775807), isNull); expect(lastRequestedCheckpointRequestId(), isNull); - expect(probeLocalTargetOp(), 9223372036854775807); + expect(probeTargetCheckpointRequestId(), 9223372036854775807); }); syncTest('does not persist placeholder checkpoint request id', (_) { @@ -766,7 +766,7 @@ void _syncTests({ "SELECT 1 FROM ps_kv WHERE key = 'last_seen_checkpoint_request_id'"), isEmpty, ); - expect(probeLocalTargetOp(), 9223372036854775807); + expect(probeTargetCheckpointRequestId(), 9223372036854775807); }); test('clearing database clears sync status', () { @@ -1216,7 +1216,7 @@ void _syncTests({ ]); // Now complete the upload process. - probeLocalTargetOp(1); + probeTargetCheckpointRequestId(1); final uploadCompleteInstructions = invokeControl('completed_upload', null); expect( @@ -1238,7 +1238,7 @@ void _syncTests({ // Complete upload process db.execute('DELETE FROM ps_crud'); - probeLocalTargetOp(1); + probeTargetCheckpointRequestId(1); expect(invokeControl('completed_upload', null), isEmpty); expect(lastRequestedCheckpointRequestId(), 1); @@ -1268,7 +1268,7 @@ void _syncTests({ ]); // Now the upload is complete and requests a write checkpoint - probeLocalTargetOp(1); + probeTargetCheckpointRequestId(1); expect(invokeControl('completed_upload', null), isEmpty); // Which triggers a new iteration @@ -1305,7 +1305,7 @@ void _syncTests({ db.execute("insert into items (id, col) values ('local2', 'data2');"); // Now the upload is complete and requests a write checkpoint - probeLocalTargetOp(1); + probeTargetCheckpointRequestId(1); expect(invokeControl('completed_upload', null), [ containsPair('LogLine', { 'severity': 'WARNING', diff --git a/dart/test/utils/migration_fixtures.dart b/dart/test/utils/migration_fixtures.dart index 7f6e6ba8..35c23b7a 100644 --- a/dart/test/utils/migration_fixtures.dart +++ b/dart/test/utils/migration_fixtures.dart @@ -542,7 +542,7 @@ Map _expectedState() { ;INSERT INTO ps_migration(id, down_migrations) VALUES(13, '[{"sql":"UPDATE ps_stream_subscriptions SET expires_at = expires_at / 1000000, last_synced_at = last_synced_at / 1000000"},{"sql":"ALTER TABLE ps_sync_state RENAME TO ps_sync_state_new"},{"sql":"CREATE TABLE ps_sync_state (\n priority INTEGER NOT NULL PRIMARY KEY,\n last_synced_at TEXT NOT NULL\n) STRICT;"},{"sql":"INSERT INTO ps_sync_state (priority, last_synced_at) SELECT priority, datetime(last_synced_at / 1000000, ''unixepoch'') FROM ps_sync_state_new"},{"sql":"DROP TABLE ps_sync_state_new"},{"sql":"DELETE FROM ps_migration WHERE id >= 13"}]')''', }; state[14] = '''${state[13]!.trim().replaceFirst(' target_op INTEGER NOT NULL DEFAULT 0,\n', '')} -;INSERT INTO ps_migration(id, down_migrations) VALUES(14, '[{"sql":"ALTER TABLE ps_buckets RENAME TO ps_buckets_14"},{"sql":"DROP INDEX ps_buckets_name"},{"sql":"CREATE TABLE ps_buckets(\\n id INTEGER PRIMARY KEY,\\n name TEXT NOT NULL,\\n last_applied_op INTEGER NOT NULL DEFAULT 0,\\n last_op INTEGER NOT NULL DEFAULT 0,\\n target_op INTEGER NOT NULL DEFAULT 0,\\n add_checksum INTEGER NOT NULL DEFAULT 0,\\n op_checksum INTEGER NOT NULL DEFAULT 0,\\n pending_delete INTEGER NOT NULL DEFAULT 0\\n) STRICT"},{"sql":"CREATE UNIQUE INDEX ps_buckets_name ON ps_buckets (name)"},{"sql":"ALTER TABLE ps_buckets ADD COLUMN count_at_last INTEGER NOT NULL DEFAULT 0"},{"sql":"ALTER TABLE ps_buckets ADD COLUMN count_since_last INTEGER NOT NULL DEFAULT 0"},{"sql":"ALTER TABLE ps_buckets ADD COLUMN downloaded_size INTEGER NOT NULL DEFAULT 0"},{"sql":"INSERT INTO ps_buckets(\\n id,\\n name,\\n last_applied_op,\\n last_op,\\n add_checksum,\\n op_checksum,\\n pending_delete,\\n count_at_last,\\n count_since_last,\\n downloaded_size\\n)\\nSELECT\\n id,\\n name,\\n last_applied_op,\\n last_op,\\n add_checksum,\\n op_checksum,\\n pending_delete,\\n count_at_last,\\n count_since_last,\\n downloaded_size\\nFROM ps_buckets_14"},{"sql":"DROP TABLE ps_buckets_14"},{"sql":"INSERT INTO ps_buckets(name, pending_delete, last_op, last_applied_op, target_op)\\nSELECT ''\$local'', 1, seen, applied, target\\n FROM (\\n SELECT\\n IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''last_seen_checkpoint_request_id''), 0) AS seen,\\n IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''last_applied_checkpoint_request_id''), 0) AS applied,\\n (SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''local_target_op'') AS target\\n )\\n WHERE EXISTS (\\n SELECT 1 FROM ps_kv WHERE key = ''local_target_op''\\n )\\nON CONFLICT(name) DO UPDATE SET\\n pending_delete = excluded.pending_delete,\\n last_op = excluded.last_op,\\n last_applied_op = excluded.last_applied_op,\\n target_op = excluded.target_op"},{"sql":"DELETE FROM ps_migration WHERE id >= 14"}]')'''; +;INSERT INTO ps_migration(id, down_migrations) VALUES(14, '[{"sql":"ALTER TABLE ps_buckets RENAME TO ps_buckets_14"},{"sql":"DROP INDEX ps_buckets_name"},{"sql":"CREATE TABLE ps_buckets(\\n id INTEGER PRIMARY KEY,\\n name TEXT NOT NULL,\\n last_applied_op INTEGER NOT NULL DEFAULT 0,\\n last_op INTEGER NOT NULL DEFAULT 0,\\n target_op INTEGER NOT NULL DEFAULT 0,\\n add_checksum INTEGER NOT NULL DEFAULT 0,\\n op_checksum INTEGER NOT NULL DEFAULT 0,\\n pending_delete INTEGER NOT NULL DEFAULT 0\\n) STRICT"},{"sql":"CREATE UNIQUE INDEX ps_buckets_name ON ps_buckets (name)"},{"sql":"ALTER TABLE ps_buckets ADD COLUMN count_at_last INTEGER NOT NULL DEFAULT 0"},{"sql":"ALTER TABLE ps_buckets ADD COLUMN count_since_last INTEGER NOT NULL DEFAULT 0"},{"sql":"ALTER TABLE ps_buckets ADD COLUMN downloaded_size INTEGER NOT NULL DEFAULT 0"},{"sql":"INSERT INTO ps_buckets(\\n id,\\n name,\\n last_applied_op,\\n last_op,\\n add_checksum,\\n op_checksum,\\n pending_delete,\\n count_at_last,\\n count_since_last,\\n downloaded_size\\n)\\nSELECT\\n id,\\n name,\\n last_applied_op,\\n last_op,\\n add_checksum,\\n op_checksum,\\n pending_delete,\\n count_at_last,\\n count_since_last,\\n downloaded_size\\nFROM ps_buckets_14"},{"sql":"DROP TABLE ps_buckets_14"},{"sql":"INSERT INTO ps_buckets(name, pending_delete, last_op, last_applied_op, target_op)\\nSELECT ''\$local'', 1, seen, applied, target\\n FROM (\\n SELECT\\n IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''last_seen_checkpoint_request_id''), 0) AS seen,\\n IFNULL((SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''last_applied_checkpoint_request_id''), 0) AS applied,\\n (SELECT CAST(value AS INTEGER) FROM ps_kv WHERE key = ''target_checkpoint_request_id'') AS target\\n )\\n WHERE EXISTS (\\n SELECT 1 FROM ps_kv WHERE key = ''target_checkpoint_request_id''\\n )\\nON CONFLICT(name) DO UPDATE SET\\n pending_delete = excluded.pending_delete,\\n last_op = excluded.last_op,\\n last_applied_op = excluded.last_applied_op,\\n target_op = excluded.target_op"},{"sql":"DELETE FROM ps_migration WHERE id >= 14"}]')'''; return state; } diff --git a/docs/schema.md b/docs/schema.md index 35b4fb5a..2140cb27 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -31,7 +31,7 @@ a checkpoint and that we have validated its checksum). 8. `count_since_last`: The amount of operations downloaded since the last verified checkpoint. Schema version 14 removes the legacy `target_op` column after migrating `$local.target_op` to -`ps_kv.local_target_op`, and deletes the `$local` row so `ps_buckets` only contains real sync +`ps_kv.target_checkpoint_request_id`, and deletes the `$local` row so `ps_buckets` only contains real sync buckets. This makes older SDKs fail with a hard SQLite error if they try to keep using the migrated database without downgrading. That failure is deliberate — including for multi-process deployments where processes with mixed SDK versions share one database — because an older SDK silently diff --git a/docs/sync.md b/docs/sync.md index 406b291d..7baf4ae3 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -41,8 +41,8 @@ The following commands are supported: 11. `current_checkpoint_request_id`: No payload. Returns the current checkpoint request sequence value as an integer result, or SQL `NULL` if absent. This command does not allocate a new id and can run outside a sync iteration. -12. `local_target_op`: Payload is `null`, an integer, or an integer string. Probes, updates or - clears the local target op and returns the previously-observed value as an integer result, or +12. `target_checkpoint_request_id`: Payload is `null`, an integer, or an integer string. Probes, updates or + clears the target checkpoint request id and returns the previously-observed value as an integer result, or SQL `NULL` if there was no target. This command can run outside of a sync iteration and does not affect it. 13. `seed_checkpoint_request_id`: Payload is a positive integer or integer string. After receiving @@ -62,19 +62,21 @@ what SDKs need to do. lost its local value and recreates service-side state when the service lost its record. - Wait for seeding to complete before creating checkpoint requests. For an upload write checkpoint, call `powersync_control('next_checkpoint_request_id', NULL)` in a transaction, post the returned - id to the service, then store the accepted id with `powersync_control('local_target_op', id)`. -- `local_target_op` is the apply gate for local writes. `next_checkpoint_request_id` only allocates + id to the service, then store the accepted id with `powersync_control('target_checkpoint_request_id', id)`. +- `target_checkpoint_request_id` is the apply gate for local writes. `next_checkpoint_request_id` only allocates ids; it does not update that gate. - To retry a checkpoint request without incrementing the counter, read `powersync_control('current_checkpoint_request_id', NULL)` and repost that id when the SDK's runtime last-applied checkpoint request id is absent or lower. - Resolve explicit checkpoint waiters from `DidCompleteSync.applied_checkpoint_request_id`. SDKs - that drive waiters from status snapshots can also watch - `UpdateSyncStatus.status.internal_last_applied_checkpoint_request_id`. Treat that status field as - runtime-only SDK state, not persisted checkpoint state or app-visible progress. + that drive waiters from the sync status should react to + `UpdateSyncStatus.status.internal_last_applied_checkpoint_request_id` on the status update that + carries it: core clears the field on later status updates without an applied request id and on + reconnects, so it is an event-style signal, not a high-water mark to poll. Treat that status + field as runtime-only SDK state, not persisted checkpoint state or app-visible progress. Most `powersync_control` commands return a JSON-encoded array of instructions for the client. -`next_checkpoint_request_id`, `current_checkpoint_request_id` and `local_target_op` return values +`next_checkpoint_request_id`, `current_checkpoint_request_id` and `target_checkpoint_request_id` return values directly. ```typescript diff --git a/docs/write-checkpoint-requests.md b/docs/write-checkpoint-requests.md index 372c880b..ce10595d 100644 --- a/docs/write-checkpoint-requests.md +++ b/docs/write-checkpoint-requests.md @@ -8,20 +8,21 @@ The state is internal core/SDK bookkeeping. Apps should not read it as user-faci ## State Model -| Key | Purpose | Who updates it | -| --- | --- | --- | -| `local_target_op` | Apply gate for local writes. Downloaded full checkpoints can apply only after the stream has seen this id. `MAX_OP_ID` means "local writes exist, but no concrete checkpoint id is known yet". | Core CRUD triggers set the sentinel; SDK upload code stores the accepted concrete id through `powersync_control('local_target_op', id)`. | -| `last_requested_checkpoint_request_id` | Allocation counter for client-created checkpoint requests. | SDKs seed it after connection reconciliation, then core increments it through `powersync_control('next_checkpoint_request_id', NULL)`. | -| `last_seen_checkpoint_request_id` | Latest full checkpoint `write_checkpoint` observed in the stream since the last local write. | Core updates it when a full checkpoint validates. Local writes clear it. | -| `last_applied_checkpoint_request_id` | Latest full checkpoint `write_checkpoint` applied locally since the last local write. | Core updates it after a full checkpoint applies. Local writes clear it. | +| Key | Purpose | Who updates it | +| -------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `target_checkpoint_request_id` | Apply gate for local writes. Downloaded full checkpoints can apply only after the stream has seen this id. `MAX_OP_ID` means "local writes exist, but no concrete checkpoint id is known yet". | Core CRUD triggers set the sentinel; SDK upload code stores the accepted concrete id through `powersync_control('target_checkpoint_request_id', id)`. | +| `last_requested_checkpoint_request_id` | Allocation counter for client-created checkpoint requests. | SDKs seed it after connection reconciliation, then core increments it through `powersync_control('next_checkpoint_request_id', NULL)`. | +| `last_seen_checkpoint_request_id` | Latest full checkpoint `write_checkpoint` observed in the stream since the last local write. | Core updates it when a full checkpoint validates. Local writes clear it. | +| `last_applied_checkpoint_request_id` | Latest full checkpoint `write_checkpoint` applied locally since the last local write. | Core updates it after a full checkpoint applies. Local writes clear it. | Why four keys? -- `local_target_op` is the gate that protects local writes. +- `target_checkpoint_request_id` is the gate that protects local writes. - `last_requested_checkpoint_request_id` is the counter used to create new requests. - `last_seen_checkpoint_request_id` answers "has the stream reached the gate yet?" -- `last_applied_checkpoint_request_id` is persisted for diagnostics. SDK waiters should use - `DidCompleteSync.applied_checkpoint_request_id` instead. +- `last_applied_checkpoint_request_id` is persisted for consistency with the legacy + `$local.last_applied_op` bookkeeping and for diagnostics. SDK waiters should use + `DidCompleteSync.applied_checkpoint_request_id` or the equivalent sync-status field instead. ## SDK Expectations @@ -36,7 +37,7 @@ SDKs should use `powersync_control` as the public API for this state: `powersync_control('next_checkpoint_request_id', NULL)`. 6. Post that id to the service checkpoint-request endpoint. 7. After the service accepts it, store it as the local write gate with - `powersync_control('local_target_op', id)`. + `powersync_control('target_checkpoint_request_id', id)`. 8. Resolve explicit waiters from `DidCompleteSync.applied_checkpoint_request_id`, not from `ps_kv`. `seed_checkpoint_request_id` stores the id verbatim and does not enforce monotonicity. SDKs own @@ -74,7 +75,7 @@ the current id when the applied id is absent or lower. A local write records CRUD and sets: ```sql -ps_kv['local_target_op'] = MAX_OP_ID +ps_kv['target_checkpoint_request_id'] = MAX_OP_ID ``` It also clears `last_seen_checkpoint_request_id` and `last_applied_checkpoint_request_id`, because @@ -94,14 +95,14 @@ POST /sync/checkpoint-request { } transaction { - previousTarget = powersync_control('local_target_op', NULL) + previousTarget = powersync_control('target_checkpoint_request_id', NULL) if previousTarget == MAX_OP_ID && ps_crud is still empty { - powersync_control('local_target_op', requestId) + powersync_control('target_checkpoint_request_id', requestId) } } ``` -`local_target_op` is intentionally separate from `last_requested_checkpoint_request_id`: allocating +`target_checkpoint_request_id` is intentionally separate from `last_requested_checkpoint_request_id`: allocating a checkpoint request id does not mean it should block or unblock local writes. ## Applying Downloaded Checkpoints @@ -114,13 +115,13 @@ can be applied. Full checkpoints and non-priority-0 partial checkpoints can publish only when: - `ps_crud` is empty, and -- `local_target_op` is absent or less than or equal to `last_seen_checkpoint_request_id`. +- `target_checkpoint_request_id` is absent or less than or equal to `last_seen_checkpoint_request_id`. Priority 0 partial syncs may publish while uploads are outstanding. If a full checkpoint validates but cannot apply because local CRUD is pending, core keeps it as a pending checkpoint. When the SDK later sends `completed_upload`, core retries it unless the -pending checkpoint is older than the current `local_target_op`. +pending checkpoint is older than the current `target_checkpoint_request_id`. After a full checkpoint applies, core emits: @@ -134,8 +135,10 @@ or equal to the requested id. Core also includes the applied request id in `UpdateSyncStatus.status.internal_last_applied_checkpoint_request_id` when a status update follows a checkpoint apply with a `write_checkpoint`. Later status updates without an applied request id, -including disconnect updates, clear the field. SDKs may use this for status-stream waiters, but -should treat it as internal, runtime-only state rather than app-visible progress or persisted +including disconnect updates, clear the field. SDKs don't need to memoize it: it is an event-style +signal, so react to the value when a status update carries it (for example, resolve waiters whose +requested id is less than or equal to the emitted value) rather than polling it as a high-water +mark. Treat it as internal, runtime-only state rather than app-visible progress or persisted checkpoint state. ## Control Commands @@ -162,7 +165,7 @@ checkpoint state. - SDKs should compare this with their runtime last-applied checkpoint request id to decide whether to repost the current id. -`powersync_control('local_target_op', value)` +`powersync_control('target_checkpoint_request_id', value)` - Payload `NULL`: return current target without changing it. - Payload `0`: clear the target. From 2c55f243c6815d06267ca357550b20a735f57ee1 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 9 Jul 2026 10:44:11 +0200 Subject: [PATCH 26/29] share ps_kv values --- crates/core/src/crud_vtab.rs | 8 ++++++-- crates/core/src/sync/storage_adapter.rs | 10 ++++++---- crates/core/src/sync/sync_local.rs | 14 +++++++++----- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/crates/core/src/crud_vtab.rs b/crates/core/src/crud_vtab.rs index f33508cd..80a79ebd 100644 --- a/crates/core/src/crud_vtab.rs +++ b/crates/core/src/crud_vtab.rs @@ -15,6 +15,10 @@ use crate::error::PowerSyncError; use crate::ext::SafeManagedStmt; use crate::schema::TableInfoFlags; use crate::state::DatabaseState; +use crate::sync::storage_adapter::{ + LAST_APPLIED_CHECKPOINT_REQUEST_ID_KEY, LAST_SEEN_CHECKPOINT_REQUEST_ID_KEY, + TARGET_CHECKPOINT_REQUEST_ID_KEY, +}; use crate::utils::MAX_OP_ID; use crate::vtab_util::*; @@ -252,8 +256,8 @@ impl SimpleCrudTransactionMode { // restart. Keeping them around could open the apply gate for a newly allocated target // id that compares below a stale seen value. db.exec_safe(formatcp!( - "INSERT OR REPLACE INTO ps_kv(key, value) VALUES('target_checkpoint_request_id', {MAX_OP_ID}); -DELETE FROM ps_kv WHERE key IN ('last_seen_checkpoint_request_id', 'last_applied_checkpoint_request_id')" + "INSERT OR REPLACE INTO ps_kv(key, value) VALUES('{TARGET_CHECKPOINT_REQUEST_ID_KEY}', {MAX_OP_ID}); +DELETE FROM ps_kv WHERE key IN ('{LAST_SEEN_CHECKPOINT_REQUEST_ID_KEY}', '{LAST_APPLIED_CHECKPOINT_REQUEST_ID_KEY}')" ))?; self.had_writes = true; } diff --git a/crates/core/src/sync/storage_adapter.rs b/crates/core/src/sync/storage_adapter.rs index f0267783..d78a30c8 100644 --- a/crates/core/src/sync/storage_adapter.rs +++ b/crates/core/src/sync/storage_adapter.rs @@ -26,14 +26,16 @@ use super::{ bucket_priority::BucketPriority, interface::BucketRequest, streaming_sync::OwnedCheckpoint, }; -const LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY: &str = "last_requested_checkpoint_request_id"; -const LAST_SEEN_CHECKPOINT_REQUEST_ID_KEY: &str = "last_seen_checkpoint_request_id"; -const LAST_APPLIED_CHECKPOINT_REQUEST_ID_KEY: &str = "last_applied_checkpoint_request_id"; +pub(crate) const LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY: &str = + "last_requested_checkpoint_request_id"; +pub(crate) const LAST_SEEN_CHECKPOINT_REQUEST_ID_KEY: &str = "last_seen_checkpoint_request_id"; +pub(crate) const LAST_APPLIED_CHECKPOINT_REQUEST_ID_KEY: &str = + "last_applied_checkpoint_request_id"; // Tracks the target used to block applying downloaded rows while local writes are outstanding. // When present, this is normally either the max-op sentinel for pending local writes or a concrete // checkpoint request id also stored in LAST_REQUESTED_CHECKPOINT_REQUEST_ID_KEY. -const TARGET_CHECKPOINT_REQUEST_ID_KEY: &str = "target_checkpoint_request_id"; +pub(crate) const TARGET_CHECKPOINT_REQUEST_ID_KEY: &str = "target_checkpoint_request_id"; /// An adapter for storing sync state. /// diff --git a/crates/core/src/sync/sync_local.rs b/crates/core/src/sync/sync_local.rs index 9555d2aa..d8770b91 100644 --- a/crates/core/src/sync/sync_local.rs +++ b/crates/core/src/sync/sync_local.rs @@ -12,8 +12,12 @@ use crate::schema::{ }; use crate::state::DatabaseState; use crate::sync::BucketPriority; +use crate::sync::storage_adapter::{ + LAST_SEEN_CHECKPOINT_REQUEST_ID_KEY, TARGET_CHECKPOINT_REQUEST_ID_KEY, +}; use crate::sync::sync_status::TimestampMicros; use crate::utils::SqlBuffer; +use const_format::formatcp; use powersync_sqlite_nostd::{self as sqlite, Destructor, ManagedStmt}; use powersync_sqlite_nostd::{Connection, ResultCode}; @@ -66,13 +70,13 @@ impl<'a> SyncOperation<'a> { if needs_check { // language=SQLite - let statement = self.db.prepare_v2( + let statement = self.db.prepare_v2(formatcp!( "SELECT 1 FROM ps_kv AS target -LEFT JOIN ps_kv AS seen ON seen.key = 'last_seen_checkpoint_request_id' -WHERE target.key = 'target_checkpoint_request_id' - AND CAST(target.value AS INTEGER) > COALESCE(CAST(seen.value AS INTEGER), 0)", - )?; +LEFT JOIN ps_kv AS seen ON seen.key = '{LAST_SEEN_CHECKPOINT_REQUEST_ID_KEY}' +WHERE target.key = '{TARGET_CHECKPOINT_REQUEST_ID_KEY}' + AND CAST(target.value AS INTEGER) > COALESCE(CAST(seen.value AS INTEGER), 0)" + ))?; if statement.step()? == ResultCode::ROW { return Ok(false); From 035b0bafa1ce675d62d77fbe14ee99e4f01be5a2 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 9 Jul 2026 10:47:34 +0200 Subject: [PATCH 27/29] cleanup docs --- docs/sync.md | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/docs/sync.md b/docs/sync.md index 7baf4ae3..23e894ef 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -52,28 +52,9 @@ The following commands are supported: ## Checkpoint Request Expectations Checkpoint request state exists to protect local writes and to support explicit "wait until synced" -requests. The detailed state model lives in `write-checkpoint-requests.md`; this section summarizes -what SDKs need to do. - -- On every connection, reconcile `EstablishSyncStream.last_checkpoint_request_id` with the service. - Post at least `1` when there is no known id, then call - `powersync_control('seed_checkpoint_request_id', acceptedId)`. - The service returns the maximum of client and service-side state, so this hydrates a client that - lost its local value and recreates service-side state when the service lost its record. -- Wait for seeding to complete before creating checkpoint requests. For an upload write checkpoint, - call `powersync_control('next_checkpoint_request_id', NULL)` in a transaction, post the returned - id to the service, then store the accepted id with `powersync_control('target_checkpoint_request_id', id)`. -- `target_checkpoint_request_id` is the apply gate for local writes. `next_checkpoint_request_id` only allocates - ids; it does not update that gate. -- To retry a checkpoint request without incrementing the counter, read - `powersync_control('current_checkpoint_request_id', NULL)` and repost that id when the SDK's - runtime last-applied checkpoint request id is absent or lower. -- Resolve explicit checkpoint waiters from `DidCompleteSync.applied_checkpoint_request_id`. SDKs - that drive waiters from the sync status should react to - `UpdateSyncStatus.status.internal_last_applied_checkpoint_request_id` on the status update that - carries it: core clears the field on later status updates without an applied request id and on - reconnects, so it is an event-style signal, not a high-water mark to poll. Treat that status - field as runtime-only SDK state, not persisted checkpoint state or app-visible progress. +requests. The state model, the per-connection reconciliation and seeding flow, and how SDKs resolve +checkpoint waiters (through `DidCompleteSync.applied_checkpoint_request_id` or the equivalent +sync-status field) are documented in `write-checkpoint-requests.md`. Most `powersync_control` commands return a JSON-encoded array of instructions for the client. `next_checkpoint_request_id`, `current_checkpoint_request_id` and `target_checkpoint_request_id` return values From d3b39c2c7ae3c973e1f2f56d77953d0afb8f05b2 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 9 Jul 2026 10:58:40 +0200 Subject: [PATCH 28/29] try latest-stable xcode --- .github/actions/xcframework/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/xcframework/action.yml b/.github/actions/xcframework/action.yml index 6ec6afbc..21207846 100644 --- a/.github/actions/xcframework/action.yml +++ b/.github/actions/xcframework/action.yml @@ -24,8 +24,7 @@ runs: - name: Set up XCode uses: maxim-lobanov/setup-xcode@ed7a3b1fda3918c0306d1b724322adc0b8cc0a90 # v1.7.0 with: - # TODO: Update to latest-stable once GH installs iOS 26 simulators - xcode-version: '^16.4.0' + xcode-version: latest-stable - name: List simulators shell: bash From 55937515cd5d77aab964bca27f7c476e0e42f748 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Thu, 9 Jul 2026 14:00:48 +0200 Subject: [PATCH 29/29] seed_checkpoint_request_id should not be a sync event --- crates/core/src/sync/interface.rs | 36 ++++++++----- crates/core/src/sync/streaming_sync.rs | 4 -- dart/test/goldens/simple_iteration.json | 5 -- dart/test/goldens/starting_stream.json | 5 -- dart/test/sync_stream_test.dart | 5 +- dart/test/sync_test.dart | 67 ++++++++++++++++--------- docs/sync.md | 11 ++-- docs/write-checkpoint-requests.md | 2 + 8 files changed, 75 insertions(+), 60 deletions(-) diff --git a/crates/core/src/sync/interface.rs b/crates/core/src/sync/interface.rs index be21d84e..b85d0994 100644 --- a/crates/core/src/sync/interface.rs +++ b/crates/core/src/sync/interface.rs @@ -90,10 +90,6 @@ pub enum SyncEvent<'a> { /// /// In response, we'll stop the current iteration to begin another one with the new token. DidRefreshToken, - /// Seeds the checkpoint request counter from service state. - SeedCheckpointRequestId { - request_id: i64, - }, /// Notifies the sync client that the current CRUD upload (for which the client SDK is /// responsible) has finished. /// @@ -249,6 +245,29 @@ pub fn register(db: *mut sqlite::sqlite3, state: Rc) -> Result<() } }), "stop" => SyncControlRequest::StopSyncStream, + "seed_checkpoint_request_id" => { + let has_sync_iteration = { + let client = state.sync_client.borrow(); + client + .as_ref() + .map(|client| client.has_sync_iteration()) + .unwrap_or(false) + }; + + if !has_sync_iteration { + return Err(PowerSyncError::state_error("No iteration is active")); + } + + let request_id = parse_positive_i64_payload( + *payload, + "checkpoint request id", + "checkpoint request id must be an integer or integer string", + )?; + let adapter = state.storage_adapter(db)?; + adapter.seed_checkpoint_request_id(request_id)?; + ctx.result_int64(request_id); + return Ok(()); + } "next_checkpoint_request_id" => { let has_sync_iteration = { let client = state.sync_client.borrow(); @@ -316,15 +335,6 @@ pub fn register(db: *mut sqlite::sqlite3, state: Rc) -> Result<() }, }), "refreshed_token" => SyncControlRequest::SyncEvent(SyncEvent::DidRefreshToken), - "seed_checkpoint_request_id" => { - SyncControlRequest::SyncEvent(SyncEvent::SeedCheckpointRequestId { - request_id: parse_positive_i64_payload( - *payload, - "checkpoint request id", - "checkpoint request id must be an integer or integer string", - )?, - }) - } "completed_upload" => SyncControlRequest::SyncEvent(SyncEvent::UploadFinished), "update_subscriptions" => { SyncControlRequest::SyncEvent(SyncEvent::DidUpdateSubscriptions { diff --git a/crates/core/src/sync/streaming_sync.rs b/crates/core/src/sync/streaming_sync.rs index 208553a7..381a097b 100644 --- a/crates/core/src/sync/streaming_sync.rs +++ b/crates/core/src/sync/streaming_sync.rs @@ -562,10 +562,6 @@ impl StreamingSyncIteration { .update(|s| s.disconnect(), &mut event.instructions); break false; } - SyncEvent::SeedCheckpointRequestId { request_id } => { - self.adapter.seed_checkpoint_request_id(request_id)?; - continue; - } SyncEvent::TextLine { data } => SyncLineWithSource::from_text(data)?, SyncEvent::BinaryLine { data } => SyncLineWithSource::from_binary(data)?, SyncEvent::UploadFinished => { diff --git a/dart/test/goldens/simple_iteration.json b/dart/test/goldens/simple_iteration.json index 4df0ff79..d26c21bf 100644 --- a/dart/test/goldens/simple_iteration.json +++ b/dart/test/goldens/simple_iteration.json @@ -33,11 +33,6 @@ } ] }, - { - "operation": "seed_checkpoint_request_id", - "data": 1, - "output": [] - }, { "operation": "line_text", "data": { diff --git a/dart/test/goldens/starting_stream.json b/dart/test/goldens/starting_stream.json index 0c26397b..987da69c 100644 --- a/dart/test/goldens/starting_stream.json +++ b/dart/test/goldens/starting_stream.json @@ -38,10 +38,5 @@ } } ] - }, - { - "operation": "seed_checkpoint_request_id", - "data": 1, - "output": [] } ] diff --git a/dart/test/sync_stream_test.dart b/dart/test/sync_stream_test.dart index 0e86f488..b062d945 100644 --- a/dart/test/sync_stream_test.dart +++ b/dart/test/sync_stream_test.dart @@ -68,10 +68,7 @@ void main() { List control(String operation, Object? data) { final result = controlRaw(operation, data); if (operation == 'start' && establishesSyncStream(result)) { - return [ - ...result, - ...controlRaw('seed_checkpoint_request_id', 1), - ]; + controlRaw('seed_checkpoint_request_id', 1); } return result; } diff --git a/dart/test/sync_test.dart b/dart/test/sync_test.dart index 8ee4dfe9..3f3d0eb6 100644 --- a/dart/test/sync_test.dart +++ b/dart/test/sync_test.dart @@ -60,7 +60,12 @@ void _syncTests({ db.execute('commit'); final [row] = result; - return jsonDecode(row.columnAt(0)); + final rawResult = row.columnAt(0); + if (rawResult is String) { + return jsonDecode(rawResult); + } else { + return const []; + } } Object? invokeControlScalar(String operation, Object? data) { @@ -83,6 +88,10 @@ void _syncTests({ return row.columnAt(0); } + int seedCheckpointRequestId(Object? requestId) { + return invokeControlScalar('seed_checkpoint_request_id', requestId) as int; + } + bool establishesSyncStream(List instructions) { return instructions.any((instruction) => instruction is Map && instruction.containsKey('EstablishSyncStream')); @@ -98,10 +107,7 @@ void _syncTests({ } if (operation == 'start' && establishesSyncStream(result)) { - final seedResult = matcher.enabled - ? matcher.invoke('seed_checkpoint_request_id', 1) - : invokeControlRaw('seed_checkpoint_request_id', 1); - return [...result, ...seedResult]; + seedCheckpointRequestId(1); } return result; @@ -246,15 +252,12 @@ void _syncTests({ }); syncTest('app_metadata is passed to EstablishSyncStream request', (_) { - final startInstructions = [ - ...invokeControlRaw( - 'start', - json.encode({ - 'app_metadata': {'key1': 'value1', 'key2': 'value2'} - }), - ), - ...invokeControlRaw('seed_checkpoint_request_id', 1), - ]; + final startInstructions = invokeControlRaw( + 'start', + json.encode({ + 'app_metadata': {'key1': 'value1', 'key2': 'value2'} + }), + ); expect( startInstructions, @@ -470,7 +473,7 @@ void _syncTests({ expect(currentCheckpointRequestId(), isNull); invokeControlRaw('start', null); - invokeControlRaw('seed_checkpoint_request_id', 1); + expect(seedCheckpointRequestId(1), 1); expect(currentCheckpointRequestId(), 1); expect(currentCheckpointRequestId(), 1); @@ -494,7 +497,7 @@ void _syncTests({ syncTest('seeds requested checkpoint request ids from service state', (_) { final startInstructions = invokeControlRaw('start', null); expect(streamLastCheckpointRequestId(startInstructions), isNull); - invokeControlRaw('seed_checkpoint_request_id', 41); + expect(seedCheckpointRequestId(41), 41); expect(nextCheckpointRequestId(), 42); expect(lastRequestedCheckpointRequestId(), 42); @@ -505,7 +508,7 @@ void _syncTests({ contains(containsPair('EstablishSyncStream', anything)), ); expect(streamLastCheckpointRequestId(restartInstructions), 42); - expect(invokeControlRaw('seed_checkpoint_request_id', 100), isEmpty); + expect(seedCheckpointRequestId(100), 100); expect(nextCheckpointRequestId(), 101); expect(lastRequestedCheckpointRequestId(), 101); @@ -513,13 +516,13 @@ void _syncTests({ syncTest('stores seeded checkpoint request ids verbatim', (_) { invokeControlRaw('start', null); - invokeControlRaw('seed_checkpoint_request_id', 41); + expect(seedCheckpointRequestId(41), 41); expect(lastRequestedCheckpointRequestId(), 41); // Core does not enforce monotonicity when seeding. SDKs own reconciliation and seed the // effective state accepted by the service, which may be below the local counter (e.g. after // switching users). - invokeControlRaw('seed_checkpoint_request_id', 5); + expect(seedCheckpointRequestId(5), 5); expect(lastRequestedCheckpointRequestId(), 5); expect(nextCheckpointRequestId(), 6); }); @@ -552,12 +555,23 @@ void _syncTests({ syncTest('accepts text checkpoint request ids when seeding', (_) { invokeControlRaw('start', null); - invokeControlRaw('seed_checkpoint_request_id', '41'); + expect(seedCheckpointRequestId('41'), 41); expect(lastRequestedCheckpointRequestId(), 41); expect(nextCheckpointRequestId(), 42); }); + syncTest('requires active sync iteration before seeding checkpoint ids', (_) { + expect( + () => seedCheckpointRequestId(1), + throwsA(isSqliteException( + 21, + contains('No iteration is active'), + )), + ); + expect(lastRequestedCheckpointRequestId(), isNull); + }); + syncTest('requires checkpoint request state before allocating checkpoint ids', (_) { invokeControlRaw('start', null); @@ -572,7 +586,9 @@ void _syncTests({ expect(probeTargetCheckpointRequestId(), isNull); }); - syncTest('probes and updates target checkpoint request id without sync iteration', (_) { + syncTest( + 'probes and updates target checkpoint request id without sync iteration', + (_) { expect(probeTargetCheckpointRequestId(), isNull); expect(probeTargetCheckpointRequestId(1), isNull); expect(lastRequestedCheckpointRequestId(), isNull); @@ -583,7 +599,9 @@ void _syncTests({ expect(probeTargetCheckpointRequestId(), 2); }); - syncTest('accepts text checkpoint request ids for target checkpoint request id', (_) { + syncTest( + 'accepts text checkpoint request ids for target checkpoint request id', + (_) { expect(probeTargetCheckpointRequestId('1'), isNull); expect(lastRequestedCheckpointRequestId(), isNull); expect(probeTargetCheckpointRequestId(), 1); @@ -599,9 +617,10 @@ void _syncTests({ ); }); - syncTest('target checkpoint request id does not update checkpoint request id', (_) { + syncTest('target checkpoint request id does not update checkpoint request id', + (_) { invokeControlRaw('start', null); - invokeControlRaw('seed_checkpoint_request_id', 10); + expect(seedCheckpointRequestId(10), 10); expect(lastRequestedCheckpointRequestId(), 10); expect(probeTargetCheckpointRequestId(7), isNull); diff --git a/docs/sync.md b/docs/sync.md index 23e894ef..4a51f125 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -45,9 +45,10 @@ The following commands are supported: clears the target checkpoint request id and returns the previously-observed value as an integer result, or SQL `NULL` if there was no target. This command can run outside of a sync iteration and does not affect it. -13. `seed_checkpoint_request_id`: Payload is a positive integer or integer string. After receiving - `EstablishSyncStream`, SDKs should reconcile the local hint with service-side - checkpoint-request state, then seed core with the accepted positive id. +13. `seed_checkpoint_request_id`: Payload is a positive integer or integer string. During an active + sync iteration, after receiving `EstablishSyncStream`, SDKs should reconcile the local hint with + service-side checkpoint-request state, then seed core with the accepted positive id. Returns the + seeded id as an integer result. ## Checkpoint Request Expectations @@ -57,8 +58,8 @@ checkpoint waiters (through `DidCompleteSync.applied_checkpoint_request_id` or t sync-status field) are documented in `write-checkpoint-requests.md`. Most `powersync_control` commands return a JSON-encoded array of instructions for the client. -`next_checkpoint_request_id`, `current_checkpoint_request_id` and `target_checkpoint_request_id` return values -directly. +`seed_checkpoint_request_id`, `next_checkpoint_request_id`, `current_checkpoint_request_id` and +`target_checkpoint_request_id` return values directly. ```typescript type Instruction = { LogLine: LogLine } diff --git a/docs/write-checkpoint-requests.md b/docs/write-checkpoint-requests.md index ce10595d..9106aa99 100644 --- a/docs/write-checkpoint-requests.md +++ b/docs/write-checkpoint-requests.md @@ -146,7 +146,9 @@ checkpoint state. `powersync_control('seed_checkpoint_request_id', id)` - Payload: positive integer or integer string. +- Returns: seeded checkpoint request id as a SQLite integer. - Stores the reconciled checkpoint-request counter seed. +- Requires an active sync iteration. - Must be called after connection reconciliation and before allocating new ids. `powersync_control('next_checkpoint_request_id', NULL)`