#MoonCDC

#Installation and usage

///|
import {
  "pangbit/mooncdc",
}

///|
test "parse a PostgreSQL WAL position" {
  let position = @mooncdc.Lsn::parse("16/B374D848")
  assert_eq(position.to_string(), "16/B374D848")
  assert_true(position > @mooncdc.Lsn::parse("0/FFFFFFFF"))
}

///|
test "pipeline health before startup and during shutdown" {
  let control = @mooncdc.ApplyControl()
  assert_true(control.health().live)
  assert_false(control.health().ready)
  assert_eq(control.activities().length(), 0)
  control.stop()
  assert_eq(control.health().status, "stopping")
}

let config = @mooncdc.ConnectionConfig::new(
  host="127.0.0.1", port=55418, user="cdc",
  password=password_from_environment, database="postgres",
  security=@mooncdc.LocalPlaintext,
)
@mooncdc.subscribe(
  config, slot="my_slot", publication="my_publication",
  start=@mooncdc.Resume, checkpoint_path="./state/checkpoint",
  sub => {
    for ;; {
      let tx = sub.next()
      persist_transaction_idempotently(tx) // 业务内容与 tx.id 一起持久化
      sub.ack(tx)                          // 之后才允许推进源端反馈
    }
  },
)

#Guarantees and boundaries

• Only committed transactions are delivered, preserving event order, table/type information, transaction IDs and WAL positions.
• Null, Text and UnchangedToast represent SQL NULL, text and unchanged external fields. subscribe(..., binary=true) requests binary transfer; Binary(Bytes) retains original bytes for caller decoding with column OIDs. Text remains the default, and binary mode can still contain fallback text fields.
• Standalone subscriptions may opt into protocol=V2Streaming, buffering uncommitted segments until commit. See API for resource and rollback contracts. The pipeline continues to use protocol v1.
• ack advances only across a contiguous completed prefix. The checkpoint file, atomic replacement and parent directory must be synced before feedback changes. Received position is not business durability.
• tx.id identifies a transaction; tx.event_id(index) identifies a data change. Repeated metadata does not change data-event IDs.
• messages=true supports transaction-bound logical messages and a separate callback for nontransactional messages. Payload bytes remain unchanged; schema_snapshot interprets Supabase ETL DDL payloads. Pipeline schema history can be persisted with run_pipeline(..., schema_store=store); see API.
• Custom Destination implementations integrate through copy_snapshot_to and sub.apply_to. Accepted never authorizes acknowledgement. A durable barrier is required for checkpoints/feedback, including empty flushes and normal drain.
• Background heartbeats continue. Frames, transactions, metadata and outstanding queues have limits; exceeding them closes the connection without acknowledging.
• SCRAM-SHA-256 and verified remote TLS are supported. Plaintext accepts only numeric loopback addresses. Initial passwords are printable ASCII; non-ASCII passwords requiring SASLprep are explicitly rejected.
• run_pipeline maintains durable per-table states, concurrent independent snapshots, failed-copy restart and catch-up. Completed tables retain progress; full queues apply backpressure. Newly published tables are discovered and copied upon their first WAL event.
• Value.decode_text(oid) returns typed Cells, supporting exact numbers, date/time, JSON, UUID, bytea and one-dimensional arrays. SQL NULL, JSON null and unchanged TOAST remain distinct. Unknown scalars retain text.
• StateStore supports file and PostgreSQL backends for pipeline/subscription checkpoints and destination metadata. Table-copy errors persist and support timed or manual retries. See API.
• ClickHouseDestination supports ReplacingMergeTree current-state views and MergeTree event histories, with snapshot/CDC writes, column DDL and durable recovery. See supported scope.
• This module supports PostgreSQL → ClickHouse only. It excludes BigQuery, Snowflake and Iceberg. DuckLake is a roadmap item, not an implemented feature.
• Automatic primary election and two-phase transactions are not provided. Schema support is destination-specific. Cross-system exactly-once delivery is not promised.

#Local validation

moon update
docker compose -p mooncdc-test -f integration/compose.yaml up -d --wait
moon run tools/setup-db.mbtx pg18
moon run tools/setup-db.mbtx pg17
moon check --target native --deny-warn
moon test --target native
moon run tools/setup-tls.mbtx pg18
moon run tools/setup-tls.mbtx pg17
MOONCDC_TEST_PORT=55418 MOONCDC_TEST_CA=.test-artifacts/tls-pg18/server.crt moon run tools/live-test.mbtx
MOONCDC_TEST_PORT=55417 MOONCDC_TEST_CA=.test-artifacts/tls-pg17/server.crt moon run tools/live-test.mbtx
moon run examples/index
moon run examples/cache
moon run tools/recovery-test.mbtx
moon run tools/checkpoint-crash-test.mbtx

#Documentation and maintenance

• Examples · API · Testing
• Operations and WAL monitoring · Releasing
• Development · Contributing · Changelog
• Dependencies and protocol provenance · Security