From 94919185a9d357bd8f7e4bafaa729fc720feb487 Mon Sep 17 00:00:00 2001 From: George Lambert Date: Tue, 15 Sep 2026 23:43:01 -0400 Subject: [PATCH] S27: HPKE-Base content wrap + public-key directory; reject xor on send Routing/error fields stay clear. Content is X25519-HKDF-SHA256-ChaCha20. --- go/internal/leaf/leaf.go | 4 +- go/internal/leaf/leaf_test.go | 23 ++++- python/secure_messaging/__init__.py | 6 ++ python/secure_messaging/content.py | 63 ++++++++++++++ python/secure_messaging/hpke.py | 121 ++++++++++++++++++++++++++ python/secure_messaging/pubkey_dir.py | 93 ++++++++++++++++++++ python/secure_messaging/router.py | 11 ++- tests/test_sm.py | 56 ++++++++++-- 8 files changed, 364 insertions(+), 13 deletions(-) create mode 100644 python/secure_messaging/content.py create mode 100644 python/secure_messaging/hpke.py create mode 100644 python/secure_messaging/pubkey_dir.py diff --git a/go/internal/leaf/leaf.go b/go/internal/leaf/leaf.go index 51be009..3069592 100644 --- a/go/internal/leaf/leaf.go +++ b/go/internal/leaf/leaf.go @@ -7,6 +7,7 @@ import ( "log" "net/http" "net/url" + "os" "strings" "time" @@ -113,7 +114,8 @@ func (n *Node) handleSend(msg *nats.Msg) { n.fail(msg, "SM-MISSING-TO", "mailbox", lid) return } - if alg != "" && alg != "npe" && alg != "lab-xor" && alg != "plain-lab" { + allowLab := os.Getenv("SM_ALLOW_LAB") == "1" + if alg != "npe" && !(allowLab && (alg == "lab-xor" || alg == "plain-lab")) { n.fail(msg, "SM-BAD-ALG", "mailbox", lid) return } diff --git a/go/internal/leaf/leaf_test.go b/go/internal/leaf/leaf_test.go index 5a57b70..085ff26 100644 --- a/go/internal/leaf/leaf_test.go +++ b/go/internal/leaf/leaf_test.go @@ -56,7 +56,7 @@ func TestSMSubjectsAck(t *testing.T) { t.Fatal(err) } defer n.Shutdown() - good := []byte(`{"to":"npe.inbox.x","alg":"lab-xor","ct":"abcd","from_lookup_id":"lid-good"}`) + good := []byte(`{"to":"npe.inbox.x","alg":"npe","ct":"abcd","enc":"00","from_lookup_id":"lid-good"}`) msg, err := n.nc.Request("verae.sm.send", good, time.Second) if err != nil { t.Fatal(err) @@ -89,7 +89,7 @@ func TestSMSendRejectsMissingTo(t *testing.T) { t.Fatal(err) } defer n.Shutdown() - msg, err := n.nc.Request("verae.sm.send", []byte(`{"alg":"lab-xor","ct":"ab"}`), time.Second) + msg, err := n.nc.Request("verae.sm.send", []byte(`{"alg":"npe","ct":"ab"}`), time.Second) if err != nil { t.Fatal(err) } @@ -100,13 +100,30 @@ func TestSMSendRejectsMissingTo(t *testing.T) { } } +func TestSMSendRejectsXORContent(t *testing.T) { + n, err := Start("") + if err != nil { + t.Fatal(err) + } + defer n.Shutdown() + msg, err := n.nc.Request("verae.sm.send", []byte(`{"to":"npe.inbox.x","alg":"lab-xor","ct":"ab"}`), time.Second) + if err != nil { + t.Fatal(err) + } + var ack map[string]any + _ = json.Unmarshal(msg.Data, &ack) + if ack["error_code"] != "SM-BAD-ALG" { + t.Fatalf("%s", msg.Data) + } +} + func TestSMSendRejectsEmptyCiphertext(t *testing.T) { n, err := Start("") if err != nil { t.Fatal(err) } defer n.Shutdown() - msg, err := n.nc.Request("verae.sm.send", []byte(`{"to":"npe.inbox.x","alg":"lab-xor","ct":""}`), time.Second) + msg, err := n.nc.Request("verae.sm.send", []byte(`{"to":"npe.inbox.x","alg":"npe","ct":""}`), time.Second) if err != nil { t.Fatal(err) } diff --git a/python/secure_messaging/__init__.py b/python/secure_messaging/__init__.py index 67006f6..39446af 100644 --- a/python/secure_messaging/__init__.py +++ b/python/secure_messaging/__init__.py @@ -5,6 +5,8 @@ from .error_bundle import NetworkErrorBundle # system-key + sender-only from .admin_history import AdminHistory # DataCube-shaped append-only config log from .npe_adapter import NpeRequired # fail-closed NPE from .router import RouteResult, handle_send # success/failure send policy +from .pubkey_dir import PubKeyDir # public-key directory +from .content import wrap_content, unwrap_content, routing_view # HPKE content __all__ = [ "SignedConfig", @@ -15,4 +17,8 @@ __all__ = [ "NpeRequired", "RouteResult", "handle_send", + "PubKeyDir", + "wrap_content", + "unwrap_content", + "routing_view", ] diff --git a/python/secure_messaging/content.py b/python/secure_messaging/content.py new file mode 100644 index 0000000..992393e --- /dev/null +++ b/python/secure_messaging/content.py @@ -0,0 +1,63 @@ +"""NATS content wrapping: HPKE body, routing/error in the clear. + +Clear (broker may see): subject, ``to``, ``from_lookup_id``, ``alg``, +error_code, dest_class, lookup_id. +Never clear: inner content JSON / PHI / object bytes. +""" + +from __future__ import annotations # annotations + +import json # inner body +import os # urandom +from typing import Any, Dict, Mapping, Optional # types + +from .envelope import lookup_id # opaque sender handle +from .hpke import open_ct, seal # HPKE-Base +from .pubkey_dir import PubKeyDir # directory + + +def wrap_content( + *, + to_handle: str, + sender: str, + body: Mapping[str, Any], + directory: PubKeyDir, +) -> Dict[str, Any]: + """Encrypt body to the recipient's directory public key.""" + rec = directory.get(to_handle) + if not rec: # unknown endpoint + raise KeyError("directory has no public key for " + to_handle) + pk = bytes.fromhex(rec["enc_pk"]) + inner = json.dumps(body, sort_keys=True, separators=(",", ":")).encode() + aad = ("to=" + to_handle).encode() # bind ciphertext to routing dest + enc, ct = seal(pk, inner, aad=aad) + nonce = os.urandom(16) + return { + "v": 1, + "alg": "npe", + "mode": "passthrough", + "to": rec.get("inbox") or to_handle, # routing in the clear + "to_handle": to_handle, + "from_lookup_id": lookup_id(sender, nonce), + "nonce": nonce.hex(), + "enc": enc.hex(), # HPKE encapsulated key (not content) + "ct": ct.hex(), # content ciphertext + } + + +def unwrap_content(env: Mapping[str, Any], *, handle: str, directory: PubKeyDir) -> Dict[str, Any]: + """Open content as the named endpoint. Broker cannot do this.""" + sk = directory.load_sk(handle) + to_handle = str(env.get("to_handle") or handle) + aad = ("to=" + to_handle).encode() + pt = open_ct(sk, bytes.fromhex(env["enc"]), bytes.fromhex(env["ct"]), aad=aad) + return json.loads(pt.decode()) + + +def routing_view(env: Mapping[str, Any]) -> Dict[str, Any]: + """What an untrusted router is allowed to look at.""" + return { + k: env[k] + for k in ("v", "alg", "mode", "to", "to_handle", "from_lookup_id", "nonce") + if k in env + } diff --git a/python/secure_messaging/hpke.py b/python/secure_messaging/hpke.py new file mode 100644 index 0000000..67a1f64 --- /dev/null +++ b/python/secure_messaging/hpke.py @@ -0,0 +1,121 @@ +"""HPKE-Base X25519-HKDF-SHA256-ChaCha20-Poly1305 (RFC 9180). + +Production E2E for NATS *content*. Routing fields stay in the clear. +The broker never gets the recipient private key. +""" + +from __future__ import annotations # annotations + +import os # eph nonce +from typing import Tuple # enc, ct + +from cryptography.hazmat.primitives import hashes, hmac # HKDF pieces +from cryptography.hazmat.primitives.asymmetric.x25519 import ( # KEM + X25519PrivateKey, + X25519PublicKey, +) +from cryptography.hazmat.primitives.ciphers.aead import ChaCha20Poly1305 # AEAD +from cryptography.hazmat.primitives.serialization import ( # raw keys + Encoding, + NoEncryption, + PrivateFormat, + PublicFormat, +) + +KEM_ID = 0x0020 # DHKEM(X25519, HKDF-SHA256) +KDF_ID = 0x0001 # HKDF-SHA256 +AEAD_ID = 0x0003 # ChaCha20Poly1305 +MODE_BASE = 0x00 # HPKE Base +SUITE_ID = b"HPKE" + KEM_ID.to_bytes(2, "big") + KDF_ID.to_bytes(2, "big") + AEAD_ID.to_bytes(2, "big") +N_ENC = 32 # X25519 public +N_PK = 32 +N_SK = 32 +N_NOISE = 12 # ChaCha nonce length in HPKE +N_K = 32 # key +AAD_DEFAULT = b"verae.npe.v1" # bind content to this system + + +def _hmac(key: bytes, data: bytes) -> bytes: + h = hmac.HMAC(key, hashes.SHA256()) # RFC HMAC + h.update(data) + return h.finalize() + + +def _extract(salt: bytes, ikm: bytes) -> bytes: + if not salt: # HMAC key + salt = b"\x00" * 32 + return _hmac(salt, ikm) + + +def _expand(prk: bytes, info: bytes, length: int) -> bytes: + out = b"" # T(0)|T(1)... + t = b"" + i = 1 + while len(out) < length: + t = _hmac(prk, t + info + bytes([i])) + out += t + i += 1 + return out[:length] + + +def _labeled_extract(salt: bytes, label: bytes, ikm: bytes) -> bytes: + labeled = b"HPKE-v1" + SUITE_ID + label + ikm # RFC 9180 + return _extract(salt, labeled) + + +def _labeled_expand(prk: bytes, label: bytes, info: bytes, length: int) -> bytes: + labeled = length.to_bytes(2, "big") + b"HPKE-v1" + SUITE_ID + label + info + return _expand(prk, labeled, length) + + +def generate_keypair() -> Tuple[bytes, bytes]: + """Return (raw_sk, raw_pk) 32+32 bytes.""" + sk = X25519PrivateKey.generate() # CSPRNG + pk = sk.public_key() + return ( + sk.private_bytes(Encoding.Raw, PrivateFormat.Raw, NoEncryption()), + pk.public_bytes(Encoding.Raw, PublicFormat.Raw), + ) + + +def _dh(sk: bytes, pk: bytes) -> bytes: + priv = X25519PrivateKey.from_private_bytes(sk) + pub = X25519PublicKey.from_public_bytes(pk) + return priv.exchange(pub) + + +def _extract_and_expand(dh: bytes, kem_context: bytes) -> bytes: + eae_prk = _labeled_extract(b"", b"eae_prk", dh) + return _labeled_expand(eae_prk, b"shared_secret", kem_context, 32) + + +def _key_schedule(shared: bytes, info: bytes) -> Tuple[bytes, bytes]: + psk_id_hash = _labeled_extract(b"", b"psk_id_hash", b"") + info_hash = _labeled_extract(b"", b"info_hash", info) + ks_ctx = bytes([MODE_BASE]) + psk_id_hash + info_hash + secret = _labeled_extract(shared, b"secret", b"") + key = _labeled_expand(secret, b"key", ks_ctx, N_K) + base_nonce = _labeled_expand(secret, b"base_nonce", ks_ctx, N_NOISE) + return key, base_nonce + + +def seal(recipient_pk: bytes, plaintext: bytes, aad: bytes = AAD_DEFAULT) -> Tuple[bytes, bytes]: + """HPKE-Base seal. Returns (encapped_key, ciphertext).""" + eph_sk, eph_pk = generate_keypair() # ephemeral + dh = _dh(eph_sk, recipient_pk) + kem_context = eph_pk + recipient_pk + shared = _extract_and_expand(dh, kem_context) + key, nonce = _key_schedule(shared, b"") + ct = ChaCha20Poly1305(key).encrypt(nonce, plaintext, aad) + return eph_pk, ct + + +def open_ct(recipient_sk: bytes, enc: bytes, ciphertext: bytes, aad: bytes = AAD_DEFAULT) -> bytes: + """HPKE-Base open. Raises on auth failure.""" + rec_sk = X25519PrivateKey.from_private_bytes(recipient_sk) + rec_pk = rec_sk.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw) + dh = _dh(recipient_sk, enc) + kem_context = enc + rec_pk + shared = _extract_and_expand(dh, kem_context) + key, nonce = _key_schedule(shared, b"") + return ChaCha20Poly1305(key).decrypt(nonce, ciphertext, aad) diff --git a/python/secure_messaging/pubkey_dir.py b/python/secure_messaging/pubkey_dir.py new file mode 100644 index 0000000..0ea24aa --- /dev/null +++ b/python/secure_messaging/pubkey_dir.py @@ -0,0 +1,93 @@ +"""Public-key directory for NPE/HPKE endpoint encryption. + +Only *public* material is listed. Private keys stay in private/ (0600). +Every service that needs E2E looks up handles here (HTTP or files). +""" + +from __future__ import annotations # annotations + +import json # records +import os # chmod +import time # updated_ts +from pathlib import Path # dir root +from typing import Any, Dict, List, Optional # types + +from .hpke import generate_keypair # X25519 + + +class PubKeyDir: + """File-backed directory: public/.json + private/.sk.""" + + def __init__(self, root: Path): + self.root = Path(root) # /opt/pfc/etc/npe-dir + self.public = self.root / "public" + self.private = self.root / "private" + self.public.mkdir(parents=True, exist_ok=True) + self.private.mkdir(parents=True, exist_ok=True) + + def create_endpoint(self, handle: str) -> Dict[str, Any]: + """Generate X25519 keypair; write public record + private sk.""" + handle = _safe(handle) + sk, pk = generate_keypair() + rec = { + "handle": handle, + "kem": "x25519", + "kdf": "hkdf-sha256", + "aead": "chacha20poly1305", + "suite": "HPKE-Base X25519-HKDF-SHA256-ChaCha20-Poly1305", + "enc_pk": pk.hex(), + "inbox": "npe.inbox." + pk[:16].hex(), + "updated_ts": time.time(), + } + (self.public / (handle + ".json")).write_text(json.dumps(rec, indent=2) + "\n") + skp = self.private / (handle + ".sk") + skp.write_bytes(sk) + os.chmod(skp, 0o600) + return rec + + def get(self, handle: str) -> Optional[Dict[str, Any]]: + p = self.public / (_safe(handle) + ".json") + if not p.exists(): + return None + return json.loads(p.read_text()) + + def list(self) -> List[Dict[str, Any]]: + rows = [] + for p in sorted(self.public.glob("*.json")): + rows.append(json.loads(p.read_text())) + return rows + + def put_public(self, rec: Dict[str, Any]) -> Dict[str, Any]: + """Publish someone else's public key (no private).""" + handle = _safe(str(rec.get("handle") or "")) + if not handle or not rec.get("enc_pk"): + raise ValueError("handle and enc_pk required") + rec = dict(rec) + rec["handle"] = handle + rec["updated_ts"] = time.time() + (self.public / (handle + ".json")).write_text(json.dumps(rec, indent=2) + "\n") + return rec + + def load_sk(self, handle: str) -> bytes: + p = self.private / (_safe(handle) + ".sk") + if not p.exists(): + raise FileNotFoundError("no private key for " + handle) + return p.read_bytes() + + def available(self) -> Dict[str, Any]: + """What every E2E service can see: handles + public keys only.""" + recs = self.list() + return { + "ok": True, + "directory": str(self.root), + "suite": "HPKE-Base X25519-HKDF-SHA256-ChaCha20-Poly1305", + "handles": [r["handle"] for r in recs], + "keys": recs, + } + + +def _safe(handle: str) -> str: + h = "".join(c for c in handle if c.isalnum() or c in "-_") + if not h: + raise ValueError("empty handle") + return h diff --git a/python/secure_messaging/router.py b/python/secure_messaging/router.py index 37db339..66cd7ed 100644 --- a/python/secure_messaging/router.py +++ b/python/secure_messaging/router.py @@ -13,7 +13,8 @@ from typing import Any, Dict, List, Optional # types from .error_bundle import NetworkErrorBundle, build_bundle, log_summary # failure path -ALLOWED_ALG = {"npe", "lab-xor", "plain-lab"} # signed-config crypto.mode set +ALLOWED_ALG = {"npe"} # production E2E; lab-xor/plain-lab rejected for content +LAB_ALG = {"lab-xor", "plain-lab"} # only if SM_ALLOW_LAB=1 class RouteResult: @@ -65,7 +66,13 @@ def handle_send( return _fail("SM-PLAINTEXT-BODY", "mailbox", lid, sender_pub, system_pub, "body field forbidden") if not to: # catalog reject: missing to return _fail("SM-MISSING-TO", "mailbox", lid, sender_pub, system_pub, "missing to") - if alg and alg not in ALLOWED_ALG: # alg not in signed config + import os # SM_ALLOW_LAB + + allow_lab = os.environ.get("SM_ALLOW_LAB") == "1" + allowed = set(ALLOWED_ALG) + if allow_lab: + allowed |= LAB_ALG + if alg and alg not in allowed: # xor/plain are not production content return _fail("SM-BAD-ALG", "mailbox", lid, sender_pub, system_pub, "alg not allowed") if alg != "plain-lab" and not ct: # empty ciphertext when not plain-lab return _fail("SM-EMPTY-CT", "mailbox", lid, sender_pub, system_pub, "empty ciphertext") diff --git a/tests/test_sm.py b/tests/test_sm.py index edb8a88..25fb4f7 100644 --- a/tests/test_sm.py +++ b/tests/test_sm.py @@ -93,23 +93,65 @@ class HistoryTests(unittest.TestCase): h.append_change(actor="", prev_text="", new_text="x") +class HpkeDirTests(unittest.TestCase): + def test_roundtrip_and_router_cannot_see_body(self): + from secure_messaging.content import routing_view, unwrap_content, wrap_content + from secure_messaging.pubkey_dir import PubKeyDir + + with tempfile.TemporaryDirectory() as td: + d = PubKeyDir(Path(td)) + d.create_endpoint("alice") + d.create_endpoint("pfc-repl") + avail = d.available() + self.assertIn("pfc-repl", avail["handles"]) + self.assertTrue(all("sk" not in json.dumps(k) for k in avail["keys"])) + env = wrap_content( + to_handle="pfc-repl", + sender="alice", + body={"bytes_hex": "ab", "note": "secret-phi"}, + directory=d, + ) + self.assertEqual(env["alg"], "npe") + rv = routing_view(env) + self.assertNotIn("ct", rv) + blob = json.dumps(env) + self.assertNotIn("secret-phi", blob) + self.assertNotIn("bytes_hex", routing_view(env)) + pt = unwrap_content(env, handle="pfc-repl", directory=d) + self.assertEqual(pt["note"], "secret-phi") + with self.assertRaises(Exception): + unwrap_content(env, handle="alice", directory=d) + + class RouterTests(unittest.TestCase): def test_good_passthrough(self): - from secure_messaging.envelope import seal from secure_messaging.router import handle_send - env = seal(to="npe.inbox.abc", sender="alice", body={"note": "hello"}, mode="lab-xor", lab_key=b"lab") - r = handle_send(json.dumps(env.wire()).encode()) + env = { + "to": "npe.inbox.abc", + "to_handle": "pfc-repl", + "alg": "npe", + "ct": "abcd", + "enc": "00", + "from_lookup_id": "lid-npe", + } + r = handle_send(json.dumps(env).encode()) self.assertTrue(r.accepted) - self.assertEqual(r.ack()["lookup_id"], env.from_lookup_id) + self.assertEqual(r.ack()["lookup_id"], "lid-npe") self.assertNotIn("ct", r.ack()) self.assertEqual(r.events, []) + def test_xor_rejected_for_content(self): + from secure_messaging.router import handle_send + + r = handle_send(b'{"to":"npe.inbox.x","alg":"lab-xor","ct":"ab"}') + self.assertEqual(r.error_code, "SM-BAD-ALG") + def test_missing_to_failure_path(self): from secure_messaging.router import handle_send from secure_messaging.error_bundle import open_sender, open_system - r = handle_send(b'{"alg":"lab-xor","ct":"ab","from_lookup_id":"lid-x"}') + r = handle_send(b'{"alg":"npe","ct":"ab","from_lookup_id":"lid-x"}') self.assertFalse(r.accepted) self.assertEqual(r.error_code, "SM-MISSING-TO") subjects = [e["subject"] for e in r.events] @@ -125,9 +167,9 @@ class RouterTests(unittest.TestCase): def test_empty_ct_and_plaintext_body(self): from secure_messaging.router import handle_send - r = handle_send(b'{"to":"npe.inbox.x","alg":"lab-xor","ct":""}') + r = handle_send(b'{"to":"npe.inbox.x","alg":"npe","ct":""}') self.assertEqual(r.error_code, "SM-EMPTY-CT") - r2 = handle_send(b'{"to":"npe.inbox.x","alg":"lab-xor","ct":"ab","body":{"secret":1}}') + r2 = handle_send(b'{"to":"npe.inbox.x","alg":"npe","ct":"ab","body":{"secret":1}}') self.assertEqual(r2.error_code, "SM-PLAINTEXT-BODY")