S27: HPKE-Base content wrap + public-key directory; reject xor on send

Routing/error fields stay clear. Content is X25519-HKDF-SHA256-ChaCha20.

go/internal/leaf/leaf.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"net/http"
 	"net/url"
+	"os"
 	"strings"
 	"time"
 
@@ -113,7 +114,8 @@ func (n *Node) handleSend(msg *nats.Msg) {
 		n.fail(msg, "SM-MISSING-TO", "mailbox", lid)
 		return
 	}
-	if alg != "" && alg != "npe" && alg != "lab-xor" && alg != "plain-lab" {
+	allowLab := os.Getenv("SM_ALLOW_LAB") == "1"
+	if alg != "npe" && !(allowLab && (alg == "lab-xor" || alg == "plain-lab")) {
 		n.fail(msg, "SM-BAD-ALG", "mailbox", lid)
 		return
 	}

go/internal/leaf/leaf_test.go

@@ -56,7 +56,7 @@ func TestSMSubjectsAck(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer n.Shutdown()
-	good := []byte(`{"to":"npe.inbox.x","alg":"lab-xor","ct":"abcd","from_lookup_id":"lid-good"}`)
+	good := []byte(`{"to":"npe.inbox.x","alg":"npe","ct":"abcd","enc":"00","from_lookup_id":"lid-good"}`)
 	msg, err := n.nc.Request("verae.sm.send", good, time.Second)
 	if err != nil {
 		t.Fatal(err)
@@ -89,7 +89,7 @@ func TestSMSendRejectsMissingTo(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer n.Shutdown()
-	msg, err := n.nc.Request("verae.sm.send", []byte(`{"alg":"lab-xor","ct":"ab"}`), time.Second)
+	msg, err := n.nc.Request("verae.sm.send", []byte(`{"alg":"npe","ct":"ab"}`), time.Second)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -100,13 +100,30 @@ func TestSMSendRejectsMissingTo(t *testing.T) {
 	}
 }
 
+func TestSMSendRejectsXORContent(t *testing.T) {
+	n, err := Start("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer n.Shutdown()
+	msg, err := n.nc.Request("verae.sm.send", []byte(`{"to":"npe.inbox.x","alg":"lab-xor","ct":"ab"}`), time.Second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var ack map[string]any
+	_ = json.Unmarshal(msg.Data, &ack)
+	if ack["error_code"] != "SM-BAD-ALG" {
+		t.Fatalf("%s", msg.Data)
+	}
+}
+
 func TestSMSendRejectsEmptyCiphertext(t *testing.T) {
 	n, err := Start("")
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer n.Shutdown()
-	msg, err := n.nc.Request("verae.sm.send", []byte(`{"to":"npe.inbox.x","alg":"lab-xor","ct":""}`), time.Second)
+	msg, err := n.nc.Request("verae.sm.send", []byte(`{"to":"npe.inbox.x","alg":"npe","ct":""}`), time.Second)
 	if err != nil {
 		t.Fatal(err)
 	}

python/secure_messaging/__init__.py

@@ -5,6 +5,8 @@ from .error_bundle import NetworkErrorBundle  # system-key + sender-only
 from .admin_history import AdminHistory  # DataCube-shaped append-only config log
 from .npe_adapter import NpeRequired  # fail-closed NPE
 from .router import RouteResult, handle_send  # success/failure send policy
+from .pubkey_dir import PubKeyDir  # public-key directory
+from .content import wrap_content, unwrap_content, routing_view  # HPKE content
 
 __all__ = [
     "SignedConfig",
@@ -15,4 +17,8 @@ __all__ = [
     "NpeRequired",
     "RouteResult",
     "handle_send",
+    "PubKeyDir",
+    "wrap_content",
+    "unwrap_content",
+    "routing_view",
 ]

python/secure_messaging/content.py

@@ -0,0 +1,63 @@
+"""NATS content wrapping: HPKE body, routing/error in the clear.
+
+Clear (broker may see): subject, ``to``, ``from_lookup_id``, ``alg``,
+error_code, dest_class, lookup_id.
+Never clear: inner content JSON / PHI / object bytes.
+"""
+
+from __future__ import annotations  # annotations
+
+import json  # inner body
+import os  # urandom
+from typing import Any, Dict, Mapping, Optional  # types
+
+from .envelope import lookup_id  # opaque sender handle
+from .hpke import open_ct, seal  # HPKE-Base
+from .pubkey_dir import PubKeyDir  # directory
+
+
+def wrap_content(
+    *,
+    to_handle: str,
+    sender: str,
+    body: Mapping[str, Any],
+    directory: PubKeyDir,
+) -> Dict[str, Any]:
+    """Encrypt body to the recipient's directory public key."""
+    rec = directory.get(to_handle)
+    if not rec:  # unknown endpoint
+        raise KeyError("directory has no public key for " + to_handle)
+    pk = bytes.fromhex(rec["enc_pk"])
+    inner = json.dumps(body, sort_keys=True, separators=(",", ":")).encode()
+    aad = ("to=" + to_handle).encode()  # bind ciphertext to routing dest
+    enc, ct = seal(pk, inner, aad=aad)
+    nonce = os.urandom(16)
+    return {
+        "v": 1,
+        "alg": "npe",
+        "mode": "passthrough",
+        "to": rec.get("inbox") or to_handle,  # routing in the clear
+        "to_handle": to_handle,
+        "from_lookup_id": lookup_id(sender, nonce),
+        "nonce": nonce.hex(),
+        "enc": enc.hex(),  # HPKE encapsulated key (not content)
+        "ct": ct.hex(),  # content ciphertext
+    }
+
+
+def unwrap_content(env: Mapping[str, Any], *, handle: str, directory: PubKeyDir) -> Dict[str, Any]:
+    """Open content as the named endpoint. Broker cannot do this."""
+    sk = directory.load_sk(handle)
+    to_handle = str(env.get("to_handle") or handle)
+    aad = ("to=" + to_handle).encode()
+    pt = open_ct(sk, bytes.fromhex(env["enc"]), bytes.fromhex(env["ct"]), aad=aad)
+    return json.loads(pt.decode())
+
+
+def routing_view(env: Mapping[str, Any]) -> Dict[str, Any]:
+    """What an untrusted router is allowed to look at."""
+    return {
+        k: env[k]
+        for k in ("v", "alg", "mode", "to", "to_handle", "from_lookup_id", "nonce")
+        if k in env
+    }

python/secure_messaging/hpke.py

@@ -0,0 +1,121 @@
+"""HPKE-Base X25519-HKDF-SHA256-ChaCha20-Poly1305 (RFC 9180).
+
+Production E2E for NATS *content*. Routing fields stay in the clear.
+The broker never gets the recipient private key.
+"""
+
+from __future__ import annotations  # annotations
+
+import os  # eph nonce
+from typing import Tuple  # enc, ct
+
+from cryptography.hazmat.primitives import hashes, hmac  # HKDF pieces
+from cryptography.hazmat.primitives.asymmetric.x25519 import (  # KEM
+    X25519PrivateKey,
+    X25519PublicKey,
+)
+from cryptography.hazmat.primitives.ciphers.aead import ChaCha20Poly1305  # AEAD
+from cryptography.hazmat.primitives.serialization import (  # raw keys
+    Encoding,
+    NoEncryption,
+    PrivateFormat,
+    PublicFormat,
+)
+
+KEM_ID = 0x0020  # DHKEM(X25519, HKDF-SHA256)
+KDF_ID = 0x0001  # HKDF-SHA256
+AEAD_ID = 0x0003  # ChaCha20Poly1305
+MODE_BASE = 0x00  # HPKE Base
+SUITE_ID = b"HPKE" + KEM_ID.to_bytes(2, "big") + KDF_ID.to_bytes(2, "big") + AEAD_ID.to_bytes(2, "big")
+N_ENC = 32  # X25519 public
+N_PK = 32
+N_SK = 32
+N_NOISE = 12  # ChaCha nonce length in HPKE
+N_K = 32  # key
+AAD_DEFAULT = b"verae.npe.v1"  # bind content to this system
+
+
+def _hmac(key: bytes, data: bytes) -> bytes:
+    h = hmac.HMAC(key, hashes.SHA256())  # RFC HMAC
+    h.update(data)
+    return h.finalize()
+
+
+def _extract(salt: bytes, ikm: bytes) -> bytes:
+    if not salt:  # HMAC key
+        salt = b"\x00" * 32
+    return _hmac(salt, ikm)
+
+
+def _expand(prk: bytes, info: bytes, length: int) -> bytes:
+    out = b""  # T(0)|T(1)...
+    t = b""
+    i = 1
+    while len(out) < length:
+        t = _hmac(prk, t + info + bytes([i]))
+        out += t
+        i += 1
+    return out[:length]
+
+
+def _labeled_extract(salt: bytes, label: bytes, ikm: bytes) -> bytes:
+    labeled = b"HPKE-v1" + SUITE_ID + label + ikm  # RFC 9180
+    return _extract(salt, labeled)
+
+
+def _labeled_expand(prk: bytes, label: bytes, info: bytes, length: int) -> bytes:
+    labeled = length.to_bytes(2, "big") + b"HPKE-v1" + SUITE_ID + label + info
+    return _expand(prk, labeled, length)
+
+
+def generate_keypair() -> Tuple[bytes, bytes]:
+    """Return (raw_sk, raw_pk) 32+32 bytes."""
+    sk = X25519PrivateKey.generate()  # CSPRNG
+    pk = sk.public_key()
+    return (
+        sk.private_bytes(Encoding.Raw, PrivateFormat.Raw, NoEncryption()),
+        pk.public_bytes(Encoding.Raw, PublicFormat.Raw),
+    )
+
+
+def _dh(sk: bytes, pk: bytes) -> bytes:
+    priv = X25519PrivateKey.from_private_bytes(sk)
+    pub = X25519PublicKey.from_public_bytes(pk)
+    return priv.exchange(pub)
+
+
+def _extract_and_expand(dh: bytes, kem_context: bytes) -> bytes:
+    eae_prk = _labeled_extract(b"", b"eae_prk", dh)
+    return _labeled_expand(eae_prk, b"shared_secret", kem_context, 32)
+
+
+def _key_schedule(shared: bytes, info: bytes) -> Tuple[bytes, bytes]:
+    psk_id_hash = _labeled_extract(b"", b"psk_id_hash", b"")
+    info_hash = _labeled_extract(b"", b"info_hash", info)
+    ks_ctx = bytes([MODE_BASE]) + psk_id_hash + info_hash
+    secret = _labeled_extract(shared, b"secret", b"")
+    key = _labeled_expand(secret, b"key", ks_ctx, N_K)
+    base_nonce = _labeled_expand(secret, b"base_nonce", ks_ctx, N_NOISE)
+    return key, base_nonce
+
+
+def seal(recipient_pk: bytes, plaintext: bytes, aad: bytes = AAD_DEFAULT) -> Tuple[bytes, bytes]:
+    """HPKE-Base seal. Returns (encapped_key, ciphertext)."""
+    eph_sk, eph_pk = generate_keypair()  # ephemeral
+    dh = _dh(eph_sk, recipient_pk)
+    kem_context = eph_pk + recipient_pk
+    shared = _extract_and_expand(dh, kem_context)
+    key, nonce = _key_schedule(shared, b"")
+    ct = ChaCha20Poly1305(key).encrypt(nonce, plaintext, aad)
+    return eph_pk, ct
+
+
+def open_ct(recipient_sk: bytes, enc: bytes, ciphertext: bytes, aad: bytes = AAD_DEFAULT) -> bytes:
+    """HPKE-Base open. Raises on auth failure."""
+    rec_sk = X25519PrivateKey.from_private_bytes(recipient_sk)
+    rec_pk = rec_sk.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw)
+    dh = _dh(recipient_sk, enc)
+    kem_context = enc + rec_pk
+    shared = _extract_and_expand(dh, kem_context)
+    key, nonce = _key_schedule(shared, b"")
+    return ChaCha20Poly1305(key).decrypt(nonce, ciphertext, aad)

python/secure_messaging/pubkey_dir.py

@@ -0,0 +1,93 @@
+"""Public-key directory for NPE/HPKE endpoint encryption.
+
+Only *public* material is listed. Private keys stay in private/ (0600).
+Every service that needs E2E looks up handles here (HTTP or files).
+"""
+
+from __future__ import annotations  # annotations
+
+import json  # records
+import os  # chmod
+import time  # updated_ts
+from pathlib import Path  # dir root
+from typing import Any, Dict, List, Optional  # types
+
+from .hpke import generate_keypair  # X25519
+
+
+class PubKeyDir:
+    """File-backed directory: public/<handle>.json + private/<handle>.sk."""
+
+    def __init__(self, root: Path):
+        self.root = Path(root)  # /opt/pfc/etc/npe-dir
+        self.public = self.root / "public"
+        self.private = self.root / "private"
+        self.public.mkdir(parents=True, exist_ok=True)
+        self.private.mkdir(parents=True, exist_ok=True)
+
+    def create_endpoint(self, handle: str) -> Dict[str, Any]:
+        """Generate X25519 keypair; write public record + private sk."""
+        handle = _safe(handle)
+        sk, pk = generate_keypair()
+        rec = {
+            "handle": handle,
+            "kem": "x25519",
+            "kdf": "hkdf-sha256",
+            "aead": "chacha20poly1305",
+            "suite": "HPKE-Base X25519-HKDF-SHA256-ChaCha20-Poly1305",
+            "enc_pk": pk.hex(),
+            "inbox": "npe.inbox." + pk[:16].hex(),
+            "updated_ts": time.time(),
+        }
+        (self.public / (handle + ".json")).write_text(json.dumps(rec, indent=2) + "\n")
+        skp = self.private / (handle + ".sk")
+        skp.write_bytes(sk)
+        os.chmod(skp, 0o600)
+        return rec
+
+    def get(self, handle: str) -> Optional[Dict[str, Any]]:
+        p = self.public / (_safe(handle) + ".json")
+        if not p.exists():
+            return None
+        return json.loads(p.read_text())
+
+    def list(self) -> List[Dict[str, Any]]:
+        rows = []
+        for p in sorted(self.public.glob("*.json")):
+            rows.append(json.loads(p.read_text()))
+        return rows
+
+    def put_public(self, rec: Dict[str, Any]) -> Dict[str, Any]:
+        """Publish someone else's public key (no private)."""
+        handle = _safe(str(rec.get("handle") or ""))
+        if not handle or not rec.get("enc_pk"):
+            raise ValueError("handle and enc_pk required")
+        rec = dict(rec)
+        rec["handle"] = handle
+        rec["updated_ts"] = time.time()
+        (self.public / (handle + ".json")).write_text(json.dumps(rec, indent=2) + "\n")
+        return rec
+
+    def load_sk(self, handle: str) -> bytes:
+        p = self.private / (_safe(handle) + ".sk")
+        if not p.exists():
+            raise FileNotFoundError("no private key for " + handle)
+        return p.read_bytes()
+
+    def available(self) -> Dict[str, Any]:
+        """What every E2E service can see: handles + public keys only."""
+        recs = self.list()
+        return {
+            "ok": True,
+            "directory": str(self.root),
+            "suite": "HPKE-Base X25519-HKDF-SHA256-ChaCha20-Poly1305",
+            "handles": [r["handle"] for r in recs],
+            "keys": recs,
+        }
+
+
+def _safe(handle: str) -> str:
+    h = "".join(c for c in handle if c.isalnum() or c in "-_")
+    if not h:
+        raise ValueError("empty handle")
+    return h

python/secure_messaging/router.py

@@ -13,7 +13,8 @@ from typing import Any, Dict, List, Optional  # types
 from .error_bundle import NetworkErrorBundle, build_bundle, log_summary  # failure path
 
 
-ALLOWED_ALG = {"npe", "lab-xor", "plain-lab"}  # signed-config crypto.mode set
+ALLOWED_ALG = {"npe"}  # production E2E; lab-xor/plain-lab rejected for content
+LAB_ALG = {"lab-xor", "plain-lab"}  # only if SM_ALLOW_LAB=1
 
 
 class RouteResult:
@@ -65,7 +66,13 @@ def handle_send(
         return _fail("SM-PLAINTEXT-BODY", "mailbox", lid, sender_pub, system_pub, "body field forbidden")
     if not to:  # catalog reject: missing to
         return _fail("SM-MISSING-TO", "mailbox", lid, sender_pub, system_pub, "missing to")
-    if alg and alg not in ALLOWED_ALG:  # alg not in signed config
+    import os  # SM_ALLOW_LAB
+
+    allow_lab = os.environ.get("SM_ALLOW_LAB") == "1"
+    allowed = set(ALLOWED_ALG)
+    if allow_lab:
+        allowed |= LAB_ALG
+    if alg and alg not in allowed:  # xor/plain are not production content
         return _fail("SM-BAD-ALG", "mailbox", lid, sender_pub, system_pub, "alg not allowed")
     if alg != "plain-lab" and not ct:  # empty ciphertext when not plain-lab
         return _fail("SM-EMPTY-CT", "mailbox", lid, sender_pub, system_pub, "empty ciphertext")

tests/test_sm.py

@@ -93,23 +93,65 @@ class HistoryTests(unittest.TestCase):
                 h.append_change(actor="", prev_text="", new_text="x")
 
 
+class HpkeDirTests(unittest.TestCase):
+    def test_roundtrip_and_router_cannot_see_body(self):
+        from secure_messaging.content import routing_view, unwrap_content, wrap_content
+        from secure_messaging.pubkey_dir import PubKeyDir
+
+        with tempfile.TemporaryDirectory() as td:
+            d = PubKeyDir(Path(td))
+            d.create_endpoint("alice")
+            d.create_endpoint("pfc-repl")
+            avail = d.available()
+            self.assertIn("pfc-repl", avail["handles"])
+            self.assertTrue(all("sk" not in json.dumps(k) for k in avail["keys"]))
+            env = wrap_content(
+                to_handle="pfc-repl",
+                sender="alice",
+                body={"bytes_hex": "ab", "note": "secret-phi"},
+                directory=d,
+            )
+            self.assertEqual(env["alg"], "npe")
+            rv = routing_view(env)
+            self.assertNotIn("ct", rv)
+            blob = json.dumps(env)
+            self.assertNotIn("secret-phi", blob)
+            self.assertNotIn("bytes_hex", routing_view(env))
+            pt = unwrap_content(env, handle="pfc-repl", directory=d)
+            self.assertEqual(pt["note"], "secret-phi")
+            with self.assertRaises(Exception):
+                unwrap_content(env, handle="alice", directory=d)
+
+
 class RouterTests(unittest.TestCase):
     def test_good_passthrough(self):
-        from secure_messaging.envelope import seal
         from secure_messaging.router import handle_send
 
-        env = seal(to="npe.inbox.abc", sender="alice", body={"note": "hello"}, mode="lab-xor", lab_key=b"lab")
-        r = handle_send(json.dumps(env.wire()).encode())
+        env = {
+            "to": "npe.inbox.abc",
+            "to_handle": "pfc-repl",
+            "alg": "npe",
+            "ct": "abcd",
+            "enc": "00",
+            "from_lookup_id": "lid-npe",
+        }
+        r = handle_send(json.dumps(env).encode())
         self.assertTrue(r.accepted)
-        self.assertEqual(r.ack()["lookup_id"], env.from_lookup_id)
+        self.assertEqual(r.ack()["lookup_id"], "lid-npe")
         self.assertNotIn("ct", r.ack())
         self.assertEqual(r.events, [])
 
+    def test_xor_rejected_for_content(self):
+        from secure_messaging.router import handle_send
+
+        r = handle_send(b'{"to":"npe.inbox.x","alg":"lab-xor","ct":"ab"}')
+        self.assertEqual(r.error_code, "SM-BAD-ALG")
+
     def test_missing_to_failure_path(self):
         from secure_messaging.router import handle_send
         from secure_messaging.error_bundle import open_sender, open_system
 
-        r = handle_send(b'{"alg":"lab-xor","ct":"ab","from_lookup_id":"lid-x"}')
+        r = handle_send(b'{"alg":"npe","ct":"ab","from_lookup_id":"lid-x"}')
         self.assertFalse(r.accepted)
         self.assertEqual(r.error_code, "SM-MISSING-TO")
         subjects = [e["subject"] for e in r.events]
@@ -125,9 +167,9 @@ class RouterTests(unittest.TestCase):
     def test_empty_ct_and_plaintext_body(self):
         from secure_messaging.router import handle_send
 
-        r = handle_send(b'{"to":"npe.inbox.x","alg":"lab-xor","ct":""}')
+        r = handle_send(b'{"to":"npe.inbox.x","alg":"npe","ct":""}')
         self.assertEqual(r.error_code, "SM-EMPTY-CT")
-        r2 = handle_send(b'{"to":"npe.inbox.x","alg":"lab-xor","ct":"ab","body":{"secret":1}}')
+        r2 = handle_send(b'{"to":"npe.inbox.x","alg":"npe","ct":"ab","body":{"secret":1}}')
         self.assertEqual(r2.error_code, "SM-PLAINTEXT-BODY")