FMSG-001 TCP+TLS Standard (#17)

* implement compression

* rm hyperbole

* deflate vs. compress wording

* TLS per FMSG-001

* rm redundant text

Author: markmnl

AGENTS.md

@@ -11,7 +11,7 @@ All code MUST conform to the specification. When in doubt, re-read SPEC.md and f
 - Serialize and parse wire fields in the exact order defined in SPEC.md.
 - Use the flag bit assignments from SPEC.md (bit 0 = has pid, bit 1 = has add to, bit 2 = common type, etc.).im
 - Reject/accept response codes must match SPEC.md — do not invent new codes.
-- Resolve `_fmsg.<domain>` using A/AAAA records only (never TXT, MX, or SRV).
+- Resolve `fmsg.<domain>` using A/AAAA records only (never TXT, MX, or SRV).
 - Validate sender IP before issuing CHALLENGE.
 - Connection 2 for challenge MUST target the same IP as Connection 1.
 - Topic field is only present on the wire when pid is absent.

README.md

@@ -3,7 +3,7 @@
 [![Go 1.25](https://github.com/markmnl/fmsgd/actions/workflows/go1.25.yml/badge.svg)](https://github.com/markmnl/fmsgd/actions/workflows/go1.25.yml)
 
 
-Implementation of [fmsg](https://github.com/markmnl/fmsg) host written in Go! Uses local filesystem and PostgreSQL database to store messages per the [fmsg-store-postgres standard](https://github.com/markmnl/fmsg/blob/main/STANDARDS.md).
+Implementation of [fmsg](https://github.com/markmnl/fmsg) host written in Go! Uses local filesystem and PostgreSQL database to store messages.
 
 ## Building from source
 
@@ -16,13 +16,16 @@ Tested with Go 1.25 on Linux and Windows, AMD64 and ARM
 
 ## Environment
 
-`FMSG_DATA_DIR`, `FMSG_DOMAIN` and `FMSG_ID_URL` are required to be set and valid; otherwise fmsgd will abort on startup. In addition to these `FMSG_` varibles, `PG` variables need to be set for the PostgreSQL database to use, refer to: https://www.postgresql.org/docs/current/libpq-envars.html
+`FMSG_DATA_DIR`, `FMSG_DOMAIN`, `FMSG_ID_URL`, `FMSG_TLS_CERT` and `FMSG_TLS_KEY` are required to be set and valid; otherwise fmsgd will abort on startup. In addition to these `FMSG_` varibles, `PG` variables need to be set for the PostgreSQL database to use, refer to: https://www.postgresql.org/docs/current/libpq-envars.html
 
 | Variable                   | Default | Description                                                                                                                                             |
 |----------------------------|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------|
 | FMSG_DATA_DIR              |         | Path where messages will be stored. e.g. /opt/fmsg/data                                                                                                 |
 | FMSG_DOMAIN                |         | Domain name this host is located. e.g. example.com                                                                                                      |
 | FMSG_ID_URL                |         | Base HTTP URL for fmsg Id API, e.g. http://localhost:5000                                                                                                    |
+| FMSG_TLS_CERT              |         | Path to TLS certificate file (PEM). Certificate must match `fmsg.<FMSG_DOMAIN>`.                                                                       |
+| FMSG_TLS_KEY               |         | Path to TLS private key file (PEM).                                                                                                                     |
+| FMSG_TLS_INSECURE_SKIP_VERIFY | false | Set to "true" to skip TLS certificate verification on outgoing connections. For development/testing only.                                               |
 | FMSG_MAX_MSG_SIZE          | 10240   | Bytes. Maximum size above which to reject messages greater than before downloading them.                                                                |
 | FMSG_PORT                  | 4930    | TCP port to listen on                                                                                                                                   |
 | FMSG_MAX_PAST_TIME_DELTA   | 604800  | Seconds. Duration since message timestamp to reject if greater than. Note sending host could have been holding messages waiting for us to be reachable. |
@@ -34,8 +37,8 @@ Tested with Go 1.25 on Linux and Windows, AMD64 and ARM
 | FMSG_RETRY_MAX_AGE         | 86400   | Seconds. Maximum age of a message since creation before giving up on delivery retries (default 1 day).                                                 |
 | FMSG_POLL_INTERVAL         | 10      | Seconds. How often the sender polls the database for pending messages.                                                                                 |
 | FMSG_MAX_CONCURRENT_SEND   | 1024    | Maximum number of concurrent outbound message deliveries.                                                                                              |
-| FMSG_SKIP_DOMAIN_IP_CHECK  | false   | Set to "true" to skip verifying this host's external IP is in the _fmsg DNS authorised IP set on startup.                                              |
-| FMSG_SKIP_AUTHORISED_IPS  | false   | Set to "true" to skip verifying remote hosts IP is in the _fmsg DNS authorised IP set during message exchange. WARNING setting this true is effectivly disables sender verification.                                              |
+| FMSG_SKIP_DOMAIN_IP_CHECK  | false   | Set to "true" to skip verifying this host's external IP is in the fmsg DNS authorised IP set on startup.                                               |
+| FMSG_SKIP_AUTHORISED_IPS  | false   | Set to "true" to skip verifying remote hosts IP is in the fmsg DNS authorised IP set during message exchange. WARNING setting this true effectively disables sender verification. |
 
 
 

SPEC.md

@@ -153,7 +153,7 @@ Per-recipient codes (one byte per recipient on this host, in message order):
 
 ## 9. Domain Resolution
 
-Resolve `_fmsg.<domain>` for A/AAAA records. The sender's domain is:
+Resolve `fmsg.<domain>` for A/AAAA records. The sender's domain is:
 - The domain of _add to from_ when _has add to_ is set.
 - The domain of _from_ otherwise.
 
@@ -175,7 +175,7 @@ One message per connection. Two TCP connections used: Connection 1 (message tran
 
 Host A delivers iff _from_ or _add to from_ belongs to Host A's domain. For each unique recipient domain:
 
-1. Resolve recipient domain IPs via `_fmsg.<domain>`. Connect to first responsive IP (Connection 1). Retry with backoff if unreachable.
+1. Resolve recipient domain IPs via `fmsg.<domain>`. Connect to first responsive IP (Connection 1). Retry with backoff if unreachable.
 2. Register the message header hash and Host B's IP in an outgoing record (for matching challenges).
 3. Transmit the message header on Connection 1.
 4. Wait for response. During this wait, be ready to handle a CHALLENGE on Connection 2 (see §10.5).
@@ -200,7 +200,7 @@ Host A delivers iff _from_ or _add to from_ belongs to Host A's domain. For each
    - If _has add to_: _add to from_ exists and is in _from_ or _to_; _add to_ has ≥ 1 distinct address.
    - ≥ 1 recipient in _to_ or _add to_ belongs to Host B's domain.
    - Common type IDs (message and attachment) are mapped.
-4. DNS-verify sender IP: resolve `_fmsg.<sender domain>`, check Connection 1 source IP is in result set. Fail → TERMINATE.
+4. DNS-verify sender IP: resolve `fmsg.<sender domain>`, check Connection 1 source IP is in result set. Fail → TERMINATE.
 5. If _size_ + attachment sizes > MAX_SIZE → respond code 4, close.
 6. Compute DELTA = now − _time_:
    - DELTA > MAX_MESSAGE_AGE → respond code 7, close.

src/dns.go

@@ -53,9 +53,9 @@ func resolverAuthenticatedData(name string, qtype uint16) (bool, error) {
 	return false, lastErr
 }
 
-// lookupAuthorisedIPs resolves _fmsg.<domain> for A and AAAA records
+// lookupAuthorisedIPs resolves fmsg.<domain> for A and AAAA records
 func lookupAuthorisedIPs(domain string) ([]net.IP, error) {
-	fmsgDomain := "_fmsg." + domain
+	fmsgDomain := "fmsg." + domain
 	ips, err := net.LookupIP(fmsgDomain)
 	if err != nil {
 		return nil, fmt.Errorf("DNS lookup for %s failed: %w", fmsgDomain, err)
@@ -114,7 +114,7 @@ func getExternalIP() (net.IP, error) {
 }
 
 // verifyDomainIP checks that this host's external IP is present in the
-// _fmsg.<domain> authorised IP set. Panics if not found.
+// fmsg.<domain> authorised IP set. Panics if not found.
 func verifyDomainIP(domain string) {
 	externalIP, err := getExternalIP()
 	if err != nil {
@@ -124,17 +124,17 @@ func verifyDomainIP(domain string) {
 
 	authorisedIPs, err := lookupAuthorisedIPs(domain)
 	if err != nil {
-		log.Panicf("ERROR: failed to lookup _fmsg.%s: %s", domain, err)
+		log.Panicf("ERROR: failed to lookup fmsg.%s: %s", domain, err)
 	}
 
 	for _, ip := range authorisedIPs {
 		if externalIP.Equal(ip) {
-			log.Printf("INFO: external IP %s found in _fmsg.%s authorised IPs", externalIP, domain)
+			log.Printf("INFO: external IP %s found in fmsg.%s authorised IPs", externalIP, domain)
 			return
 		}
 	}
 
-	log.Panicf("ERROR: external IP %s not found in _fmsg.%s authorised IPs %v", externalIP, domain, authorisedIPs)
+	log.Panicf("ERROR: external IP %s not found in fmsg.%s authorised IPs %v", externalIP, domain, authorisedIPs)
 }
 
 // checkDomainIP verifies the external IP is authorised unless

src/host.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"compress/zlib"
+	"crypto/tls"
 	"encoding/binary"
 	"encoding/hex"
 	"errors"
@@ -160,12 +161,45 @@ var MinUploadRate float64 = 5000
 var ReadBufferSize = 1600
 var MaxMessageSize = uint32(1024 * 10)
 var SkipAuthorisedIPs = false
+var TLSInsecureSkipVerify = false
 var DataDir = "got on startup"
 var Domain = "got on startup"
 var IDURI = "got on startup"
 var AtRune, _ = utf8.DecodeRuneInString("@")
 var MinNetIODeadline = 6 * time.Second
 
+var serverTLSConfig *tls.Config
+
+func buildServerTLSConfig() *tls.Config {
+	certFile := os.Getenv("FMSG_TLS_CERT")
+	keyFile := os.Getenv("FMSG_TLS_KEY")
+	if certFile == "" || keyFile == "" {
+		log.Fatalf("ERROR: FMSG_TLS_CERT and FMSG_TLS_KEY must be set")
+	}
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		log.Fatalf("ERROR: loading TLS certificate: %s", err)
+	}
+	return &tls.Config{
+		Certificates: []tls.Certificate{cert},
+		MinVersion:   tls.VersionTLS12,
+		CipherSuites: []uint16{
+			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+		},
+		NextProtos: []string{"fmsg/1"},
+	}
+}
+
+func buildClientTLSConfig(serverName string) *tls.Config {
+	return &tls.Config{
+		ServerName:         serverName,
+		MinVersion:         tls.VersionTLS12,
+		InsecureSkipVerify: TLSInsecureSkipVerify,
+		NextProtos:         []string{"fmsg/1"},
+	}
+}
+
 // loadEnvConfig reads env vars (after godotenv.Load so .env is picked up).
 func loadEnvConfig() {
 	Port = env.GetIntDefault("FMSG_PORT", 4930)
@@ -177,6 +211,7 @@ func loadEnvConfig() {
 	ReadBufferSize = env.GetIntDefault("FMSG_READ_BUFFER_SIZE", 1600)
 	MaxMessageSize = uint32(env.GetIntDefault("FMSG_MAX_MSG_SIZE", 1024*10))
 	SkipAuthorisedIPs = os.Getenv("FMSG_SKIP_AUTHORISED_IPS") == "true"
+	TLSInsecureSkipVerify = os.Getenv("FMSG_TLS_INSECURE_SKIP_VERIFY") == "true"
 }
 
 // Updates DataDir from environment, panics if not a valid directory.
@@ -204,7 +239,7 @@ func setDomain() {
 	}
 	Domain = domain
 
-	// verify our external IP is in the _fmsg authorised IP set
+	// verify our external IP is in the fmsg authorised IP set
 	checkDomainIP(domain)
 }
 
@@ -518,7 +553,7 @@ func verifySenderIP(c net.Conn, senderDomain string) error {
 
 	authorisedIPs, err := lookupAuthorisedIPs(senderDomain)
 	if err != nil {
-		log.Printf("WARN: DNS lookup failed for _fmsg.%s: %s", senderDomain, err)
+		log.Printf("WARN: DNS lookup failed for fmsg.%s: %s", senderDomain, err)
 		return fmt.Errorf("DNS verification failed")
 	}
 
@@ -528,7 +563,7 @@ func verifySenderIP(c net.Conn, senderDomain string) error {
 		}
 	}
 
-	log.Printf("WARN: remote IP %s not in authorised IPs for _fmsg.%s", remoteIP.String(), senderDomain)
+	log.Printf("WARN: remote IP %s not in authorised IPs for fmsg.%s", remoteIP.String(), senderDomain)
 	return fmt.Errorf("DNS verification failed")
 }
 
@@ -1031,14 +1066,14 @@ func readHeader(c net.Conn) (*FMsgHeader, *bufio.Reader, error) {
 // TODO [Spec step 2]: The spec defines challenge modes (NEVER, ALWAYS,
 // HAS_NOT_PARTICIPATED, DIFFERENT_DOMAIN) as implementation choices.
 // Currently defaults to ALWAYS. Implement configurable challenge mode.
-func challenge(conn net.Conn, h *FMsgHeader) error {
+func challenge(conn net.Conn, h *FMsgHeader, senderDomain string) error {
 
 	// Connection 2 MUST target the same IP as Connection 1 (spec 2.1).
 	remoteHost, _, err := net.SplitHostPort(conn.RemoteAddr().String())
 	if err != nil {
 		return fmt.Errorf("failed to parse remote address for challenge: %w", err)
 	}
-	conn2, err := net.Dial("tcp", net.JoinHostPort(remoteHost, fmt.Sprintf("%d", RemotePort)))
+	conn2, err := tls.Dial("tcp", net.JoinHostPort(remoteHost, fmt.Sprintf("%d", RemotePort)), buildClientTLSConfig("fmsg."+senderDomain))
 	if err != nil {
 		return err
 	}
@@ -1506,7 +1541,7 @@ func handleConn(c net.Conn) {
 		return
 	}
 
-	if err := challenge(c, header); err != nil {
+	if err := challenge(c, header, determineSenderDomain(header)); err != nil {
 		log.Printf("ERROR: Challenge failed to, %s: %s", c.RemoteAddr().String(), err)
 		abortConn(c)
 		return
@@ -1576,6 +1611,9 @@ func main() {
 	setDomain()
 	setIDURL()
 
+	// load TLS configuration (must be after loadEnvConfig for FMSG_TLS_INSECURE_SKIP_VERIFY)
+	serverTLSConfig = buildServerTLSConfig()
+
 	// start sender in background (small delay so listener is ready first)
 	go func() {
 		time.Sleep(1 * time.Second)
@@ -1584,7 +1622,7 @@ func main() {
 
 	// start listening
 	addr := fmt.Sprintf("%s:%d", listenAddress, Port)
-	ln, err := net.Listen("tcp", addr)
+	ln, err := tls.Listen("tcp", addr, serverTLSConfig)
 	if err != nil {
 		log.Fatal(err)
 	}

src/sender.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"crypto/tls"
 	"database/sql"
 	"encoding/hex"
 	"fmt"
@@ -398,21 +399,23 @@ func deliverMessage(target pendingTarget) {
 
 	targetIPs, err := lookupAuthorisedIPs(target.Domain)
 	if err != nil {
-		log.Printf("ERROR: sender: DNS lookup for _fmsg.%s failed: %s", target.Domain, err)
+		log.Printf("ERROR: sender: DNS lookup for fmsg.%s failed: %s", target.Domain, err)
 		return
 	}
 
 	var conn net.Conn
+	dialer := &net.Dialer{Timeout: 10 * time.Second}
+	tlsConf := buildClientTLSConfig("fmsg." + target.Domain)
 	for _, ip := range targetIPs {
 		addr := net.JoinHostPort(ip.String(), fmt.Sprintf("%d", RemotePort))
-		conn, err = net.DialTimeout("tcp", addr, 10*time.Second)
+		conn, err = tls.DialWithDialer(dialer, "tcp", addr, tlsConf)
 		if err == nil {
 			break
 		}
 		log.Printf("WARN: sender: connect to %s failed: %s", addr, err)
 	}
 	if conn == nil {
-		log.Printf("ERROR: sender: could not connect to any IP for _fmsg.%s", target.Domain)
+		log.Printf("ERROR: sender: could not connect to any IP for fmsg.%s", target.Domain)
 		return
 	}
 	defer conn.Close()