Skip to content

Instantly share code, notes, and snippets.

@bassosimone

bassosimone/step-by-step-telegram.patch Secret

Created June 11, 2022 12:28
Show Gist options
  • Save bassosimone/f6e680d35805174d1f150bc15ef754af to your computer and use it in GitHub Desktop.
Save bassosimone/f6e680d35805174d1f150bc15ef754af to your computer and use it in GitHub Desktop.
Download ZIP
Raw
step-by-step-telegram.patch
diff --git a/internal/engine/allexperiments.go b/internal/engine/allexperiments.go
index f702118..1c77f98 100644
--- a/internal/engine/allexperiments.go
+++ b/internal/engine/allexperiments.go
@@ -21,7 +21,6 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/experiment/sniblocking"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/stunreachability"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/tcpping"
- "github.com/ooni/probe-cli/v3/internal/engine/experiment/telegram"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/tlsping"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/tlstool"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/tor"
@@ -30,6 +29,7 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/experiment/vanillator"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/whatsapp"
+ "github.com/ooni/probe-cli/v3/internal/experiment/telegram"
)
var experimentsByName = map[string]func(*Session) *ExperimentBuilder{
diff --git a/internal/engine/experiment/tcpping/tcpping.go b/internal/engine/experiment/tcpping/tcpping.go
index ef56a01..f024d25 100644
--- a/internal/engine/experiment/tcpping/tcpping.go
+++ b/internal/engine/experiment/tcpping/tcpping.go
@@ -10,13 +10,13 @@ import (
"net/url"
"time"
- "github.com/ooni/probe-cli/v3/internal/measurex"
+ "github.com/ooni/probe-cli/v3/internal/measurexlite"
"github.com/ooni/probe-cli/v3/internal/model"
)
const (
testName = "tcpping"
- testVersion = "0.1.0"
+ testVersion = "0.2.0"
)
// Config contains the experiment configuration.
@@ -49,7 +49,7 @@ type TestKeys struct {
// SinglePing contains the results of a single ping.
type SinglePing struct {
- TCPConnect []*measurex.ArchivalTCPConnect `json:"tcp_connect"`
+ TCPConnect *model.ArchivalTCPConnectResult `json:"tcp_connect"`
}
// Measurer performs the measurement.
@@ -101,44 +101,43 @@ func (m *Measurer) Run(
if parsed.Port() == "" {
return errMissingPort
}
- tk := new(TestKeys)
+ tk := &TestKeys{}
measurement.TestKeys = tk
- out := make(chan *measurex.EndpointMeasurement)
- mxmx := measurex.NewMeasurerWithDefaultSettings()
- go m.tcpPingLoop(ctx, mxmx, parsed.Host, out)
+ out := make(chan *SinglePing)
+ go m.tcpPingLoop(ctx, measurement.MeasurementStartTimeSaved, sess.Logger(), parsed.Host, out)
for len(tk.Pings) < int(m.config.repetitions()) {
- meas := <-out
- tk.Pings = append(tk.Pings, &SinglePing{
- TCPConnect: measurex.NewArchivalTCPConnectList(meas.Connect),
- })
+ tk.Pings = append(tk.Pings, <-out)
}
return nil // return nil so we always submit the measurement
}
// tcpPingLoop sends all the ping requests and emits the results onto the out channel.
-func (m *Measurer) tcpPingLoop(ctx context.Context, mxmx *measurex.Measurer,
- address string, out chan<- *measurex.EndpointMeasurement) {
+func (m *Measurer) tcpPingLoop(ctx context.Context, zeroTime time.Time, logger model.Logger,
+ address string, out chan<- *SinglePing) {
ticker := time.NewTicker(m.config.delay())
defer ticker.Stop()
for i := int64(0); i < m.config.repetitions(); i++ {
- go m.tcpPingAsync(ctx, mxmx, address, out)
+ go m.tcpPingAsync(ctx, i, zeroTime, logger, address, out)
<-ticker.C
}
}
// tcpPingAsync performs a TCP ping and emits the result onto the out channel.
-func (m *Measurer) tcpPingAsync(ctx context.Context, mxmx *measurex.Measurer,
- address string, out chan<- *measurex.EndpointMeasurement) {
- out <- m.tcpConnect(ctx, mxmx, address)
+func (m *Measurer) tcpPingAsync(ctx context.Context, index int64, zeroTime time.Time,
+ logger model.Logger, address string, out chan<- *SinglePing) {
+ out <- m.tcpConnect(ctx, index, zeroTime, logger, address)
}
// tcpConnect performs a TCP connect and returns the result to the caller.
-func (m *Measurer) tcpConnect(ctx context.Context, mxmx *measurex.Measurer,
- address string) *measurex.EndpointMeasurement {
+func (m *Measurer) tcpConnect(ctx context.Context, index int64,
+ zeroTime time.Time, logger model.Logger, address string) *SinglePing {
// TODO(bassosimone): make the timeout user-configurable
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
- return mxmx.TCPConnect(ctx, address)
+ dialer := measurexlite.NewDialer(index, logger, zeroTime)
+ conn, _ := dialer.DialContext(ctx, "tcp", address)
+ measurexlite.MaybeClose(conn)
+ return &SinglePing{dialer.TCPConnectResult()}
}
// NewExperimentMeasurer creates a new ExperimentMeasurer.
diff --git a/internal/engine/experiment/tlsping/tlsping.go b/internal/engine/experiment/tlsping/tlsping.go
index 8049cc9..fbc2cda 100644
--- a/internal/engine/experiment/tlsping/tlsping.go
+++ b/internal/engine/experiment/tlsping/tlsping.go
@@ -13,14 +13,14 @@ import (
"strings"
"time"
- "github.com/ooni/probe-cli/v3/internal/measurex"
+ "github.com/ooni/probe-cli/v3/internal/measurexlite"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
const (
testName = "tlsping"
- testVersion = "0.1.0"
+ testVersion = "0.2.0"
)
// Config contains the experiment configuration.
@@ -77,9 +77,9 @@ type TestKeys struct {
// SinglePing contains the results of a single ping.
type SinglePing struct {
- NetworkEvents []*measurex.ArchivalNetworkEvent `json:"network_events"`
- TCPConnect []*measurex.ArchivalTCPConnect `json:"tcp_connect"`
- TLSHandshakes []*measurex.ArchivalQUICTLSHandshakeEvent `json:"tls_handshakes"`
+ NetworkEvents []*model.ArchivalNetworkEvent `json:"network_events"`
+ TCPConnect *model.ArchivalTCPConnectResult `json:"tcp_connect"`
+ TLSHandshake *model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshake"`
}
// Measurer performs the measurement.
@@ -133,49 +133,64 @@ func (m *Measurer) Run(
}
tk := new(TestKeys)
measurement.TestKeys = tk
- out := make(chan *measurex.EndpointMeasurement)
- mxmx := measurex.NewMeasurerWithDefaultSettings()
- go m.tlsPingLoop(ctx, mxmx, parsed.Host, out)
+ out := make(chan *SinglePing)
+ go m.tlsPingLoop(ctx, measurement.MeasurementStartTimeSaved, sess.Logger(), parsed.Host, out)
for len(tk.Pings) < int(m.config.repetitions()) {
- meas := <-out
- tk.Pings = append(tk.Pings, &SinglePing{
- NetworkEvents: measurex.NewArchivalNetworkEventList(meas.ReadWrite),
- TCPConnect: measurex.NewArchivalTCPConnectList(meas.Connect),
- TLSHandshakes: measurex.NewArchivalQUICTLSHandshakeEventList(meas.TLSHandshake),
- })
+ tk.Pings = append(tk.Pings, <-out)
}
return nil // return nil so we always submit the measurement
}
// tlsPingLoop sends all the ping requests and emits the results onto the out channel.
-func (m *Measurer) tlsPingLoop(ctx context.Context, mxmx *measurex.Measurer,
- address string, out chan<- *measurex.EndpointMeasurement) {
+func (m *Measurer) tlsPingLoop(ctx context.Context, zeroTime time.Time,
+ logger model.Logger, address string, out chan<- *SinglePing) {
ticker := time.NewTicker(m.config.delay())
defer ticker.Stop()
for i := int64(0); i < m.config.repetitions(); i++ {
- go m.tlsPingAsync(ctx, mxmx, address, out)
+ go m.tlsPingAsync(ctx, i, zeroTime, logger, address, out)
<-ticker.C
}
}
// tlsPingAsync performs a TLS ping and emits the result onto the out channel.
-func (m *Measurer) tlsPingAsync(ctx context.Context, mxmx *measurex.Measurer,
- address string, out chan<- *measurex.EndpointMeasurement) {
- out <- m.tlsConnectAndHandshake(ctx, mxmx, address)
+func (m *Measurer) tlsPingAsync(ctx context.Context, index int64, zeroTime time.Time,
+ logger model.Logger, address string, out chan<- *SinglePing) {
+ out <- m.tlsConnectAndHandshake(ctx, index, zeroTime, logger, address)
}
// tlsConnectAndHandshake performs a TCP connect followed by a TLS handshake
// and returns the results of these operations to the caller.
-func (m *Measurer) tlsConnectAndHandshake(ctx context.Context, mxmx *measurex.Measurer,
- address string) *measurex.EndpointMeasurement {
+func (m *Measurer) tlsConnectAndHandshake(ctx context.Context, index int64,
+ zeroTime time.Time, logger model.Logger, address string) (out *SinglePing) {
// TODO(bassosimone): make the timeout user-configurable
- ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+ deadline := time.Now().Add(3 * time.Second)
+ ctx, cancel := context.WithDeadline(ctx, deadline)
defer cancel()
- return mxmx.TLSConnectAndHandshake(ctx, address, &tls.Config{
+ out = &SinglePing{
+ NetworkEvents: []*model.ArchivalNetworkEvent{},
+ TCPConnect: nil,
+ TLSHandshake: nil,
+ }
+ dialer := measurexlite.NewDialer(index, logger, zeroTime)
+ conn, err := dialer.DialContext(ctx, "tcp", address)
+ out.TCPConnect = dialer.TCPConnectResult()
+ if err != nil {
+ return
+ }
+ defer conn.Close()
+ handshaker := measurexlite.NewTLSHandshaker(index, logger, zeroTime)
+ tlsConfig := &tls.Config{
NextProtos: strings.Split(m.config.alpn(), " "),
RootCAs: netxlite.NewDefaultCertPool(),
ServerName: m.config.sni(address),
- })
+ }
+ const maxEvents = 255
+ connWrap := measurexlite.WrapConn(index, zeroTime, conn, maxEvents)
+ tlsConn, _ := handshaker.Handshake(ctx, connWrap, tlsConfig)
+ defer measurexlite.MaybeClose(tlsConn)
+ out.TLSHandshake = handshaker.TLSHandshakeResult()
+ out.NetworkEvents = connWrap.NetworkEvents()
+ return
}
// NewExperimentMeasurer creates a new ExperimentMeasurer.
diff --git a/internal/experiment/telegram/datacenters.go b/internal/experiment/telegram/datacenters.go
new file mode 100644
index 0000000..2ddc41e
--- /dev/null
+++ b/internal/experiment/telegram/datacenters.go
@@ -0,0 +1,83 @@
+package telegram
+
+//
+// Telegram data centers
+//
+
+import (
+ "context"
+ "net"
+ "sync"
+ "time"
+
+ "github.com/ooni/probe-cli/v3/internal/measurexlite"
+ "github.com/ooni/probe-cli/v3/internal/model"
+ "github.com/ooni/probe-cli/v3/internal/runtimex"
+)
+
+// startMeasuringDataCenters starts goroutines for measuring data centers.
+func (mx *Measurer) startMeasuringDataCenters(ctx context.Context,
+ wg *sync.WaitGroup, logger model.Logger, zeroTime time.Time, tk *TestKeys) {
+ addresses := []string{
+ "149.154.175.50",
+ "149.154.167.51",
+ "149.154.175.100",
+ "149.154.167.91",
+ "149.154.171.5",
+ "95.161.76.100",
+ }
+ for _, addr := range addresses {
+ wg.Add(2)
+ go mx.measureDataCenterEndpoint(ctx, wg, logger, zeroTime, tk, addr, "80")
+ go mx.measureDataCenterEndpoint(ctx, wg, logger, zeroTime, tk, addr, "443")
+ }
+}
+
+// measureDataCenterEndpoint measures a single data center endpoint.
+func (mx *Measurer) measureDataCenterEndpoint(ctx context.Context, wg *sync.WaitGroup,
+ logger model.Logger, zeroTime time.Time, tk *TestKeys, addr, port string) {
+
+ const dcTimeout = 7 * time.Second
+ ctx, cancel := context.WithTimeout(ctx, dcTimeout)
+ defer cancel()
+
+ defer wg.Done() // synchronize with the controller
+ dcurl := measurexlite.NewURL("http", addr, port, "/").String()
+ ol := measurexlite.NewOperationLogger(logger, "POST %s", dcurl)
+ index := tk.newIndex()
+ endpoint := net.JoinHostPort(addr, port)
+ tk.registerSubmeasurement(index, endpoint, "datacenter")
+
+ // 1. dial a TCP connection with the DC endpoint
+ dialer := measurexlite.NewDialer(index, logger, zeroTime)
+ conn, err := dialer.DialContext(ctx, "tcp", endpoint)
+ tk.addTCPConnectResult(dialer.TCPConnectResult())
+ if err != nil {
+ ol.Stop(err)
+ return
+ }
+ defer conn.Close()
+
+ // 2. if we arrive here, there's at least a DC acessible at TCP/IP level.
+ tk.onAccessibleDCTCPIP()
+
+ // 3. perform an HTTP round trip with the DC endpoint
+ const networkEventsBuffer = 16
+ cw := measurexlite.WrapConn(index, zeroTime, conn, networkEventsBuffer)
+ req, err := measurexlite.NewHTTPRequestWithContext(ctx, "POST", dcurl, nil)
+ runtimex.PanicOnError(err, "http.NewRequestWithContext failed unexpectedly")
+ const maxBodySnapshotSize = 1 << 17
+ txp := measurexlite.NewHTTPBodyReaderTransport(index, logger, zeroTime, maxBodySnapshotSize)
+ resp, _, _, err := txp.BodyRoundTripWithTCPConn(req, cw)
+ tk.addHTTPRequestResult(txp.HTTPRequestResult())
+ tk.addNetworkEvents(cw.NetworkEvents())
+ if err != nil {
+ ol.Stop(err)
+ return
+ }
+ defer resp.Body.Close()
+
+ // 4. if we arrive here, the DC is accessible at HTTP level.
+ tk.onAccessibleDCHTTP()
+ ol.Stop(nil)
+}
diff --git a/internal/experiment/telegram/doc.go b/internal/experiment/telegram/doc.go
new file mode 100644
index 0000000..6a112ca
--- /dev/null
+++ b/internal/experiment/telegram/doc.go
@@ -0,0 +1,4 @@
+// Package telegram contains the Telegram network experiment.
+//
+// See https://github.com/ooni/spec/blob/master/nettests/ts-020-telegram.md.
+package telegram
diff --git a/internal/experiment/telegram/measurer.go b/internal/experiment/telegram/measurer.go
new file mode 100644
index 0000000..61159fb
--- /dev/null
+++ b/internal/experiment/telegram/measurer.go
@@ -0,0 +1,83 @@
+package telegram
+
+//
+// Measurer
+//
+
+import (
+ "context"
+ "errors"
+ "sync"
+
+ "github.com/ooni/probe-cli/v3/internal/model"
+)
+
+const (
+ testName = "telegram"
+ testVersion = "0.3.0"
+)
+
+// Config contains the telegram experiment config.
+type Config struct{}
+
+// Measurer performs the measurement
+type Measurer struct {
+ // Config contains the experiment settings. If empty we
+ // will be using default settings.
+ Config Config
+}
+
+// ExperimentName implements ExperimentMeasurer.ExperimentName
+func (mx *Measurer) ExperimentName() string {
+ return testName
+}
+
+// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion
+func (mx *Measurer) ExperimentVersion() string {
+ return testVersion
+}
+
+// Run implements ExperimentMeasurer.Run
+func (mx *Measurer) Run(ctx context.Context, sess model.ExperimentSession,
+ measurement *model.Measurement, callbacks model.ExperimentCallbacks) error {
+ tk := NewTestKeys()
+ measurement.TestKeys = tk
+ wg := &sync.WaitGroup{}
+ mx.startMeasuringDataCenters(ctx, wg, sess.Logger(), measurement.MeasurementStartTimeSaved, tk)
+ mx.startMeasuringWeb(ctx, wg, sess.Logger(), measurement.MeasurementStartTimeSaved, tk)
+ wg.Wait()
+ return nil
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+ return &Measurer{Config: config}
+}
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with ooniprobe
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+ HTTPBlocking bool `json:"telegram_http_blocking"`
+ TCPBlocking bool `json:"telegram_tcp_blocking"`
+ WebBlocking bool `json:"telegram_web_blocking"`
+ IsAnomaly bool `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+ sk := SummaryKeys{IsAnomaly: false}
+ tk, ok := measurement.TestKeys.(*TestKeys)
+ if !ok {
+ return sk, errors.New("invalid test keys type")
+ }
+ tcpBlocking := tk.TelegramTCPBlocking
+ httpBlocking := tk.TelegramHTTPBlocking
+ webBlocking := tk.TelegramWebFailure != nil
+ sk.TCPBlocking = tcpBlocking
+ sk.HTTPBlocking = httpBlocking
+ sk.WebBlocking = webBlocking
+ sk.IsAnomaly = webBlocking || httpBlocking || tcpBlocking
+ return sk, nil
+}
diff --git a/internal/experiment/telegram/testkeys.go b/internal/experiment/telegram/testkeys.go
new file mode 100644
index 0000000..886402f
--- /dev/null
+++ b/internal/experiment/telegram/testkeys.go
@@ -0,0 +1,219 @@
+package telegram
+
+//
+// Definition of the test keys
+//
+
+import (
+ "sync"
+
+ "github.com/ooni/probe-cli/v3/internal/measurexlite"
+ "github.com/ooni/probe-cli/v3/internal/model"
+ "github.com/ooni/probe-cli/v3/internal/netxlite"
+)
+
+// TestKeysSubMeasurement describes a sub-measurement.
+type TestKeysSubMeasurement struct {
+ // Index is the sub-measurement index
+ Index int64 `json:"index"`
+
+ // Input is the sub-measurement input
+ Input string `json:"input"`
+
+ // Type is the sub-measurement type
+ Type string `json:"type"`
+}
+
+// TestKeys contains telegram test keys.
+type TestKeys struct {
+ // Submeasurements contains info on sub-measurements.
+ Submeasurements []*TestKeysSubMeasurement `json:"submeasurements"`
+
+ // NetworkEvents contains network events.
+ NetworkEvents []*model.ArchivalNetworkEvent `json:"network_events"`
+
+ // Queries contains DNS events.
+ Queries []*model.ArchivalDNSLookupResult `json:"queries"`
+
+ // Requests contains HTTP requests.
+ Requests []*model.ArchivalHTTPRequestResult `json:"requests"`
+
+ // TCPConnect contains TCP connect events.
+ TCPConnect []*model.ArchivalTCPConnectResult `json:"tcp_connect"`
+
+ // TLSHandshakes contains the TLS handshakes.
+ TLSHandshakes []*model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshakes"`
+
+ // TelegramHTTPBlocking tells the user whether the DCs are blocked via HTTP blocking.
+ TelegramHTTPBlocking bool `json:"telegram_http_blocking"`
+
+ // TelegramTCPBlocking tells the user whether the DCs are blocked via TCP/IP blocking.
+ TelegramTCPBlocking bool `json:"telegram_tcp_blocking"`
+
+ // TelegramWebBogon indicates we resolved a bogon IP address.
+ TelegramWebBogon bool `json:"telegram_web_bogon"`
+
+ // TelegramWebFailure is the failure obtained when accessing telegram web.
+ TelegramWebFailure *string `json:"telegram_web_failure"`
+
+ // TelegramWebSeenTransparentHTTPProxy is set to true if we've seen a transparent HTTP proxy.
+ TelegramWebSeenTransparentHTTPProxy bool `json:"telegram_web_transparent_http_proxy"`
+
+ // TelegramWebStatus is the status of telegram web.
+ TelegramWebStatus string `json:"telegram_web_status"`
+
+ // TelegramWebUnexpectedASN means we saw an unexpected ASN.
+ TelegramWebUnexpectedASN bool `json:"telegram_web_unexpected_asn"`
+
+ // curIndex is the current index we're using
+ curIndex int64
+
+ // mu provides mutual exclusion.
+ mu sync.Mutex
+}
+
+// NewTestKeys creates new telegram TestKeys.
+func NewTestKeys() *TestKeys {
+ return &TestKeys{
+ Submeasurements: []*TestKeysSubMeasurement{},
+ NetworkEvents: []*model.ArchivalNetworkEvent{},
+ Queries: []*model.ArchivalDNSLookupResult{},
+ Requests: []*model.ArchivalHTTPRequestResult{},
+ TCPConnect: []*model.ArchivalTCPConnectResult{},
+ TLSHandshakes: []*model.ArchivalTLSOrQUICHandshakeResult{},
+ TelegramHTTPBlocking: true,
+ TelegramTCPBlocking: true,
+ TelegramWebBogon: false,
+ TelegramWebFailure: nil,
+ TelegramWebSeenTransparentHTTPProxy: false,
+ TelegramWebStatus: "ok",
+ TelegramWebUnexpectedASN: false,
+ curIndex: 0,
+ mu: sync.Mutex{},
+ }
+}
+
+// newIndex returns a new index for a sub-experiment.
+func (tk *TestKeys) newIndex() int64 {
+ tk.mu.Lock()
+ tk.curIndex++
+ idx := tk.curIndex
+ tk.mu.Unlock()
+ return idx
+}
+
+// addTCPConnectResult adds an ArchivalTCPConnectResult to the test keys.
+func (tk *TestKeys) addTCPConnectResult(ev *model.ArchivalTCPConnectResult) {
+ tk.mu.Lock()
+ tk.TCPConnect = append(tk.TCPConnect, ev)
+ tk.mu.Unlock()
+}
+
+// addHTTPRequestResult adds an ArchivalHTTPRequestResult to the test keys.
+func (tk *TestKeys) addHTTPRequestResult(ev *model.ArchivalHTTPRequestResult) {
+ tk.mu.Lock()
+ tk.Requests = append(tk.Requests, ev)
+ tk.mu.Unlock()
+}
+
+// addNetworkEvents adds []*ArchivalNetworkEvent to the test keys.
+func (tk *TestKeys) addNetworkEvents(ev []*model.ArchivalNetworkEvent) {
+ tk.mu.Lock()
+ tk.NetworkEvents = append(tk.NetworkEvents, ev...)
+ tk.mu.Unlock()
+}
+
+// onAccessibleDCTCPIP is called when a DC is accessible at TCP/IP level.
+func (tk *TestKeys) onAccessibleDCTCPIP() {
+ tk.mu.Lock()
+ tk.TelegramTCPBlocking = false
+ tk.mu.Unlock()
+}
+
+// onAccessibleDCHTTP is called when a DC is accessible at HTTP level.
+func (tk *TestKeys) onAccessibleDCHTTP() {
+ tk.mu.Lock()
+ tk.TelegramHTTPBlocking = false
+ tk.mu.Unlock()
+}
+
+// registerSubmeasurement register a submeasurement.
+func (tk *TestKeys) registerSubmeasurement(index int64, input string, smtype string) {
+ tk.mu.Lock()
+ tk.Submeasurements = append(tk.Submeasurements, &TestKeysSubMeasurement{
+ Index: index,
+ Input: input,
+ Type: smtype,
+ })
+ tk.mu.Unlock()
+}
+
+// addDNSLookupResults saves DNS lookup results.
+func (tk *TestKeys) addDNSLookupResults(ev []*model.ArchivalDNSLookupResult) {
+ tk.mu.Lock()
+ tk.Queries = append(tk.Queries, ev...)
+ tk.mu.Unlock()
+}
+
+// addTLSHandshakeResult saves a TLS handshake result.
+func (tk *TestKeys) addTLSHandshakeResult(ev *model.ArchivalTLSOrQUICHandshakeResult) {
+ tk.mu.Lock()
+ tk.TLSHandshakes = append(tk.TLSHandshakes, ev)
+ tk.mu.Unlock()
+}
+
+// onWebFailure informs the test keys that there was a failure when testing telegram web
+func (tk *TestKeys) onWebFailure(err error) {
+ tk.mu.Lock()
+ tk.TelegramWebFailure = measurexlite.NewFailure(err)
+ tk.TelegramWebStatus = "blocked"
+ tk.mu.Unlock()
+}
+
+// onWebRequestFailed informs the test keys that the web request failed.
+func (tk *TestKeys) onWebRequestFailed() {
+ tk.mu.Lock()
+ failure := netxlite.FailureHTTPRequestFailed
+ tk.TelegramWebFailure = &failure
+ tk.TelegramWebStatus = "blocked"
+ tk.mu.Unlock()
+}
+
+// onUnexpectedWebRedirect informs the test keys that we've seen an unexpected redirect URL.
+func (tk *TestKeys) onUnexpectedWebRedirect() {
+ tk.mu.Lock()
+ failure := netxlite.FailureHTTPUnexpectedRedirectError
+ tk.TelegramWebFailure = &failure
+ tk.TelegramWebStatus = "blocked"
+ tk.mu.Unlock()
+}
+
+// onWebMissingTitle informs the test keys the the web title is missing.
+func (tk *TestKeys) onWebMissingTitle() {
+ tk.mu.Lock()
+ failure := netxlite.FailureTelegramMissingTitleError
+ tk.TelegramWebFailure = &failure
+ tk.TelegramWebStatus = "blocked"
+ tk.mu.Unlock()
+}
+
+// onWebTransparentHTTPProxy informs the test keys that we've seen a transparent HTTP proxy.
+func (tk *TestKeys) onWebTransparentHTTPProxy() {
+ tk.mu.Lock()
+ tk.TelegramWebSeenTransparentHTTPProxy = true
+ tk.mu.Unlock()
+}
+
+// onResolvedBogonAddr informs the test keys that we've resolved a bogon addr.
+func (tk *TestKeys) onResolvedBogonAddr() {
+ tk.mu.Lock()
+ tk.TelegramWebBogon = true
+ tk.mu.Unlock()
+}
+
+// onUnexpectedASN means we saw an IP address in an unexpected ASN.
+func (tk *TestKeys) onUnexpectedASN() {
+ tk.mu.Lock()
+ tk.TelegramWebUnexpectedASN = true
+ tk.mu.Unlock()
+}
diff --git a/internal/experiment/telegram/web.go b/internal/experiment/telegram/web.go
new file mode 100644
index 0000000..a86d694
--- /dev/null
+++ b/internal/experiment/telegram/web.go
@@ -0,0 +1,244 @@
+package telegram
+
+//
+// Telegram Web
+//
+
+import (
+ "bytes"
+ "context"
+ "crypto/tls"
+ "net"
+ "sync"
+ "time"
+
+ "github.com/ooni/probe-cli/v3/internal/engine/geolocate"
+ "github.com/ooni/probe-cli/v3/internal/measurexlite"
+ "github.com/ooni/probe-cli/v3/internal/model"
+ "github.com/ooni/probe-cli/v3/internal/netxlite"
+ "github.com/ooni/probe-cli/v3/internal/runtimex"
+)
+
+// webDomain is the domain used by Telegram's web interface.
+const webDomain = "web.telegram.org"
+
+// startMeasuringWeb starts goroutines for measuring web.telegram.org.
+func (mx *Measurer) startMeasuringWeb(ctx context.Context,
+ wg *sync.WaitGroup, logger model.Logger, zeroTime time.Time, tk *TestKeys) {
+
+ index := tk.newIndex()
+ tk.registerSubmeasurement(index, webDomain, "dnslookup")
+ ol := measurexlite.NewOperationLogger(logger, "DNSLookup %s", webDomain)
+
+ const dnsTimeout = 7 * time.Second
+ dnsCtx, cancel := context.WithTimeout(ctx, dnsTimeout)
+ defer cancel()
+ reso := measurexlite.NewTrustedRecursiveResolver2(index, logger, zeroTime)
+ addrs, err := reso.LookupHost(dnsCtx, webDomain)
+ tk.addDNSLookupResults(reso.DNSLookupResults())
+ ol.Stop(err)
+
+ if err != nil {
+ return
+ }
+ runtimex.PanicIfTrue(len(addrs) < 1, "LookupHost returned no addresses")
+
+ for _, addr := range addrs {
+ if net.ParseIP(addr) == nil && netxlite.IsBogon(addr) {
+ tk.onResolvedBogonAddr()
+ continue
+ }
+ asn, _, _ := geolocate.LookupASN(addr)
+ if asn != 0 && asn != telegramASN {
+ tk.onUnexpectedASN()
+ // fallthrough and test the IP address nonetheless
+ }
+ wg.Add(2)
+ go mx.measureWebEndpointHTTP(ctx, wg, logger, zeroTime, tk, addr)
+ go mx.measureWebEndpointHTTPS(ctx, wg, logger, zeroTime, tk, addr)
+ }
+}
+
+// telegramASN is telegram's ASN
+const telegramASN = 62041
+
+// measureWebEndpointHTTP measures a web.telegram.org endpoint using HTTP
+func (mx *Measurer) measureWebEndpointHTTP(ctx context.Context, wg *sync.WaitGroup,
+ logger model.Logger, zeroTime time.Time, tk *TestKeys, address string) {
+
+ const webTimeout = 7 * time.Second
+ ctx, cancel := context.WithTimeout(ctx, webTimeout)
+ defer cancel()
+
+ defer wg.Done() // synchronize with the controller
+ weburl := measurexlite.NewURL("http", webDomain, "", "/")
+ endpoint := net.JoinHostPort(address, "80")
+ ol := measurexlite.NewOperationLogger(logger, "GET %s @ %s", weburl.String(), endpoint)
+ index := tk.newIndex()
+ tk.registerSubmeasurement(index, endpoint, "web_http")
+
+ // 1. establish a TCP connection with the endpoint
+ dialer := measurexlite.NewDialer(index, logger, zeroTime)
+ conn, err := dialer.DialContext(ctx, "tcp", endpoint)
+ tk.addTCPConnectResult(dialer.TCPConnectResult())
+ if err != nil {
+ switch err.Error() {
+ case netxlite.FailureHostUnreachable: // happens when IPv6 not available
+ case netxlite.FailureNetworkUnreachable: // ditto
+ default:
+ tk.onWebFailure(err)
+ }
+ ol.Stop(err)
+ return
+ }
+ defer conn.Close()
+
+ // 2. fetch the webpage at the endpoint
+ const networkEventsBuffer = 16
+ cw := measurexlite.WrapConn(index, zeroTime, conn, networkEventsBuffer)
+ req, err := measurexlite.NewHTTPRequestWithContext(ctx, "GET", weburl.String(), nil)
+ runtimex.PanicOnError(err, "measurexlite.NewHTTPRequestWithContext failed unexpectedly")
+ req.Host = webDomain
+ const maxBodySnapshotSize = 1 << 17
+ txp := measurexlite.NewHTTPBodyReaderTransport(index, logger, zeroTime, maxBodySnapshotSize)
+ resp, body, _, err := txp.BodyRoundTripWithTCPConn(req, cw)
+ tk.addHTTPRequestResult(txp.HTTPRequestResult())
+ tk.addNetworkEvents(cw.NetworkEvents())
+ if err != nil {
+ tk.onWebFailure(err)
+ ol.Stop(err)
+ return
+ }
+ resp.Body.Close()
+
+ // 3. we expect to see a redirect (301 or 307) to https://web.telegram.org/
+ if resp.StatusCode == 301 || resp.StatusCode == 307 {
+ location, err := resp.Location()
+ if err != nil {
+ tk.onWebFailure(err)
+ ol.Stop(err)
+ return
+ }
+ if location.Host != webDomain || location.Scheme != "https" {
+ tk.onUnexpectedWebRedirect()
+ ol.StopString(netxlite.FailureHTTPUnexpectedRedirectError)
+ return
+ }
+ ol.Stop(nil) // this is what we __exactly__ expect to occur
+ return
+ }
+
+ // 4. transparent proxy case
+ //
+ // Technically speaking, the following should not happen. We know the HTTP version
+ // of web.telegram.org just redirects to the HTTPS version.
+ //
+ // So, if this happens, it should be a transparent proxy, right?
+ tk.onWebTransparentHTTPProxy()
+
+ // 4.1. we expect to see a successful reply
+ if resp.StatusCode != 200 {
+ tk.onWebRequestFailed()
+ ol.StopString(netxlite.FailureHTTPRequestFailed)
+ return
+ }
+
+ // 4.2. we expect to see the telegram web title
+ if !webCheckForTitle(body) {
+ tk.onWebMissingTitle()
+ ol.StopString(netxlite.FailureTelegramMissingTitleError)
+ return
+ }
+
+ // 4.3. we're mostly good here
+ ol.Stop(nil)
+}
+
+// measureWebEndpointHTTPS measures a web.telegram.org endpoint using HTTPS
+func (mx *Measurer) measureWebEndpointHTTPS(ctx context.Context, wg *sync.WaitGroup,
+ logger model.Logger, zeroTime time.Time, tk *TestKeys, address string) {
+
+ const webTimeout = 7 * time.Second
+ ctx, cancel := context.WithTimeout(ctx, webTimeout)
+ defer cancel()
+
+ defer wg.Done() // synchronize with the controller
+ weburl := measurexlite.NewURL("https", webDomain, "", "/")
+ endpoint := net.JoinHostPort(address, "443")
+ ol := measurexlite.NewOperationLogger(logger, "GET %s @ %s", weburl.String(), endpoint)
+ index := tk.newIndex()
+ tk.registerSubmeasurement(index, endpoint, "web_https")
+
+ // 1. establish a TCP connection with the endpoint
+ dialer := measurexlite.NewDialer(index, logger, zeroTime)
+ conn, err := dialer.DialContext(ctx, "tcp", endpoint)
+ tk.addTCPConnectResult(dialer.TCPConnectResult())
+ if err != nil {
+ switch err.Error() {
+ case netxlite.FailureHostUnreachable: // happens when IPv6 not available
+ case netxlite.FailureNetworkUnreachable: // ditto
+ default:
+ tk.onWebFailure(err)
+ }
+ ol.Stop(err)
+ return
+ }
+ defer conn.Close()
+
+ // 2. perform TLS handshake with the endpoint
+ const networkEventsBuffer = 64
+ cw := measurexlite.WrapConn(index, zeroTime, conn, networkEventsBuffer)
+ thx := measurexlite.NewTLSHandshaker(index, logger, zeroTime)
+ config := &tls.Config{
+ NextProtos: []string{"h2", "http/1.1"},
+ RootCAs: netxlite.NewDefaultCertPool(),
+ ServerName: webDomain,
+ }
+ tlsConn, err := thx.Handshake(ctx, cw, config)
+ tk.addTLSHandshakeResult(thx.TLSHandshakeResult())
+ if err != nil {
+ tk.onWebFailure(err)
+ ol.Stop(err)
+ return
+ }
+ defer tlsConn.Close()
+
+ // 3. fetch the webpage at the endpoint
+ req, err := measurexlite.NewHTTPRequestWithContext(ctx, "GET", weburl.String(), nil)
+ runtimex.PanicOnError(err, "measurexlite.NewHTTPRequestWithContext failed unexpectedly")
+ req.Host = webDomain
+ const maxBodySnapshotSize = 1 << 17
+ txp := measurexlite.NewHTTPBodyReaderTransport(index, logger, zeroTime, maxBodySnapshotSize)
+ resp, body, _, err := txp.BodyRoundTripWithTLSConn(req, tlsConn)
+ tk.addHTTPRequestResult(txp.HTTPRequestResult())
+ tk.addNetworkEvents(cw.NetworkEvents())
+ if err != nil {
+ tk.onWebFailure(err)
+ ol.Stop(err)
+ return
+ }
+ resp.Body.Close()
+
+ // 4. we expect to see a successful reply
+ if resp.StatusCode != 200 {
+ tk.onWebRequestFailed()
+ ol.StopString(netxlite.FailureHTTPRequestFailed)
+ return
+ }
+
+ // 5. we expect to see the telegram web title
+ if !webCheckForTitle(body) {
+ tk.onWebMissingTitle()
+ ol.StopString(netxlite.FailureTelegramMissingTitleError)
+ return
+ }
+
+ // 6. it seems we're all good
+ ol.Stop(nil)
+}
+
+// webCheckForTitle ensures the webpage contains the expected title
+func webCheckForTitle(body []byte) bool {
+ title := []byte(`<title>Telegram Web</title>`)
+ return bytes.Contains(body, title)
+}
diff --git a/internal/measurexlite/archival.go b/internal/measurexlite/archival.go
new file mode 100644
index 0000000..c7797ff
--- /dev/null
+++ b/internal/measurexlite/archival.go
@@ -0,0 +1,320 @@
+package measurexlite
+
+//
+// Generates the OONI archival data format.
+//
+// See https://github.com/ooni/spec/tree/master/data-formats.
+//
+
+import (
+ "crypto/tls"
+ "crypto/x509"
+ "errors"
+ "log"
+ "math"
+ "net"
+ "net/http"
+ "strconv"
+ "time"
+
+ "github.com/miekg/dns"
+ "github.com/ooni/probe-cli/v3/internal/engine/geolocate"
+ "github.com/ooni/probe-cli/v3/internal/model"
+ "github.com/ooni/probe-cli/v3/internal/netxlite"
+)
+
+// newArchivalDNSLookupResultFromLookupHost creates a new ArchivalDNSLookupResult
+// entry from the results of Resolver.LookupHost.
+//
+// The returned lookup result pretends the resolver issues an ANY query.
+//
+// You SHOULD use this factory for getaddrinfo and for encrypted DNS resolvers, for
+// which it does not matter much to see the original messages.
+func newArchivalDNSLookupResultFromLookupHost(index int64, reso model.Resolver, started time.Duration,
+ domain string, addrs []string, err error, finished time.Duration) (out *model.ArchivalDNSLookupResult) {
+ out = &model.ArchivalDNSLookupResult{
+ Answers: archivalAnswersFromAddrs(addrs),
+ Engine: reso.Network(),
+ Failure: NewFailure(err),
+ Hostname: domain,
+ QueryType: "ANY",
+ ResolverHostname: nil,
+ ResolverPort: nil,
+ ResolverAddress: reso.Address(),
+ T: finished.Seconds(),
+ TransactionID: index,
+ }
+ if out.Failure == nil && len(out.Answers) <= 0 {
+ log.Printf("BUG: newArchivalDNSLookupResultFromLookupHost: no failure and no addresses")
+ failure := netxlite.FailureProbeBug
+ out.Failure = &failure
+ }
+ return
+}
+
+// archivalAnswersFromAddrs constructs a []model.ArchivalDNSAnswer from a list of addresses.
+func archivalAnswersFromAddrs(addrs []string) (out []model.ArchivalDNSAnswer) {
+ for _, addr := range addrs {
+ ipv6, err := netxlite.IsIPv6(addr)
+ if err != nil {
+ log.Printf("BUG: newArchivalDNSLookupResultFromLookupHost: invalid IPAddr: %s", addrs)
+ continue
+ }
+ asn, org, _ := geolocate.LookupASN(addr)
+ switch ipv6 {
+ case false:
+ out = append(out, model.ArchivalDNSAnswer{
+ ASN: int64(asn),
+ ASOrgName: org,
+ AnswerType: "A",
+ Hostname: "",
+ IPv4: addr,
+ IPv6: "",
+ TTL: nil,
+ })
+ case true:
+ out = append(out, model.ArchivalDNSAnswer{
+ ASN: int64(asn),
+ ASOrgName: org,
+ AnswerType: "AAAA",
+ Hostname: "",
+ IPv4: "",
+ IPv6: addr,
+ TTL: nil,
+ })
+ }
+ }
+ return
+}
+
+// newArchivalDNSLookupResultFromRoundTrip constructs an ArchivalDNSLookupResult
+// from the information available right after a round trip.
+func newArchivalDNSLookupResultFromRoundTrip(index int64,
+ txp model.DNSTransport, started time.Duration, query model.DNSQuery, resp model.DNSResponse,
+ err error, finished time.Duration) (out *model.ArchivalDNSLookupResult) {
+ out = &model.ArchivalDNSLookupResult{
+ Answers: []model.ArchivalDNSAnswer{},
+ Engine: txp.Network(),
+ Failure: NewFailure(err),
+ Hostname: query.Domain(),
+ QueryType: dns.TypeToString[query.Type()],
+ ResolverHostname: nil,
+ ResolverPort: nil,
+ ResolverAddress: txp.Address(),
+ T: finished.Seconds(),
+ TransactionID: index,
+ }
+ if resp != nil {
+ addrs, err := resp.DecodeLookupHost()
+ if err != nil {
+ out.Failure = NewFailure(err)
+ } else {
+ out.Answers = archivalAnswersFromAddrs(addrs)
+ }
+ }
+ if out.Failure == nil && len(out.Answers) <= 0 {
+ log.Printf("BUG: newArchivalDNSLookupResultFromRoundTrip: no failure and no addresses")
+ failure := netxlite.FailureProbeBug
+ out.Failure = &failure
+ }
+ return out
+}
+
+// newArchivalTCPConnectResult generates a model.ArchivalTCPConnectResult
+// from the available information right after connect returns.
+func newArchivalTCPConnectResult(index int64, started time.Duration, address string,
+ conn net.Conn, err error, finished time.Duration) *model.ArchivalTCPConnectResult {
+ ip, port := archivalSplitHostPort(address)
+ return &model.ArchivalTCPConnectResult{
+ IP: ip,
+ Port: archivalPortToString(port),
+ Status: model.ArchivalTCPConnectStatus{
+ Blocked: nil,
+ Failure: NewFailure(err),
+ Success: err == nil,
+ },
+ T: finished.Seconds(),
+ TransactionID: index,
+ }
+}
+
+// archivalSplitHostPort is like net.SplitHostPort but does not return an error.
+func archivalSplitHostPort(endpoint string) (string, string) {
+ addr, port, err := net.SplitHostPort(endpoint)
+ if err != nil {
+ log.Printf("BUG: archivalSplitHostPort: invalid endpoint: %s", endpoint)
+ }
+ return addr, port
+}
+
+// archivalPortToString is like strconv.Atoi but does not return an error.
+func archivalPortToString(sport string) int {
+ port, err := strconv.Atoi(sport)
+ if err != nil || port < 0 || port > math.MaxUint16 {
+ log.Printf("BUG: archivalStrconvAtoi: invalid port: %s", sport)
+ }
+ return port
+}
+
+// newArchivalTLSOrQUICHandshakeResult generates a model.ArchivalTLSOrQUICHandshakeResult
+// from the available information right after the TLS handshake returns.
+func newArchivalTLSOrQUICHandshakeResult(
+ index int64, started time.Duration, address string, config *tls.Config,
+ state tls.ConnectionState, err error, finished time.Duration) *model.ArchivalTLSOrQUICHandshakeResult {
+ return &model.ArchivalTLSOrQUICHandshakeResult{
+ Address: address,
+ CipherSuite: tls.CipherSuiteName(state.CipherSuite),
+ Failure: NewFailure(err),
+ NegotiatedProtocol: state.NegotiatedProtocol,
+ NoTLSVerify: config.InsecureSkipVerify,
+ PeerCertificates: newArchivalTLSPeerCertificates(state, err),
+ ServerName: config.ServerName,
+ T: finished.Seconds(),
+ TLSVersion: netxlite.TLSVersionString(state.Version),
+ TransactionID: index,
+ }
+}
+
+// newArchivalTLSPeerCertificates extracts the certificates either from
+// the list of certificates or from the error that occurred.
+func newArchivalTLSPeerCertificates(
+ state tls.ConnectionState, err error) (out []model.ArchivalBinaryData) {
+ var x509HostnameError x509.HostnameError
+ if errors.As(err, &x509HostnameError) {
+ // Test case: https://wrong.host.badssl.com/
+ out = append(out, model.NewArchivalBinaryData(x509HostnameError.Certificate.Raw))
+ return
+ }
+ var x509UnknownAuthorityError x509.UnknownAuthorityError
+ if errors.As(err, &x509UnknownAuthorityError) {
+ // Test case: https://self-signed.badssl.com/. This error has
+ // never been among the ones returned by MK.
+ out = append(out, model.NewArchivalBinaryData(x509UnknownAuthorityError.Cert.Raw))
+ return
+ }
+ var x509CertificateInvalidError x509.CertificateInvalidError
+ if errors.As(err, &x509CertificateInvalidError) {
+ // Test case: https://expired.badssl.com/
+ out = append(out, model.NewArchivalBinaryData(x509CertificateInvalidError.Cert.Raw))
+ return
+ }
+ for _, cert := range state.PeerCertificates {
+ out = append(out, model.NewArchivalBinaryData(cert.Raw))
+ }
+ return
+}
+
+// newArchivalNetworkEvent creates a new model.ArchivalNetworkEvent.
+func newArchivalNetworkEvent(index int64, started time.Duration, operation string, network string,
+ address string, count int, err error, finished time.Duration) *model.ArchivalNetworkEvent {
+ return &model.ArchivalNetworkEvent{
+ Address: address,
+ Failure: NewFailure(err),
+ NumBytes: int64(count),
+ Operation: operation,
+ Proto: network,
+ T: finished.Seconds(),
+ Tags: []string{},
+ TransactionID: index,
+ }
+}
+
+// newArchivalHTTPRequestResult creates a new model.ArchivalHTTPRequestResult
+// from the information available right after the HTTP round trip.
+func newArchivalHTTPRequestResult(index int64, started time.Duration, txp model.HTTPTransport,
+ remoteAddr string, req *http.Request, resp *http.Response, body []byte, truncated bool,
+ err error, finished time.Duration) *model.ArchivalHTTPRequestResult {
+ requestHeaders := req.Header.Clone()
+ if req.Host != "" {
+ requestHeaders.Set("Host", req.Host) // hint from NewHTTPRequestWithContext
+ }
+ out := &model.ArchivalHTTPRequestResult{
+ Failure: NewFailure(err),
+ Request: model.ArchivalHTTPRequest{
+ Body: model.ArchivalMaybeBinaryData{},
+ BodyIsTruncated: false,
+ HeadersList: archivalHeadersList(requestHeaders),
+ Headers: archivalHeadersMap(requestHeaders),
+ Method: req.Method,
+ Tor: model.ArchivalHTTPTor{},
+ Transport: txp.Network(),
+ URL: req.URL.String(),
+ },
+ Response: model.ArchivalHTTPResponse{
+ Body: model.ArchivalMaybeBinaryData{}, // to be set later
+ BodyIsTruncated: truncated,
+ Code: archivalMaybeResponseCode(resp),
+ HeadersList: archivalMaybeHeadersList(resp),
+ Headers: archivalMaybeHeadersMap(resp),
+ Locations: archivalMaybeLocations(resp),
+ },
+ T: finished.Seconds(),
+ TransactionID: index,
+ }
+ if len(body) > 0 {
+ out.Response.Body.Value = string(body)
+ }
+ return out
+}
+
+// archivalHeadersList is an internal factory for creating a list of headers
+func archivalHeadersList(hdrs http.Header) (out []model.ArchivalHTTPHeader) {
+ for key, values := range hdrs {
+ for _, value := range values {
+ value := model.ArchivalMaybeBinaryData{Value: value}
+ out = append(out, model.ArchivalHTTPHeader{
+ Key: key,
+ Value: value,
+ })
+ }
+ }
+ return
+}
+
+// archivalHeadersMap is an internal factory to represents headers as a map.
+//
+// Caveat: with the map representation we can only represent a single
+// value for every key. Hence the list representation, which is more accurate.
+func archivalHeadersMap(hdrs http.Header) (out map[string]model.ArchivalMaybeBinaryData) {
+ out = make(map[string]model.ArchivalMaybeBinaryData)
+ for key, values := range hdrs {
+ if len(values) < 1 {
+ continue
+ }
+ value := model.ArchivalMaybeBinaryData{Value: values[0]}
+ out[key] = value
+ }
+ return
+}
+
+// archivalMaybeResponseCode conditionally extracts the response code.
+func archivalMaybeResponseCode(resp *http.Response) (code int64) {
+ if resp != nil {
+ code = int64(resp.StatusCode)
+ }
+ return
+}
+
+// archivalMaybeHeadersList conditionally extract headers and represents them as a list.
+func archivalMaybeHeadersList(resp *http.Response) (out []model.ArchivalHTTPHeader) {
+ if resp != nil {
+ out = archivalHeadersList(resp.Header)
+ }
+ return
+}
+
+// archivalMaybeHeadersList conditionally extract headers and represents them as a map.
+func archivalMaybeHeadersMap(resp *http.Response) (out map[string]model.ArchivalMaybeBinaryData) {
+ if resp != nil {
+ out = archivalHeadersMap(resp.Header)
+ }
+ return
+}
+
+// archivalMaybeLocations conditionally extracts the values of the Location header.
+func archivalMaybeLocations(resp *http.Response) (out []string) {
+ if resp != nil {
+ out = resp.Header.Values("Location")
+ }
+ return
+}
diff --git a/internal/measurexlite/conn.go b/internal/measurexlite/conn.go
new file mode 100644
index 0000000..2a8a8bd
--- /dev/null
+++ b/internal/measurexlite/conn.go
@@ -0,0 +1,117 @@
+package measurexlite
+
+//
+// Measuring and managing net.Conn.
+//
+
+import (
+ "net"
+ "time"
+
+ "github.com/ooni/probe-cli/v3/internal/model"
+ "github.com/ooni/probe-cli/v3/internal/netxlite"
+)
+
+// MaybeClose is a convenience function for closing a conn
+// only when such a conn isn't nil.
+func MaybeClose(closer net.Conn) (err error) {
+ if closer != nil {
+ err = closer.Close()
+ }
+ return
+}
+
+// WrapConn wraps a connection with a Conn instance that collects
+// read and write events occurring during the conn's lifetime.
+//
+// Arguments:
+//
+// - index is the index of this measurement;
+//
+// - zeroTime is when we started measuring;
+//
+// - conn is the conn to wrap;
+//
+// - buffer controls how many I/O events we'll buffer into the internal
+// channel before starting to discard subsequent I/O events.
+//
+// You own the returned conn and should close it when done. Calling close
+// will just forward the call to the wrapped conn.
+func WrapConn(index int64, zeroTime time.Time, conn net.Conn, buffer int) *Conn {
+ return &Conn{
+ Conn: conn,
+ index: index,
+ out: make(chan *model.ArchivalNetworkEvent, buffer),
+ zeroTime: zeroTime,
+ }
+}
+
+// Conn is a wrapper for net.Conn saving network events.
+//
+// The zero-value struct is invalid, please use WrapConn to instantiate.
+type Conn struct {
+ // Conn is the underlying conn.
+ net.Conn
+
+ // index is the index of this measurement.
+ index int64
+
+ // out is where we post the network events
+ out chan *model.ArchivalNetworkEvent
+
+ // zeroTime is the time considered as zero.
+ zeroTime time.Time
+}
+
+// Read implements net.Conn.Read and saves network events.
+//
+// We use an internal channel for delivering network events. The WrapConn
+// constructor specifies the channel's buffer. When the buffer is full, this
+// function will just discard events.
+func (c *Conn) Read(b []byte) (int, error) {
+ network := c.RemoteAddr().Network()
+ addr := c.RemoteAddr().String()
+ started := time.Since(c.zeroTime)
+ count, err := c.Conn.Read(b)
+ finished := time.Since(c.zeroTime)
+ select {
+ case c.out <- newArchivalNetworkEvent(
+ c.index, started, netxlite.ReadOperation, network, addr, count, err, finished):
+ default: // buffer is full
+ }
+ return count, err
+}
+
+// Write implements net.Conn.Write and saves network events.
+//
+// We use an internal channel for delivering network events. The WrapConn
+// constructor specifies the channel's buffer. When the buffer is full, this
+// function will just discard events.
+func (c *Conn) Write(b []byte) (int, error) {
+ network := c.RemoteAddr().Network()
+ addr := c.RemoteAddr().String()
+ started := time.Since(c.zeroTime)
+ count, err := c.Conn.Write(b)
+ finished := time.Since(c.zeroTime)
+ select {
+ case c.out <- newArchivalNetworkEvent(
+ c.index, started, netxlite.WriteOperation, network, addr, count, err, finished):
+ default: // buffer is full
+ }
+ return count, err
+}
+
+// NetworkEvents reads all the events collected so far.
+//
+// This function will just drain the internal channel used for delivering
+// events until there is no more event to read.
+func (c *Conn) NetworkEvents() (out []*model.ArchivalNetworkEvent) {
+ for {
+ select {
+ case ev := <-c.out:
+ out = append(out, ev)
+ default:
+ return // we drained the channel
+ }
+ }
+}
diff --git a/internal/measurexlite/dialer.go b/internal/measurexlite/dialer.go
new file mode 100644
index 0000000..58430a5
--- /dev/null
+++ b/internal/measurexlite/dialer.go
@@ -0,0 +1,78 @@
+package measurexlite
+
+//
+// Dialing and measuring dials
+//
+
+import (
+ "context"
+ "net"
+ "time"
+
+ "github.com/ooni/probe-cli/v3/internal/model"
+ "github.com/ooni/probe-cli/v3/internal/netxlite"
+)
+
+// Dialer dials network connections. This Dialer is meant to measure connecting
+// to a remote TCP/UDP endpoint and does not include DNS lookups.
+//
+// The zero-value struct is invalid, please use NewDialer to create a new instance.
+//
+// We internally use a channel with a single position in the buffer to tracking
+// dialing events. The intended usage is that you will drain this channel calling
+// TCPConnectResult after each DialContext has terminated.
+//
+// Because of this design, the common use case is that you create a Dialer and
+// just use it once for extracting the dial results.
+type Dialer struct {
+ index int64
+ logger model.Logger
+ m chan *model.ArchivalTCPConnectResult
+ zeroTime time.Time
+}
+
+// NewDialer creates a new dialer for performing measurements.
+//
+// Arguments:
+//
+// - index is the index of this measurement;
+//
+// - logger is the logger to use;
+//
+// - zeroTime is the time to use as reference for generating observations.
+func NewDialer(index int64, logger model.Logger, zeroTime time.Time) *Dialer {
+ return &Dialer{
+ index: index,
+ logger: logger,
+ m: make(chan *model.ArchivalTCPConnectResult, 1),
+ zeroTime: zeroTime,
+ }
+}
+
+// DialContext establishes a network connection and saves the related observation
+// inside the dialer. You can extract the observation using TCPConnectResult.
+//
+// Use the context to control the maximum dialing time. The underlying dialer used
+// for dialing will additionally impose a reasonable watchdog timeout.
+//
+// This function assumes that address contains an IPv4/IPv6 address and a port. When
+// addrss contains a domain name, this function will immediately return an error.
+func (d *Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+ dialer := netxlite.NewDialerWithoutResolver(d.logger)
+ started := time.Now()
+ conn, err := dialer.DialContext(ctx, "tcp", address)
+ finished := time.Now()
+ select {
+ case d.m <- newArchivalTCPConnectResult(
+ d.index, started.Sub(d.zeroTime), address, conn, err, finished.Sub(d.zeroTime)):
+ default: // buffer is full
+ }
+ return conn, err
+}
+
+// TCPConnectResult returns the saved observation. This function MUST be called right
+// after DialContext has returned. Calling this function before that blocks until a
+// TCPConnectResult becomes available.
+func (d *Dialer) TCPConnectResult() *model.ArchivalTCPConnectResult {
+ return <-d.m
+}
diff --git a/internal/measurexlite/doc.go b/internal/measurexlite/doc.go
new file mode 100644
index 0000000..73a89e4
--- /dev/null
+++ b/internal/measurexlite/doc.go
@@ -0,0 +1,6 @@
+// Package measurexlite contains support code for writing experiments.
+//
+// This package encourages a measurement tactic we call "step-by-step", where
+// you are supposed to perform each measurement step in isolation and immediately
+// record its results and take action depending on that.
+package measurexlite
diff --git a/internal/measurexlite/failure.go b/internal/measurexlite/failure.go
new file mode 100644
index 0000000..341c858
--- /dev/null
+++ b/internal/measurexlite/failure.go
@@ -0,0 +1,29 @@
+package measurexlite
+
+//
+// Mapping errors to OONI failures.
+//
+// See https://github.com/ooni/spec/blob/master/data-formats/df-007-errors.md.
+//
+
+import (
+ "errors"
+
+ "github.com/ooni/probe-cli/v3/internal/netxlite"
+)
+
+// NewFailure converts an error to a OONI failure. If the given error is
+// nil, we return nil to the caller. If the given error is not nil and has
+// not been wrapped, we use netxlite.NewTopLevelGenericErrWrapper. Hence,
+// the returned string will always be an OONI failure.
+func NewFailure(err error) (str *string) {
+ if err != nil {
+ var wrapper *netxlite.ErrWrapper
+ if !errors.As(err, &wrapper) {
+ err = netxlite.NewTopLevelGenericErrWrapper(err)
+ }
+ v := err.Error()
+ str = &v
+ }
+ return
+}
diff --git a/internal/measurexlite/http.go b/internal/measurexlite/http.go
new file mode 100644
index 0000000..eed6ad4
--- /dev/null
+++ b/internal/measurexlite/http.go
@@ -0,0 +1,140 @@
+package measurexlite
+
+//
+// Performing and measuring HTTP transports.
+//
+
+import (
+ "context"
+ "io"
+ "net"
+ "net/http"
+ "time"
+
+ "github.com/ooni/probe-cli/v3/internal/model"
+ "github.com/ooni/probe-cli/v3/internal/netxlite"
+)
+
+// HTTPBodyReaderTransport is like HTTPTransport each that the round trip
+// includes reading a configurable body snapshot.
+//
+// This struct will also save events occurring while doing that.
+//
+// The zero-value struct is invalid, please use NewHTTPBodyReaderTransport
+// to create a valid instance of this type.
+//
+// We internally use a single-buffer channel to record the result of
+// performing the round trip and reading the body snapshot. The intended
+// usage is that you will call BodyRoundTrip to perform the round trip
+// and read the body. Then, you will call HTTPRequestResult to obtain the
+// event summarizing what happened at HTTP level.
+type HTTPBodyReaderTransport struct {
+ index int64
+ logger model.Logger
+ m chan *model.ArchivalHTTPRequestResult
+ maxBodySnapshotSize int64
+ zeroTime time.Time
+}
+
+// NewHTTPBodyReaderTransport creates a new HTTPBodyReaderTransport instance.
+func NewHTTPBodyReaderTransport(index int64, logger model.Logger,
+ zeroTime time.Time, maxBodySnapshotSize int64) *HTTPBodyReaderTransport {
+ return &HTTPBodyReaderTransport{
+ index: index,
+ logger: logger,
+ m: make(chan *model.ArchivalHTTPRequestResult, 1),
+ maxBodySnapshotSize: maxBodySnapshotSize,
+ zeroTime: zeroTime,
+ }
+}
+
+// BodyRoundTripWithTCPConn performs the HTTP round trip and reads a snapshot of the
+// response body using the given net.Conn for the round trip.
+//
+// Arguments:
+//
+// - req is the HTTP request;
+//
+// - conn is the TCP conn to use.
+//
+// Return values:
+//
+// - the HTTP response (nil on failure);
+//
+// - the response body snapshot (nil/zero-length on failure);
+//
+// - whether the response body is truncated;
+//
+// - the error (nil on success).
+//
+// Note that we don't return an error when the response body is truncated.
+func (txp *HTTPBodyReaderTransport) BodyRoundTripWithTCPConn(
+ req *http.Request, conn net.Conn) (*http.Response, []byte, bool, error) {
+ dialer := netxlite.NewSingleUseDialer(conn)
+ tlsDialer := netxlite.NewNullTLSDialer()
+ child := netxlite.NewHTTPTransport(txp.logger, dialer, tlsDialer)
+ return txp.do(req, child, conn.RemoteAddr().String())
+}
+
+// BodyRoundTripWithTLSConn is like BodyRoundTripWithTCPConn for a TLSConn.
+func (txp *HTTPBodyReaderTransport) BodyRoundTripWithTLSConn(
+ req *http.Request, conn model.TLSConn) (*http.Response, []byte, bool, error) {
+ dialer := netxlite.NewNullDialer()
+ tlsDialer := netxlite.NewSingleUseTLSDialer(conn)
+ child := netxlite.NewHTTPTransport(txp.logger, dialer, tlsDialer)
+ return txp.do(req, child, conn.RemoteAddr().String())
+}
+
+// do is the internal function implements BodyRoundTripWithT{CP,LS}Conn.
+func (txp *HTTPBodyReaderTransport) do(req *http.Request, child model.HTTPTransport,
+ remoteAddr string) (*http.Response, []byte, bool, error) {
+ started := time.Now()
+ resp, err := child.RoundTrip(req)
+ var (
+ body []byte
+ trunc bool
+ )
+ if err == nil {
+ reader := io.LimitReader(resp.Body, txp.maxBodySnapshotSize)
+ body, err = netxlite.ReadAllContext(req.Context(), reader)
+ trunc = int64(len(body)) >= txp.maxBodySnapshotSize
+ }
+ finished := time.Now()
+ select {
+ case txp.m <- newArchivalHTTPRequestResult(txp.index, started.Sub(txp.zeroTime), child,
+ remoteAddr, req, resp, body, trunc, err, finished.Sub(txp.zeroTime)):
+ default: // buffer is full
+ }
+ return resp, body, trunc, err
+}
+
+// HTTPRequestResult returns the saved observation. This function MUST be called right
+// after RoundTripWith{TLS,}Conn has terminated. Calling this function before that blocks
+// until an HTTPRequestResult observation becomes available.
+func (txp *HTTPBodyReaderTransport) HTTPRequestResult() *model.ArchivalHTTPRequestResult {
+ return <-txp.m
+}
+
+// NewHTTPRequestHeaderForMeasuring returns an http.Header where
+// the headers are the ones we use for measuring.
+func NewHTTPRequestHeaderForMeasuring() http.Header {
+ h := http.Header{}
+ h.Set("Accept", model.HTTPHeaderAccept)
+ h.Set("Accept-Language", model.HTTPHeaderAcceptLanguage)
+ h.Set("User-Agent", model.HTTPHeaderUserAgent)
+ return h
+}
+
+// NewHTTPRequestWithContext is a convenience factory for creating
+// a new HTTP request with the typical headers we use when performing
+// measurements already set inside of req.Header.
+func NewHTTPRequestWithContext(ctx context.Context,
+ method, URL string, body io.Reader) (*http.Request, error) {
+ req, err := http.NewRequestWithContext(ctx, method, URL, body)
+ if err != nil {
+ return nil, err
+ }
+ req.Header = NewHTTPRequestHeaderForMeasuring()
+ req.Host = req.URL.Host // mainly to make archival's job easier
+ return req, nil
+}
diff --git a/internal/measurexlite/logger.go b/internal/measurexlite/logger.go
new file mode 100644
index 0000000..04451d7
--- /dev/null
+++ b/internal/measurexlite/logger.go
@@ -0,0 +1,67 @@
+package measurexlite
+
+//
+// Operation logging
+//
+
+import (
+ "fmt"
+ "sync"
+ "time"
+
+ "github.com/ooni/probe-cli/v3/internal/model"
+)
+
+// NewOperationLogger creates a new logger that logs about an in-progress operation.
+func NewOperationLogger(logger model.Logger, format string, v ...interface{}) *OperationLogger {
+ ol := &OperationLogger{
+ sighup: make(chan interface{}),
+ logger: logger,
+ once: &sync.Once{},
+ message: fmt.Sprintf(format, v...),
+ started: time.Now(),
+ wg: &sync.WaitGroup{},
+ }
+ ol.wg.Add(1)
+ go ol.logLoop()
+ return ol
+}
+
+// OperationLogger logs about an in-progress operation.
+type OperationLogger struct {
+ logger model.Logger
+ message string
+ once *sync.Once
+ sighup chan interface{}
+ started time.Time
+ wg *sync.WaitGroup
+}
+
+// logLoop waits for the operation to complete and prints progress.
+func (ol *OperationLogger) logLoop() {
+ defer ol.wg.Done()
+ timer := time.NewTimer(500 * time.Millisecond)
+ defer timer.Stop()
+ select {
+ case <-timer.C:
+ ol.logger.Infof("%s... in progress", ol.message)
+ case <-ol.sighup:
+ // we'll emit directly in stop
+ }
+}
+
+// Stop notifies this OperationLogger that the operation terminated
+// and logs the result of the operation. If the error is not nil,
+// we will use its Error value in the log message. Otherwise, we log "ok".
+func (ol *OperationLogger) Stop(err error) {
+ ol.StopString(model.ErrorToStringOrOK(err))
+}
+
+// StopString is like Stop but takes in input a string rather than an error.
+func (ol *OperationLogger) StopString(message string) {
+ ol.once.Do(func() {
+ close(ol.sighup)
+ ol.wg.Wait()
+ ol.logger.Infof("%s... %s", ol.message, message)
+ })
+}
diff --git a/internal/measurexlite/resolver.go b/internal/measurexlite/resolver.go
new file mode 100644
index 0000000..7dca69e
--- /dev/null
+++ b/internal/measurexlite/resolver.go
@@ -0,0 +1,266 @@
+package measurexlite
+
+//
+// Performing and measuring DNS lookups.
+//
+
+import (
+ "context"
+ "time"
+
+ "github.com/miekg/dns"
+ "github.com/ooni/probe-cli/v3/internal/model"
+ "github.com/ooni/probe-cli/v3/internal/netxlite"
+)
+
+// GetaddrinfoResolver is a resolver using getaddrinfo.
+//
+// The zero-value struct is invalid. Use NewGetaddrinfoResolver to instantiate.
+//
+// Internally, we use a single-buffer channel to deliver the measurement. The
+// intended usage is that you instantiate this struct, use it to collect a single
+// measurement, and then instantiate a new struct if you need to measure again.
+//
+// We'll do out best to actually use getaddrinfo. If compiled with CGO_ENABLED=0,
+// that would not be possible and we'll actually use &net.Resolver{}.
+type GetaddrinfoResolver struct {
+ index int64
+ logger model.Logger
+ m chan *model.ArchivalDNSLookupResult
+ zeroTime time.Time
+}
+
+// NewGetaddrinfoResolver creates a new getaddrinfo-based resolver.
+//
+// Arguments:
+//
+// - index is the index of this measurement;
+//
+// - logger is the logger to use;
+//
+// - zeroTime is the time to use as reference for generating observations.
+func NewGetaddrinfoResolver(index int64, logger model.Logger, zeroTime time.Time) *GetaddrinfoResolver {
+ return &GetaddrinfoResolver{
+ index: index,
+ logger: logger,
+ m: make(chan *model.ArchivalDNSLookupResult, 1),
+ zeroTime: zeroTime,
+ }
+}
+
+// LookupHost implements model.Resolver.LookupHost and saves a measurement.
+func (reso *GetaddrinfoResolver) LookupHost(ctx context.Context, domain string) ([]string, error) {
+ r := netxlite.NewStdlibResolver(reso.logger)
+ started := time.Now()
+ addrs, err := r.LookupHost(ctx, domain)
+ finished := time.Now()
+ select {
+ case reso.m <- newArchivalDNSLookupResultFromLookupHost(reso.index, r, started.Sub(reso.zeroTime),
+ domain, addrs, err, finished.Sub(reso.zeroTime)):
+ default: // buffer is full
+ }
+ return addrs, err
+}
+
+// DNSLookupResult extracts a previously saved measurement. This method will block if
+// called before calling LookupHost, because we use a single-buffer channel to measure.
+func (reso *GetaddrinfoResolver) DNSLookupResult() *model.ArchivalDNSLookupResult {
+ return <-reso.m
+}
+
+// TrustedRecursiveResolver2 (TRR2) emulates Mozilla Firefox's TRR2 mode.
+//
+// The zero-value type is invalid. Construct using NewTrustedRecursiveResolver2.
+//
+// We will first attempt to perform a DNS lookup using DNS over HTTPS (DoH) with a short
+// timeout and fallback to the system resolver in case the DoH resolver fails.
+//
+// As such, this structure allows you to extract a list of measurements that should
+// contain either one or two measurements, depending on what happened.
+//
+// We internally use a single buffer channel to save measurements. If you follow the
+// intended usage patter of calling LookupHost and then extracting observations by
+// calling DNSLookupResults, you will always be fine.
+//
+// The rationale of using this resolver is that of performing a "parasitic"
+// DNS-over-HTTPS observation as part of each measurement.
+type TrustedRecursiveResolver2 struct {
+ index int64
+ logger model.Logger
+ m chan []*model.ArchivalDNSLookupResult
+ zeroTime time.Time
+}
+
+// NewTrustedRecursiveResolver2 creates a new TrustedRecursiveResolver2 instance.
+//
+// Arguments:
+//
+// - index is the index of this measurement;
+//
+// - logger is the logger to use;
+//
+// - zeroTime is the time to use as reference for generating observations.
+func NewTrustedRecursiveResolver2(index int64, logger model.Logger, zeroTime time.Time) *TrustedRecursiveResolver2 {
+ return &TrustedRecursiveResolver2{
+ index: index,
+ logger: logger,
+ m: make(chan []*model.ArchivalDNSLookupResult, 1),
+ zeroTime: zeroTime,
+ }
+}
+
+// LookupHost implements model.Resolver.LookupHost and saves observations.
+func (reso *TrustedRecursiveResolver2) LookupHost(ctx context.Context, domain string) ([]string, error) {
+ out := []*model.ArchivalDNSLookupResult{}
+ const mozillaURL = "https://mozilla.cloudflare-dns.com/dns-query"
+ https := netxlite.NewParallelDNSOverHTTPSResolver(reso.logger, mozillaURL)
+ started := time.Now()
+ addrs, err := reso.timedLookup(ctx, domain, https)
+ finished := time.Now()
+ out = append(out, newArchivalDNSLookupResultFromLookupHost(reso.index, https,
+ started.Sub(reso.zeroTime), domain, addrs, err, finished.Sub(reso.zeroTime)))
+ https.CloseIdleConnections()
+ if err != nil {
+ stdlib := netxlite.NewStdlibResolver(reso.logger)
+ started := time.Now()
+ addrs, err = stdlib.LookupHost(ctx, domain)
+ finished := time.Now()
+ out = append(out, newArchivalDNSLookupResultFromLookupHost(reso.index, stdlib,
+ started.Sub(reso.zeroTime), domain, addrs, err, finished.Sub(reso.zeroTime)))
+ // fallthrough
+ }
+ select {
+ case reso.m <- out:
+ default: // buffer is full
+ }
+ return addrs, err
+}
+
+// timedLookup performs a timed lookup DNS lookup using a child resolver
+func (reso *TrustedRecursiveResolver2) timedLookup(
+ ctx context.Context, domain string, child model.Resolver) ([]string, error) {
+ const timeout = 1500 * time.Millisecond
+ ctx, cancel := context.WithTimeout(ctx, timeout)
+ defer cancel()
+ return child.LookupHost(ctx, domain)
+}
+
+// DNSLookupResult returns the saved observations. You should only call this method
+// right after LookupHost. Calling it before would block until an observation becomes
+// available since we use a single-buffer channel to save the observation.
+func (reso *TrustedRecursiveResolver2) DNSLookupResults() []*model.ArchivalDNSLookupResult {
+ return <-reso.m
+}
+
+// DNSOverUDPResolver is a resolver using DNS-over-UDP.
+//
+// The zero-value struct is invalid. Use NewDNSOverUDPResolver to instantiate.
+//
+// Internally, we use a single-buffer channel to deliver the measurement. The
+// intended usage is that you instantiate this struct, use it to collect a single
+// measurement, and then instantiate a new struct if you need to measure again.
+type DNSOverUDPResolver struct {
+ endpoint string
+ index int64
+ logger model.Logger
+ m map[uint16]chan *model.ArchivalDNSLookupResult
+ zeroTime time.Time
+}
+
+// NewDNSOverUDPResolver creates a new getaddrinfo-based resolver.
+//
+// Arguments:
+//
+// - endpoint is the resolver's UDP endpoint to use (e.g., 8.8.8.8:53);
+//
+// - index is the index of this measurement;
+//
+// - logger is the logger to use;
+//
+// - zeroTime is the time to use as reference for generating observations.
+//
+// Note: this resolver will fail if the endpoint contains a domain name. When your
+// resolver endpoint contains a domain name, you should resolver it to IP addrs
+// first and then you should probably try each IP address.
+func NewDNSOverUDPResolver(
+ endpoint string, index int64, logger model.Logger, zeroTime time.Time) *DNSOverUDPResolver {
+ return &DNSOverUDPResolver{
+ endpoint: endpoint,
+ index: index,
+ logger: logger,
+ m: map[uint16]chan *model.ArchivalDNSLookupResult{
+ dns.TypeA: make(chan *model.ArchivalDNSLookupResult),
+ dns.TypeAAAA: make(chan *model.ArchivalDNSLookupResult),
+ },
+ zeroTime: zeroTime,
+ }
+}
+
+// LookupHost implements model.Resolver.LookupHost and saves a measurement.
+func (reso *DNSOverUDPResolver) LookupHost(ctx context.Context, domain string) ([]string, error) {
+ ach := reso.start(ctx, dns.TypeA, domain)
+ aaaach := reso.start(ctx, dns.TypeAAAA, domain)
+ ares := <-ach
+ aaaares := <-aaaach
+ if ares.err != nil && aaaares.err != nil {
+ // Prioritize the A error like we do in netxlite
+ return nil, ares.err
+ }
+ addrs := append(ares.addrs, aaaares.addrs...)
+ if len(addrs) < 1 {
+ return nil, netxlite.NewTopLevelGenericErrWrapper(netxlite.ErrOODNSNoAnswer)
+ }
+ return addrs, nil
+}
+
+// dnsOverUDPLookupResult contains the results of a DNS-over-UDP lookup.
+type dnsOverUDPLookupResult struct {
+ addrs []string
+ err error
+}
+
+// start starts and async lookup and returns the channel where we'll post the results
+func (reso *DNSOverUDPResolver) start(
+ ctx context.Context, qtype uint16, domain string) <-chan *dnsOverUDPLookupResult {
+ out := make(chan *dnsOverUDPLookupResult)
+ go reso.runch(ctx, qtype, domain, out)
+ return out
+}
+
+// runch runs and posts the results on the given channel
+func (reso *DNSOverUDPResolver) runch(
+ ctx context.Context, qtype uint16, domain string, out chan<- *dnsOverUDPLookupResult) {
+ m := &dnsOverUDPLookupResult{}
+ m.addrs, m.err = reso.run(ctx, qtype, domain)
+ out <- m
+}
+
+// run performs a lookup using the given query type
+func (reso *DNSOverUDPResolver) run(ctx context.Context, qtype uint16, domain string) ([]string, error) {
+ query, err := netxlite.NewDNSQuery(domain, qtype, false)
+ if err != nil {
+ return nil, err
+ }
+ d := netxlite.NewDialerWithoutResolver(reso.logger)
+ txp := netxlite.WrapDNSTransport(netxlite.NewUnwrappedDNSOverUDPTransport(d, reso.endpoint))
+ start := time.Now()
+ resp, err := txp.RoundTrip(ctx, query)
+ finish := time.Now()
+ select {
+ case reso.m[qtype] <- newArchivalDNSLookupResultFromRoundTrip(reso.index,
+ txp, start.Sub(reso.zeroTime), query, resp, err, finish.Sub(reso.zeroTime)):
+ default: // buffer is full
+ }
+ return resp.DecodeLookupHost()
+}
+
+// DNSLookupResultA extracts previously saved A measurements. This method will block if called
+// before calling LookupHost, because we use a single-buffer channel to measure.
+func (reso *DNSOverUDPResolver) DNSLookupResultA() (out *model.ArchivalDNSLookupResult) {
+ return <-reso.m[dns.TypeA]
+}
+
+// DNSLookupResultAAAA is like DNSLookupResultA but for AAAA.
+func (reso *DNSOverUDPResolver) DNSLookupResultAAAA() (out *model.ArchivalDNSLookupResult) {
+ return <-reso.m[dns.TypeAAAA]
+}
diff --git a/internal/measurexlite/tls.go b/internal/measurexlite/tls.go
new file mode 100644
index 0000000..3417b2f
--- /dev/null
+++ b/internal/measurexlite/tls.go
@@ -0,0 +1,75 @@
+package measurexlite
+
+//
+// TLSHandshaking and measuring handshakes
+//
+
+import (
+ "context"
+ "crypto/tls"
+ "net"
+ "time"
+
+ "github.com/ooni/probe-cli/v3/internal/model"
+ "github.com/ooni/probe-cli/v3/internal/netxlite"
+)
+
+// TLSHandshaker performs TLS handshakes.
+//
+// The zero-value struct is invalid. Use NewTLSHandshaker to create a new instance.
+//
+// Internally, we use a single-buffer channel to deliver observations. The intended
+// usage is that you create an instance, you call Handshake, you call HandshakeResult
+// to get the result, and then you create a new instance for a new measurement.
+type TLSHandshaker struct {
+ index int64
+ logger model.Logger
+ m chan *model.ArchivalTLSOrQUICHandshakeResult
+ zeroTime time.Time
+}
+
+// NewTLSHandshaker creates a new TLS handshaker for performing measurements.
+//
+// Arguments:
+//
+// - index is the index of this measurement;
+//
+// - logger is the logger to use;
+//
+// - zeroTime is the time to use as reference for generating observations.
+func NewTLSHandshaker(index int64, logger model.Logger, zeroTime time.Time) *TLSHandshaker {
+ return &TLSHandshaker{
+ index: index,
+ logger: logger,
+ m: make(chan *model.ArchivalTLSOrQUICHandshakeResult, 1),
+ zeroTime: zeroTime,
+ }
+}
+
+// Handshake performs a TLS handshake and saves the related observation inside
+// the handshaker. You can extract the observation using TCPHandshakeResult.
+//
+// Use the context to control the maximum handshaking time. The underlying code used
+// for handshaking will additionally impose a reasonable watchdog timeout.
+func (thx *TLSHandshaker) Handshake(
+ ctx context.Context, conn net.Conn, config *tls.Config) (model.TLSConn, error) {
+ address := conn.RemoteAddr().String()
+ th := netxlite.NewTLSHandshakerStdlib(thx.logger)
+ started := time.Now()
+ tlsConn, state, err := th.Handshake(ctx, conn, config)
+ finished := time.Now()
+ select {
+ case thx.m <- newArchivalTLSOrQUICHandshakeResult(thx.index,
+ started.Sub(thx.zeroTime), address, config, state, err, finished.Sub(thx.zeroTime)):
+ default: // buffer is full
+ }
+ // this cast is safe according to model/netx.go's documentation
+ return tlsConn.(model.TLSConn), err
+}
+
+// TLSHandshakeResult returns the saved TLS handshake observation. This function
+// blocks unless it's called right after Handshake. This happens because we internally
+// use a single buffered channel to store and deliver the observation.
+func (d *TLSHandshaker) TLSHandshakeResult() *model.ArchivalTLSOrQUICHandshakeResult {
+ return <-d.m
+}
diff --git a/internal/measurexlite/url.go b/internal/measurexlite/url.go
new file mode 100644
index 0000000..6c82561
--- /dev/null
+++ b/internal/measurexlite/url.go
@@ -0,0 +1,61 @@
+package measurexlite
+
+//
+// Helper functions for working with URLs
+//
+
+import (
+ "errors"
+ "net"
+ "net/url"
+)
+
+// ErrURLIsNil indicates that you passed a nil URL to URLEndpoint.
+var ErrURLIsNil = errors.New("experiment: URL is nil")
+
+// ErrURLHostnameIsEmpty indicates that the URL's hostname is empty.
+var ErrURLHostnameIsEmpty = errors.New("experiment: URL.Hostname is empty")
+
+// ErrNoDefaultPortForURLScheme indicates we don't know which would
+// be the right default port for the URL's scheme.
+var ErrNoDefaultPortForURLScheme = errors.New("experiment: no default port for URL.Scheme")
+
+// URLEndpoint returns the URL's endpoint or an error.
+func URLEndpoint(URL *url.URL) (string, error) {
+ if URL == nil {
+ return "", ErrURLIsNil
+ }
+ hostname := URL.Hostname()
+ if hostname == "" {
+ return "", ErrURLHostnameIsEmpty
+ }
+ port := URL.Port()
+ if port == "" {
+ switch URL.Scheme {
+ case "http":
+ port = "80"
+ case "https":
+ port = "443"
+ case "dot":
+ port = "8853"
+ default:
+ return "", ErrNoDefaultPortForURLScheme
+ }
+ }
+ hostport := net.JoinHostPort(hostname, port)
+ return hostport, nil
+}
+
+// NewURL constructs a new URL. The constructed URL will not include the port
+// if the port is already the default port for the given scheme.
+func NewURL(scheme, address, port, path string) *url.URL {
+ host := address
+ switch {
+ case scheme == "http" && port == "80":
+ case scheme == "https" && port == "443":
+ case port == "":
+ default:
+ host = net.JoinHostPort(address, port)
+ }
+ return &url.URL{Scheme: scheme, Host: host, Path: path}
+}
diff --git a/internal/model/archival.go b/internal/model/archival.go
index 42e0e84..b449f63 100644
--- a/internal/model/archival.go
+++ b/internal/model/archival.go
@@ -59,6 +59,20 @@ var (
// Base types
//
+// ArchivalBinaryData is the format with which we serialize binary data.
+type ArchivalBinaryData struct {
+ Format string `json:"format"`
+ Data []byte `json:"data"`
+}
+
+// NewArchivalBinaryData generates a new ArchivalBinaryData instance.
+func NewArchivalBinaryData(data []byte) ArchivalBinaryData {
+ return ArchivalBinaryData{
+ Format: "base64",
+ Data: data,
+ }
+}
+
// ArchivalMaybeBinaryData is a possibly binary string. We use this helper class
// to define a custom JSON encoder that allows us to choose the proper
// representation depending on whether the Value field is valid UTF-8 or not.
@@ -121,6 +135,7 @@ type ArchivalDNSLookupResult struct {
ResolverPort *string `json:"resolver_port"`
ResolverAddress string `json:"resolver_address"`
T float64 `json:"t"`
+ TransactionID int64 `json:"transaction_id"`
}
// ArchivalDNSAnswer is a DNS answer.
@@ -142,10 +157,11 @@ type ArchivalDNSAnswer struct {
//
// See https://github.com/ooni/spec/blob/master/data-formats/df-005-tcpconnect.md.
type ArchivalTCPConnectResult struct {
- IP string `json:"ip"`
- Port int `json:"port"`
- Status ArchivalTCPConnectStatus `json:"status"`
- T float64 `json:"t"`
+ IP string `json:"ip"`
+ Port int `json:"port"`
+ Status ArchivalTCPConnectStatus `json:"status"`
+ T float64 `json:"t"`
+ TransactionID int64 `json:"transaction_id"`
}
// ArchivalTCPConnectStatus is the status of ArchivalTCPConnectResult.
@@ -163,16 +179,17 @@ type ArchivalTCPConnectStatus struct {
//
// See https://github.com/ooni/spec/blob/master/data-formats/df-006-tlshandshake.md
type ArchivalTLSOrQUICHandshakeResult struct {
- Address string `json:"address"`
- CipherSuite string `json:"cipher_suite"`
- Failure *string `json:"failure"`
- NegotiatedProtocol string `json:"negotiated_protocol"`
- NoTLSVerify bool `json:"no_tls_verify"`
- PeerCertificates []ArchivalMaybeBinaryData `json:"peer_certificates"`
- ServerName string `json:"server_name"`
- T float64 `json:"t"`
- Tags []string `json:"tags"`
- TLSVersion string `json:"tls_version"`
+ Address string `json:"address"`
+ CipherSuite string `json:"cipher_suite"`
+ Failure *string `json:"failure"`
+ NegotiatedProtocol string `json:"negotiated_protocol"`
+ NoTLSVerify bool `json:"no_tls_verify"`
+ PeerCertificates []ArchivalBinaryData `json:"peer_certificates"`
+ ServerName string `json:"server_name"`
+ T float64 `json:"t"`
+ Tags []string `json:"tags"`
+ TLSVersion string `json:"tls_version"`
+ TransactionID int64 `json:"transaction_id"`
}
//
@@ -183,10 +200,11 @@ type ArchivalTLSOrQUICHandshakeResult struct {
//
// See https://github.com/ooni/spec/blob/master/data-formats/df-001-httpt.md.
type ArchivalHTTPRequestResult struct {
- Failure *string `json:"failure"`
- Request ArchivalHTTPRequest `json:"request"`
- Response ArchivalHTTPResponse `json:"response"`
- T float64 `json:"t"`
+ Failure *string `json:"failure"`
+ Request ArchivalHTTPRequest `json:"request"`
+ Response ArchivalHTTPResponse `json:"response"`
+ T float64 `json:"t"`
+ TransactionID int64 `json:"transaction_id"`
}
// ArchivalHTTPRequest contains an HTTP request.
@@ -302,11 +320,12 @@ type ArchivalHTTPTor struct {
//
// See https://github.com/ooni/spec/blob/master/data-formats/df-008-netevents.md.
type ArchivalNetworkEvent struct {
- Address string `json:"address,omitempty"`
- Failure *string `json:"failure"`
- NumBytes int64 `json:"num_bytes,omitempty"`
- Operation string `json:"operation"`
- Proto string `json:"proto,omitempty"`
- T float64 `json:"t"`
- Tags []string `json:"tags,omitempty"`
+ Address string `json:"address,omitempty"`
+ Failure *string `json:"failure"`
+ NumBytes int64 `json:"num_bytes,omitempty"`
+ Operation string `json:"operation"`
+ Proto string `json:"proto,omitempty"`
+ T float64 `json:"t"`
+ Tags []string `json:"tags,omitempty"`
+ TransactionID int64 `json:"transaction_id"`
}
diff --git a/internal/model/netx.go b/internal/model/netx.go
index f33240f..9221af6 100644
--- a/internal/model/netx.go
+++ b/internal/model/netx.go
@@ -13,6 +13,7 @@ import (
"time"
"github.com/lucas-clemente/quic-go"
+ oohttp "github.com/ooni/oohttp"
)
// DNSResponse is a parsed DNS response ready for further processing.
@@ -260,6 +261,9 @@ type Resolver interface {
LookupNS(ctx context.Context, domain string) ([]*net.NS, error)
}
+// TLSConn is TLS connection.
+type TLSConn = oohttp.TLSConn
+
// TLSDialer is a Dialer dialing TLS connections.
type TLSDialer interface {
// CloseIdleConnections closes idle connections, if any.
diff --git a/internal/netxlite/certifi.go b/internal/netxlite/certifi.go
index a316f8b..76f2895 100644
--- a/internal/netxlite/certifi.go
+++ b/internal/netxlite/certifi.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// 2022-05-28 13:27:21.630174629 +0200 CEST m=+0.293627763
+// 2022-06-10 22:42:43.909308 +0200 CEST m=+0.258633251
// https://curl.haxx.se/ca/cacert.pem
package netxlite
diff --git a/internal/netxlite/dnsencoder.go b/internal/netxlite/dnsencoder.go
index 9b9e8cb..a732c33 100644
--- a/internal/netxlite/dnsencoder.go
+++ b/internal/netxlite/dnsencoder.go
@@ -10,6 +10,7 @@ import (
"github.com/miekg/dns"
"github.com/ooni/probe-cli/v3/internal/atomicx"
"github.com/ooni/probe-cli/v3/internal/model"
+ "golang.org/x/net/idna"
)
// DNSEncoderMiekg uses github.com/miekg/dns to implement the Encoder.
@@ -26,6 +27,19 @@ const (
dnsDNSSECEnabled = true
)
+// NewDNSQuery encodes a query ...
+//
+// This function also takes care of IDNA...
+func NewDNSQuery(domain string, qtype uint16, padding bool) (model.DNSQuery, error) {
+ encodedDomain, err := idna.ToASCII(domain)
+ if err != nil {
+ return nil, err
+ }
+ encoder := &DNSEncoderMiekg{}
+ query := encoder.Encode(encodedDomain, qtype, padding)
+ return query, nil
+}
+
// Encoder implements model.DNSEncoder.Encode.
func (e *DNSEncoderMiekg) Encode(domain string, qtype uint16, padding bool) model.DNSQuery {
return &dnsQuery{
diff --git a/internal/netxlite/dnsovergetaddrinfo.go b/internal/netxlite/dnsovergetaddrinfo.go
index 0aca6ed..397c382 100644
--- a/internal/netxlite/dnsovergetaddrinfo.go
+++ b/internal/netxlite/dnsovergetaddrinfo.go
@@ -20,6 +20,17 @@ type dnsOverGetaddrinfoTransport struct {
testableLookupHost func(ctx context.Context, domain string) ([]string, error)
}
+// XXX: we need to move IDNA somewhere else: using directly a transport
+// means that we have lost any IDNA-handling opportunity
+
+// XXX: if we're not using this transport to obtain the CNAME, then
+// it actually makes quite little sense to use the transport?!
+
+// NewDNSOverGetaddrinfoTransport ...
+func NewDNSOverGetaddrinfoTransport() model.DNSTransport {
+ return WrapDNSTransport(&dnsOverGetaddrinfoTransport{})
+}
+
var _ model.DNSTransport = &dnsOverGetaddrinfoTransport{}
func (txp *dnsOverGetaddrinfoTransport) RoundTrip(
diff --git a/internal/netxlite/errno.go b/internal/netxlite/errno.go
index e21b1c5..aca6da7 100644
--- a/internal/netxlite/errno.go
+++ b/internal/netxlite/errno.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2022-05-28 13:27:22.097503116 +0200 CEST m=+0.338871155
+// Generated: 2022-06-10 22:42:44.540569 +0200 CEST m=+0.306261251
package netxlite
@@ -33,6 +33,8 @@ const (
FailureDestinationAddressRequired = "destination_address_required"
FailureEOFError = "eof_error"
FailureGenericTimeoutError = "generic_timeout_error"
+ FailureHTTPRequestFailed = "http_request_failed"
+ FailureHTTPUnexpectedRedirectError = "http_unexpected_redirect_error"
FailureHostUnreachable = "host_unreachable"
FailureInterrupted = "interrupted"
FailureInvalidArgument = "invalid_argument"
@@ -47,12 +49,14 @@ const (
FailureNotConnected = "not_connected"
FailureOperationWouldBlock = "operation_would_block"
FailurePermissionDenied = "permission_denied"
+ FailureProbeBug = "probe_bug"
FailureProtocolNotSupported = "protocol_not_supported"
FailureQUICIncompatibleVersion = "quic_incompatible_version"
FailureSSLFailedHandshake = "ssl_failed_handshake"
FailureSSLInvalidCertificate = "ssl_invalid_certificate"
FailureSSLInvalidHostname = "ssl_invalid_hostname"
FailureSSLUnknownAuthority = "ssl_unknown_authority"
+ FailureTelegramMissingTitleError = "telegram_missing_title_error"
FailureTimedOut = "timed_out"
FailureWrongProtocolType = "wrong_protocol_type"
)
@@ -85,6 +89,8 @@ var failuresMap = map[string]string{
"eof_error": "eof_error",
"generic_timeout_error": "generic_timeout_error",
"host_unreachable": "host_unreachable",
+ "http_request_failed": "http_request_failed",
+ "http_unexpected_redirect_error": "http_unexpected_redirect_error",
"interrupted": "interrupted",
"invalid_argument": "invalid_argument",
"json_parse_error": "json_parse_error",
@@ -98,12 +104,14 @@ var failuresMap = map[string]string{
"not_connected": "not_connected",
"operation_would_block": "operation_would_block",
"permission_denied": "permission_denied",
+ "probe_bug": "probe_bug",
"protocol_not_supported": "protocol_not_supported",
"quic_incompatible_version": "quic_incompatible_version",
"ssl_failed_handshake": "ssl_failed_handshake",
"ssl_invalid_certificate": "ssl_invalid_certificate",
"ssl_invalid_hostname": "ssl_invalid_hostname",
"ssl_unknown_authority": "ssl_unknown_authority",
+ "telegram_missing_title_error": "telegram_missing_title_error",
"timed_out": "timed_out",
"wrong_protocol_type": "wrong_protocol_type",
}
diff --git a/internal/netxlite/errno_darwin.go b/internal/netxlite/errno_darwin.go
index bd5a109..0159fa9 100644
--- a/internal/netxlite/errno_darwin.go
+++ b/internal/netxlite/errno_darwin.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2022-05-28 13:27:21.764075578 +0200 CEST m=+0.005443607
+// Generated: 2022-06-10 22:42:44.234631 +0200 CEST m=+0.000317001
package netxlite
diff --git a/internal/netxlite/errno_darwin_test.go b/internal/netxlite/errno_darwin_test.go
index 1add0b2..d0a54a9 100644
--- a/internal/netxlite/errno_darwin_test.go
+++ b/internal/netxlite/errno_darwin_test.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2022-05-28 13:27:21.820244729 +0200 CEST m=+0.061612769
+// Generated: 2022-06-10 22:42:44.310899 +0200 CEST m=+0.076586501
package netxlite
diff --git a/internal/netxlite/errno_freebsd.go b/internal/netxlite/errno_freebsd.go
index 4e5f8f7..b140686 100644
--- a/internal/netxlite/errno_freebsd.go
+++ b/internal/netxlite/errno_freebsd.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2022-05-28 13:27:21.843034214 +0200 CEST m=+0.084402243
+// Generated: 2022-06-10 22:42:44.329738 +0200 CEST m=+0.095425959
package netxlite
diff --git a/internal/netxlite/errno_freebsd_test.go b/internal/netxlite/errno_freebsd_test.go
index dcc90e1..033b673 100644
--- a/internal/netxlite/errno_freebsd_test.go
+++ b/internal/netxlite/errno_freebsd_test.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2022-05-28 13:27:21.881328637 +0200 CEST m=+0.122696672
+// Generated: 2022-06-10 22:42:44.367495 +0200 CEST m=+0.133183792
package netxlite
diff --git a/internal/netxlite/errno_linux.go b/internal/netxlite/errno_linux.go
index f1616ec..cc5f6b9 100644
--- a/internal/netxlite/errno_linux.go
+++ b/internal/netxlite/errno_linux.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2022-05-28 13:27:21.967785506 +0200 CEST m=+0.209153549
+// Generated: 2022-06-10 22:42:44.442386 +0200 CEST m=+0.208076792
package netxlite
diff --git a/internal/netxlite/errno_linux_test.go b/internal/netxlite/errno_linux_test.go
index bbdfe68..673fae0 100644
--- a/internal/netxlite/errno_linux_test.go
+++ b/internal/netxlite/errno_linux_test.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2022-05-28 13:27:22.010048884 +0200 CEST m=+0.251416941
+// Generated: 2022-06-10 22:42:44.480751 +0200 CEST m=+0.246442792
package netxlite
diff --git a/internal/netxlite/errno_openbsd.go b/internal/netxlite/errno_openbsd.go
index a1e05ef..f9239b4 100644
--- a/internal/netxlite/errno_openbsd.go
+++ b/internal/netxlite/errno_openbsd.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2022-05-28 13:27:21.904104276 +0200 CEST m=+0.145472305
+// Generated: 2022-06-10 22:42:44.385622 +0200 CEST m=+0.151311417
package netxlite
diff --git a/internal/netxlite/errno_openbsd_test.go b/internal/netxlite/errno_openbsd_test.go
index e714ec8..35c50f3 100644
--- a/internal/netxlite/errno_openbsd_test.go
+++ b/internal/netxlite/errno_openbsd_test.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2022-05-28 13:27:21.942808293 +0200 CEST m=+0.184176336
+// Generated: 2022-06-10 22:42:44.425007 +0200 CEST m=+0.190697001
package netxlite
diff --git a/internal/netxlite/errno_windows.go b/internal/netxlite/errno_windows.go
index ed3a18c..34057d5 100644
--- a/internal/netxlite/errno_windows.go
+++ b/internal/netxlite/errno_windows.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2022-05-28 13:27:22.034720951 +0200 CEST m=+0.276088980
+// Generated: 2022-06-10 22:42:44.498351 +0200 CEST m=+0.264042667
package netxlite
diff --git a/internal/netxlite/errno_windows_test.go b/internal/netxlite/errno_windows_test.go
index cfdbfa0..54d98bb 100644
--- a/internal/netxlite/errno_windows_test.go
+++ b/internal/netxlite/errno_windows_test.go
@@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
-// Generated: 2022-05-28 13:27:22.067609692 +0200 CEST m=+0.308977732
+// Generated: 2022-06-10 22:42:44.524107 +0200 CEST m=+0.289799251
package netxlite
diff --git a/internal/netxlite/internal/generrno/main.go b/internal/netxlite/internal/generrno/main.go
index 94d48ee..33b46cb 100644
--- a/internal/netxlite/internal/generrno/main.go
+++ b/internal/netxlite/internal/generrno/main.go
@@ -170,6 +170,10 @@ var Specs = []*ErrorSpec{
NewLibraryError("SSL_invalid_certificate"),
NewLibraryError("JSON_parse_error"),
NewLibraryError("connection_already_closed"),
+ NewLibraryError("probe_bug"),
+ NewLibraryError("telegram_missing_title_error"),
+ NewLibraryError("HTTP_request_failed"),
+ NewLibraryError("HTTP_unexpected_redirect_error"),
// QUIRKS: the following errors exist to clearly flag strange
// underlying behavior implemented by platforms.
diff --git a/internal/tracex/archival.go b/internal/tracex/archival.go
index 0351c42..8d3a2f1 100644
--- a/internal/tracex/archival.go
+++ b/internal/tracex/archival.go
@@ -302,9 +302,9 @@ func NewTLSHandshakesList(begin time.Time, events []Event) (out []TLSHandshake)
return
}
-func tlsMakePeerCerts(in [][]byte) (out []MaybeBinaryValue) {
+func tlsMakePeerCerts(in [][]byte) (out []model.ArchivalBinaryData) {
for _, entry := range in {
- out = append(out, MaybeBinaryValue{Value: string(entry)})
+ out = append(out, model.NewArchivalBinaryData(entry))
}
return
}
diff --git a/internal/tracex/archival_test.go b/internal/tracex/archival_test.go
index 756e3eb..13941f4 100644
--- a/internal/tracex/archival_test.go
+++ b/internal/tracex/archival_test.go
@@ -10,6 +10,7 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/gorilla/websocket"
+ "github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
@@ -547,10 +548,12 @@ func TestNewTLSHandshakesList(t *testing.T) {
Failure: NewFailure(io.EOF),
NegotiatedProtocol: "h2",
NoTLSVerify: false,
- PeerCertificates: []MaybeBinaryValue{{
- Value: "deadbeef",
+ PeerCertificates: []model.ArchivalBinaryData{{
+ Data: []byte("deadbeef"),
+ Format: "base64",
}, {
- Value: "abad1dea",
+ Data: []byte("abad1dea"),
+ Format: "base64",
}},
ServerName: "x.org",
T: 0.055,
@@ -582,10 +585,12 @@ func TestNewTLSHandshakesList(t *testing.T) {
Failure: NewFailure(io.EOF),
NegotiatedProtocol: "h3",
NoTLSVerify: false,
- PeerCertificates: []MaybeBinaryValue{{
- Value: "deadbeef",
+ PeerCertificates: []model.ArchivalBinaryData{{
+ Data: []byte("deadbeef"),
+ Format: "base64",
}, {
- Value: "abad1dea",
+ Data: []byte("abad1dea"),
+ Format: "base64",
}},
ServerName: "x.org",
T: 0.055,
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment