feat: Introduce anonymous credentials submission path for ooniprobe-cli

.github/workflows/alltests.yml

@@ -23,4 +23,6 @@ jobs:
           go-version: "${{ steps.goversion.outputs.version }}"
           cache-key-suffix: "-alltests-${{ steps.goversion.outputs.version }}"
 
+      - run: make userauth
+
       - run: go test -race -tags shaping ./...

.github/workflows/coverage.yml

@@ -24,6 +24,8 @@ jobs:
           go-version: "${{ steps.goversion.outputs.version }}"
           cache-key-suffix: "-coverage-${{ steps.goversion.outputs.version }}"
 
+      - run: make userauth
+
       - run: ./script/linuxcoverage.bash
 
       - uses: shogo82148/actions-goveralls@v1

.github/workflows/go1.22.yml

@@ -19,6 +19,8 @@ jobs:
           go-version: ~1.22
           cache-key-suffix: "-alltests-go1.22"
 
+      - run: make userauth
+
       # We cannot run buildtool tests using an unexpected version of Go because the
       # tests check whether we're using the expected version of Go 😂😂😂😂.
       - run: go test -race -tags shaping $(go list ./...|grep -v 'internal/cmd/buildtool')

.github/workflows/gobash.yml

@@ -42,4 +42,6 @@ jobs:
         with:
           go-version: "${{ matrix.goversion }}"
 
+      - run: make userauth
+
       - run: ./script/go.bash run ./internal/cmd/buildtool generic miniooni

Makefile

@@ -21,67 +21,74 @@ list-targets:
 help:
 	@cat Makefile | grep -E '^#(quick)?help:' | sed -E -e 's/^#(quick)?help://' -e s'/^\ //'
 
+OONIPROBE_RS_STATICLIB_URL := https://github.com/ooni/ooniprobe-rs/releases/latest/download/staticlib.tar.gz
+
+userauth:
+	curl -fsSL -o staticlib.tar.gz $(OONIPROBE_RS_STATICLIB_URL)
+	tar -xzf staticlib.tar.gz -C internal/userauth
+	rm -f staticlib.tar.gz
+
 #help:
 #help: The `make CLI/darwin` command builds the ooniprobe and miniooni
 #help: command line clients for darwin/amd64 and darwin/arm64.
 .PHONY: CLI/darwin
-CLI/darwin:
+CLI/darwin: userauth
 	./script/go.bash run ./internal/cmd/buildtool darwin
 
 #help:
 #help: The `make CLI/linux-static-386` command builds and statically links the
 #help: ooniprobe and miniooni binaries for linux/386.
 .PHONY: CLI/linux-static-386
-CLI/linux-static-386:
+CLI/linux-static-386: userauth
 	./script/go.bash run ./internal/cmd/buildtool linux docker 386
 
 #help:
 #help: The `make CLI/linux-static-amd64` command builds and statically links the
 #help: ooniprobe and miniooni binaries for linux/amd64.
 .PHONY: CLI/linux-static-amd64
-CLI/linux-static-amd64:
+CLI/linux-static-amd64: userauth
 	./script/go.bash run ./internal/cmd/buildtool linux docker amd64
 
 #help:
 #help: The `make CLI/linux-static-armv6` command builds and statically links the
 #help: ooniprobe and miniooni binaries for linux/arm/v6.
 .PHONY: CLI/linux-static-armv6
-CLI/linux-static-armv6:
+CLI/linux-static-armv6: userauth
 	./script/go.bash run ./internal/cmd/buildtool linux docker armv6
 
 #help:
 #help: The `make CLI/linux-static-armv7` command builds and statically links the
 #help: ooniprobe and miniooni binaries for linux/arm/v7.
 .PHONY: CLI/linux-static-armv7
-CLI/linux-static-armv7:
+CLI/linux-static-armv7: userauth
 	./script/go.bash run ./internal/cmd/buildtool linux docker armv7
 
 #help:
 #help: The `make CLI/linux-static-arm64` command builds and statically links the
 #help: ooniprobe and miniooni binaries for linux/arm64.
 .PHONY: CLI/linux-static-arm64
-CLI/linux-static-arm64:
+CLI/linux-static-arm64: userauth
 	./script/go.bash run ./internal/cmd/buildtool linux docker arm64
 
 #help:
 #help: The `make CLI/miniooni` command creates a build of miniooni, for the current
 #help: system, putting the binary in the top-level directory.
 .PHONY: CLI/miniooni
-CLI/miniooni:
+CLI/miniooni: userauth
 	./script/go.bash run ./internal/cmd/buildtool generic miniooni
 
 #help:
 #help: The `make CLI/ooniprobe` command creates a build of ooniprobe, for the current
 #help: system, putting the binary in the top-level directory.
 .PHONY: CLI/ooniprobe
-CLI/ooniprobe:
+CLI/ooniprobe: userauth
 	./script/go.bash run ./internal/cmd/buildtool generic ooniprobe
 
 #help:
 #help: The `make CLI/windows` command builds the ooniprobe and miniooni
 #help: command line clients for windows/386 and windows/amd64.
 .PHONY: CLI/windows
-CLI/windows:
+CLI/windows: userauth
 	./script/go.bash run ./internal/cmd/buildtool windows
 
 #help:

cmd/ooniprobe/internal/cli/run/run.go

@@ -14,6 +14,7 @@ import (
 func init() {
 	cmd := root.Command("run", "Run a test group or OONI Run link")
 	noCollector := cmd.Flag("no-collector", "Disable uploading measurements to a collector").Bool()
+	noCredentials := cmd.Flag("no-creds", "Submit measurements without an anonymous credential").Bool()
 
 	var probe *ooni.Probe
 	cmd.Action(func(_ *kingpin.ParseContext) error {
@@ -40,9 +41,10 @@ func init() {
 			}
 			log.Infof("Running %s tests", color.BlueString(name))
 			conf := nettests.RunGroupConfig{
-				GroupName: name,
-				Probe:     probe,
-				RunType:   runType,
+				GroupName:     name,
+				Probe:         probe,
+				RunType:       runType,
+				NoCredentials: *noCredentials,
 			}
 			if err := nettests.RunGroup(conf); err != nil {
 				log.WithError(err).Errorf("failed to run %s", name)
@@ -65,11 +67,12 @@ func init() {
 	websitesCmd.Action(func(_ *kingpin.ParseContext) error {
 		log.Infof("Running %s tests", color.BlueString("websites"))
 		return nettests.RunGroup(nettests.RunGroupConfig{
-			GroupName:  "websites",
-			Probe:      probe,
-			InputFiles: *inputFile,
-			Inputs:     *input,
-			RunType:    model.RunTypeManual,
+			GroupName:     "websites",
+			Probe:         probe,
+			InputFiles:    *inputFile,
+			Inputs:        *input,
+			RunType:       model.RunTypeManual,
+			NoCredentials: *noCredentials,
 		})
 	})
 

cmd/ooniprobe/internal/nettests/nettests.go

@@ -57,6 +57,9 @@ type Controller struct {
 	// not set, the underlying code defaults to model.RunTypeTimed.
 	RunType model.RunType
 
+	// NoCredentials disables submitting with an anonymous credential.
+	NoCredentials bool
+
 	// numInputs is the total number of inputs
 	numInputs int
 
@@ -139,17 +142,6 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []model.Experim
 	log.Debug(color.RedString("status.queued"))
 	log.Debug(color.RedString("status.started"))
 
-	if c.Probe.Config().Sharing.UploadResults {
-		if err := exp.OpenReportContext(context.Background()); err != nil {
-			log.Debugf(
-				"%s: %s", color.RedString("failure.report_create"), err.Error(),
-			)
-		} else {
-			log.Debugf(color.RedString("status.report_create"))
-			reportID = sql.NullString{String: exp.ReportID(), Valid: true}
-		}
-	}
-
 	maxRuntime := time.Duration(c.Probe.Config().Nettests.WebsitesMaxRuntime) * time.Second
 	if c.RunType == model.RunTypeTimed && maxRuntime > 0 {
 		log.Debug("disabling maxRuntime when running in the background")
@@ -164,6 +156,16 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []model.Experim
 		log.Debug("disabling maxRuntime with user-provided input")
 		maxRuntime = 0
 	}
+
+	var submitter model.Submitter
+	if c.Probe.Config().Sharing.UploadResults {
+		if s, err := c.Session.NewSubmitter(context.Background(), !c.NoCredentials); err != nil {
+			log.WithError(err).Debug("cannot create submitter; measurements will be saved to disk")
+		} else {
+			submitter = s
+		}
+	}
+
 	start := time.Now()
 	c.ntStartTime = start
 	for idx, input := range inputs {
@@ -211,11 +213,8 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []model.Experim
 		}
 
 		saveToDisk := true
-		if c.Probe.Config().Sharing.UploadResults {
-			// Implementation note: SubmitMeasurement will fail here if we did fail
-			// to open the report but we still want to continue. There will be a
-			// bit of a spew in the logs, perhaps, but stopping seems less efficient.
-			if _, err := exp.SubmitAndUpdateMeasurementContext(context.Background(), measurement); err != nil {
+		if submitter != nil {
+			if _, err := submitter.Submit(context.Background(), measurement); err != nil {
 				log.Debug(color.RedString("failure.measurement_submission"))
 				if err := db.UploadFailed(c.msmts[idx64], err.Error()); err != nil {
 					return errors.Wrap(err, "failed to mark upload as failed")

cmd/ooniprobe/internal/nettests/run.go

@@ -14,11 +14,12 @@ import (
 
 // RunGroupConfig contains the settings for running a nettest group.
 type RunGroupConfig struct {
-	GroupName  string
-	InputFiles []string
-	Inputs     []string
-	Probe      *ooni.Probe
-	RunType    model.RunType // hint for check-in API
+	GroupName     string
+	InputFiles    []string
+	Inputs        []string
+	Probe         *ooni.Probe
+	RunType       model.RunType // hint for check-in API
+	NoCredentials bool
 }
 
 const websitesURLLimitRemoved = `WARNING: CONFIGURATION CHANGE REQUIRED:
@@ -114,6 +115,7 @@ func RunGroup(config RunGroupConfig) error {
 		ctl.InputFiles = config.InputFiles
 		ctl.Inputs = config.Inputs
 		ctl.RunType = config.RunType
+		ctl.NoCredentials = config.NoCredentials
 		ctl.SetNettestIndex(i, len(group.Nettests))
 		if err = nt.Run(ctl); err != nil {
 			// We used to emit an error here, now we emit a warning--the proper choice

internal/cmd/miniooni/main.go

@@ -35,6 +35,7 @@ type Options struct {
 	MaxRuntime          int64
 	NoJSON              bool
 	NoCollector         bool
+	NoCredentials       bool
 	ProbeServicesURL    string
 	Proxy               string
 	Random              bool
@@ -100,6 +101,13 @@ func main() {
 		"do not submit measurements to the OONI collector",
 	)
 
+	flags.BoolVar(
+		&globalOptions.NoCredentials,
+		"no-creds",
+		false,
+		"submit measurements without an anonymous credential",
+	)
+
 	flags.StringVar(
 		&globalOptions.ProbeServicesURL,
 		"probe-services",

internal/cmd/miniooni/oonirun.go

@@ -26,6 +26,7 @@ func ooniRunMain(ctx context.Context,
 		KVStore:       sess.KeyValueStore(),
 		MaxRuntime:    currentOptions.MaxRuntime,
 		NoCollector:   currentOptions.NoCollector,
+		NoCredentials: currentOptions.NoCredentials,
 		NoJSON:        currentOptions.NoJSON,
 		Random:        currentOptions.Random,
 		ReportFile:    currentOptions.ReportFile,

internal/cmd/miniooni/runx.go

@@ -22,6 +22,7 @@ func runx(ctx context.Context, sess oonirun.Session, experimentName string,
 		MaxRuntime:     currentOptions.MaxRuntime,
 		Name:           experimentName,
 		NoCollector:    currentOptions.NoCollector,
+		NoCredentials:  currentOptions.NoCredentials,
 		NoJSON:         currentOptions.NoJSON,
 		Random:         currentOptions.Random,
 		ReportFile:     currentOptions.ReportFile,

internal/cmd/oonireport/oonireport.go

@@ -88,7 +88,7 @@ func newSession(ctx context.Context) *engine.Session {
 
 // new Submitter creates a probe services client and submitter
 func newSubmitter(sess *engine.Session, ctx context.Context) model.Submitter {
-	return runtimex.Try1(sess.NewSubmitter(ctx))
+	return runtimex.Try1(sess.NewSubmitter(ctx, true))
 }
 
 // toMeasurement loads an input string as model.Measurement

internal/engine/session.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math"
 	"net/url"
 	"os"
 	"sync"
@@ -19,6 +20,7 @@ import (
 	"github.com/ooni/probe-cli/v3/internal/probeservices"
 	"github.com/ooni/probe-cli/v3/internal/runtimex"
 	"github.com/ooni/probe-cli/v3/internal/tunnel"
+	"github.com/ooni/probe-cli/v3/internal/userauth"
 	"github.com/ooni/probe-cli/v3/internal/version"
 )
 
@@ -462,12 +464,59 @@ func (s *Session) newProbeServicesClient(ctx context.Context) (*probeservices.Cl
 }
 
 // NewSubmitter creates a new submitter instance.
-func (s *Session) NewSubmitter(ctx context.Context) (model.Submitter, error) {
+func (s *Session) NewSubmitter(ctx context.Context, useAuth bool) (model.Submitter, error) {
 	psc, err := s.newProbeServicesClient(ctx)
 	if err != nil {
 		return nil, err
 	}
-	return probeservices.NewSubmitter(psc, s.Logger()), nil
+
+	// Return the probeservices submitter if the caller chooses to submit
+	// without credentials
+	if !useAuth {
+		return psc, nil
+	}
+
+	// The probe-services client itself submits without a credential; it is
+	// the baseline and the fallback for the credential path.
+	var base model.Submitter = psc
+
+	manifest, err := psc.GetManifest(ctx)
+	if err != nil {
+		s.Logger().Debugf("userauth: manifest unavailable, submitting without credential: %s", err.Error())
+		return base, nil
+	}
+	probeCC, probeASN := s.ProbeCC(), s.ProbeASNString()
+	ageRange, minMeasurementCount, err := probeservices.GetRangesFromPolicy(manifest, probeCC, probeASN)
+	if err != nil {
+		s.Logger().Debug("userauth: no usable submission policy, submitting without credential")
+		return base, nil
+	}
+
+	cred, err := userauth.NewCredentialSubmitter(ctx, userauth.CredentialSubmitterConfig{
+		BaseURL:         psc.BaseURL,
+		ProbeCC:         probeCC,
+		ProbeASN:        probeASN,
+		PublicParams:    manifest.Manifest.PublicParameters,
+		ManifestVersion: manifest.Meta.Version,
+		AgeRange: userauth.ParamRange{
+			Min: ageRange[0],
+			Max: ageRange[1],
+		},
+		MeasurementCountRange: userauth.ParamRange{
+			Min: minMeasurementCount,
+			Max: math.MaxUint32,
+		},
+		Proxy:     s.ProxyURLString(),
+		Store:     userauth.NewCredStore(s.KeyValueStore()),
+		Logger:    s.Logger(),
+		Fallback:  base,
+		UserAgent: s.SoftwareName(),
+	})
+	if err != nil {
+		s.Logger().Debugf("credentials unavailable, submitting without credential: %s", err.Error())
+		return base, nil
+	}
+	return cred, nil
 }
 
 // newOrchestraClient creates a new orchestra client. This client is registered
@@ -561,6 +610,15 @@ func (s *Session) ProxyURL() *url.URL {
 	return s.proxyURL
 }
 
+// ProxyURLString returns the proxy url as a string
+func (s *Session) ProxyURLString() string {
+	proxyURL := s.ProxyURL()
+	if proxyURL == nil {
+		return ""
+	}
+	return proxyURL.String()
+}
+
 // ResolverASNString returns the resolver ASN as a string
 func (s *Session) ResolverASNString() string {
 	return fmt.Sprintf("AS%d", s.ResolverASN())

internal/engine/session_integration_test.go

@@ -497,7 +497,7 @@ func TestSessionNewSubmitterReturnsNonNilSubmitter(t *testing.T) {
 	}
 
 	sess := newSessionForTesting(t)
-	subm, err := sess.NewSubmitter(context.Background())
+	subm, err := sess.NewSubmitter(context.Background(), false)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/engine/session_internal_test.go

@@ -232,7 +232,7 @@ func TestSessionNewSubmitterWithCancelledContext(t *testing.T) {
 	sess := newSessionForTesting(t)
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel() // fail immediately
-	subm, err := sess.NewSubmitter(ctx)
+	subm, err := sess.NewSubmitter(ctx, false)
 	if !errors.Is(err, context.Canceled) {
 		t.Fatal("not the error we expected", err)
 	}