diff --git a/.github/workflows/alltests.yml b/.github/workflows/alltests.yml index bd168add8a..93c491fc02 100644 --- a/.github/workflows/alltests.yml +++ b/.github/workflows/alltests.yml @@ -23,4 +23,6 @@ jobs: go-version: "${{ steps.goversion.outputs.version }}" cache-key-suffix: "-alltests-${{ steps.goversion.outputs.version }}" + - run: make userauth + - run: go test -race -tags shaping ./... diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index f9801c1160..5670b91fa8 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -24,6 +24,8 @@ jobs: go-version: "${{ steps.goversion.outputs.version }}" cache-key-suffix: "-coverage-${{ steps.goversion.outputs.version }}" + - run: make userauth + - run: ./script/linuxcoverage.bash - uses: shogo82148/actions-goveralls@v1 diff --git a/.github/workflows/go1.22.yml b/.github/workflows/go1.22.yml index 5850671f8e..14ca2758ad 100644 --- a/.github/workflows/go1.22.yml +++ b/.github/workflows/go1.22.yml @@ -19,6 +19,8 @@ jobs: go-version: ~1.22 cache-key-suffix: "-alltests-go1.22" + - run: make userauth + # We cannot run buildtool tests using an unexpected version of Go because the # tests check whether we're using the expected version of Go 😂😂😂😂. - run: go test -race -tags shaping $(go list ./...|grep -v 'internal/cmd/buildtool') diff --git a/.github/workflows/gobash.yml b/.github/workflows/gobash.yml index e866275e7a..5ead3b12e6 100644 --- a/.github/workflows/gobash.yml +++ b/.github/workflows/gobash.yml @@ -42,4 +42,6 @@ jobs: with: go-version: "${{ matrix.goversion }}" + - run: make userauth + - run: ./script/go.bash run ./internal/cmd/buildtool generic miniooni diff --git a/Makefile b/Makefile index 5d1c879e5c..dc1c091055 100644 --- a/Makefile +++ b/Makefile @@ -21,67 +21,74 @@ list-targets: help: @cat Makefile | grep -E '^#(quick)?help:' | sed -E -e 's/^#(quick)?help://' -e s'/^\ //' +OONIPROBE_RS_STATICLIB_URL := https://github.com/ooni/ooniprobe-rs/releases/latest/download/staticlib.tar.gz + +userauth: + curl -fsSL -o staticlib.tar.gz $(OONIPROBE_RS_STATICLIB_URL) + tar -xzf staticlib.tar.gz -C internal/userauth + rm -f staticlib.tar.gz + #help: #help: The `make CLI/darwin` command builds the ooniprobe and miniooni #help: command line clients for darwin/amd64 and darwin/arm64. .PHONY: CLI/darwin -CLI/darwin: +CLI/darwin: userauth ./script/go.bash run ./internal/cmd/buildtool darwin #help: #help: The `make CLI/linux-static-386` command builds and statically links the #help: ooniprobe and miniooni binaries for linux/386. .PHONY: CLI/linux-static-386 -CLI/linux-static-386: +CLI/linux-static-386: userauth ./script/go.bash run ./internal/cmd/buildtool linux docker 386 #help: #help: The `make CLI/linux-static-amd64` command builds and statically links the #help: ooniprobe and miniooni binaries for linux/amd64. .PHONY: CLI/linux-static-amd64 -CLI/linux-static-amd64: +CLI/linux-static-amd64: userauth ./script/go.bash run ./internal/cmd/buildtool linux docker amd64 #help: #help: The `make CLI/linux-static-armv6` command builds and statically links the #help: ooniprobe and miniooni binaries for linux/arm/v6. .PHONY: CLI/linux-static-armv6 -CLI/linux-static-armv6: +CLI/linux-static-armv6: userauth ./script/go.bash run ./internal/cmd/buildtool linux docker armv6 #help: #help: The `make CLI/linux-static-armv7` command builds and statically links the #help: ooniprobe and miniooni binaries for linux/arm/v7. .PHONY: CLI/linux-static-armv7 -CLI/linux-static-armv7: +CLI/linux-static-armv7: userauth ./script/go.bash run ./internal/cmd/buildtool linux docker armv7 #help: #help: The `make CLI/linux-static-arm64` command builds and statically links the #help: ooniprobe and miniooni binaries for linux/arm64. .PHONY: CLI/linux-static-arm64 -CLI/linux-static-arm64: +CLI/linux-static-arm64: userauth ./script/go.bash run ./internal/cmd/buildtool linux docker arm64 #help: #help: The `make CLI/miniooni` command creates a build of miniooni, for the current #help: system, putting the binary in the top-level directory. .PHONY: CLI/miniooni -CLI/miniooni: +CLI/miniooni: userauth ./script/go.bash run ./internal/cmd/buildtool generic miniooni #help: #help: The `make CLI/ooniprobe` command creates a build of ooniprobe, for the current #help: system, putting the binary in the top-level directory. .PHONY: CLI/ooniprobe -CLI/ooniprobe: +CLI/ooniprobe: userauth ./script/go.bash run ./internal/cmd/buildtool generic ooniprobe #help: #help: The `make CLI/windows` command builds the ooniprobe and miniooni #help: command line clients for windows/386 and windows/amd64. .PHONY: CLI/windows -CLI/windows: +CLI/windows: userauth ./script/go.bash run ./internal/cmd/buildtool windows #help: diff --git a/cmd/ooniprobe/internal/cli/run/run.go b/cmd/ooniprobe/internal/cli/run/run.go index 32d0d4c022..92b01978ad 100644 --- a/cmd/ooniprobe/internal/cli/run/run.go +++ b/cmd/ooniprobe/internal/cli/run/run.go @@ -14,6 +14,7 @@ import ( func init() { cmd := root.Command("run", "Run a test group or OONI Run link") noCollector := cmd.Flag("no-collector", "Disable uploading measurements to a collector").Bool() + noCredentials := cmd.Flag("no-creds", "Submit measurements without an anonymous credential").Bool() var probe *ooni.Probe cmd.Action(func(_ *kingpin.ParseContext) error { @@ -40,9 +41,10 @@ func init() { } log.Infof("Running %s tests", color.BlueString(name)) conf := nettests.RunGroupConfig{ - GroupName: name, - Probe: probe, - RunType: runType, + GroupName: name, + Probe: probe, + RunType: runType, + NoCredentials: *noCredentials, } if err := nettests.RunGroup(conf); err != nil { log.WithError(err).Errorf("failed to run %s", name) @@ -65,11 +67,12 @@ func init() { websitesCmd.Action(func(_ *kingpin.ParseContext) error { log.Infof("Running %s tests", color.BlueString("websites")) return nettests.RunGroup(nettests.RunGroupConfig{ - GroupName: "websites", - Probe: probe, - InputFiles: *inputFile, - Inputs: *input, - RunType: model.RunTypeManual, + GroupName: "websites", + Probe: probe, + InputFiles: *inputFile, + Inputs: *input, + RunType: model.RunTypeManual, + NoCredentials: *noCredentials, }) }) diff --git a/cmd/ooniprobe/internal/nettests/nettests.go b/cmd/ooniprobe/internal/nettests/nettests.go index 847b905791..9a34e47b63 100644 --- a/cmd/ooniprobe/internal/nettests/nettests.go +++ b/cmd/ooniprobe/internal/nettests/nettests.go @@ -57,6 +57,9 @@ type Controller struct { // not set, the underlying code defaults to model.RunTypeTimed. RunType model.RunType + // NoCredentials disables submitting with an anonymous credential. + NoCredentials bool + // numInputs is the total number of inputs numInputs int @@ -139,17 +142,6 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []model.Experim log.Debug(color.RedString("status.queued")) log.Debug(color.RedString("status.started")) - if c.Probe.Config().Sharing.UploadResults { - if err := exp.OpenReportContext(context.Background()); err != nil { - log.Debugf( - "%s: %s", color.RedString("failure.report_create"), err.Error(), - ) - } else { - log.Debugf(color.RedString("status.report_create")) - reportID = sql.NullString{String: exp.ReportID(), Valid: true} - } - } - maxRuntime := time.Duration(c.Probe.Config().Nettests.WebsitesMaxRuntime) * time.Second if c.RunType == model.RunTypeTimed && maxRuntime > 0 { log.Debug("disabling maxRuntime when running in the background") @@ -164,6 +156,16 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []model.Experim log.Debug("disabling maxRuntime with user-provided input") maxRuntime = 0 } + + var submitter model.Submitter + if c.Probe.Config().Sharing.UploadResults { + if s, err := c.Session.NewSubmitter(context.Background(), !c.NoCredentials); err != nil { + log.WithError(err).Debug("cannot create submitter; measurements will be saved to disk") + } else { + submitter = s + } + } + start := time.Now() c.ntStartTime = start for idx, input := range inputs { @@ -211,11 +213,8 @@ func (c *Controller) Run(builder model.ExperimentBuilder, inputs []model.Experim } saveToDisk := true - if c.Probe.Config().Sharing.UploadResults { - // Implementation note: SubmitMeasurement will fail here if we did fail - // to open the report but we still want to continue. There will be a - // bit of a spew in the logs, perhaps, but stopping seems less efficient. - if _, err := exp.SubmitAndUpdateMeasurementContext(context.Background(), measurement); err != nil { + if submitter != nil { + if _, err := submitter.Submit(context.Background(), measurement); err != nil { log.Debug(color.RedString("failure.measurement_submission")) if err := db.UploadFailed(c.msmts[idx64], err.Error()); err != nil { return errors.Wrap(err, "failed to mark upload as failed") diff --git a/cmd/ooniprobe/internal/nettests/run.go b/cmd/ooniprobe/internal/nettests/run.go index 1ad9ae2bab..5137d49a36 100644 --- a/cmd/ooniprobe/internal/nettests/run.go +++ b/cmd/ooniprobe/internal/nettests/run.go @@ -14,11 +14,12 @@ import ( // RunGroupConfig contains the settings for running a nettest group. type RunGroupConfig struct { - GroupName string - InputFiles []string - Inputs []string - Probe *ooni.Probe - RunType model.RunType // hint for check-in API + GroupName string + InputFiles []string + Inputs []string + Probe *ooni.Probe + RunType model.RunType // hint for check-in API + NoCredentials bool } const websitesURLLimitRemoved = `WARNING: CONFIGURATION CHANGE REQUIRED: @@ -114,6 +115,7 @@ func RunGroup(config RunGroupConfig) error { ctl.InputFiles = config.InputFiles ctl.Inputs = config.Inputs ctl.RunType = config.RunType + ctl.NoCredentials = config.NoCredentials ctl.SetNettestIndex(i, len(group.Nettests)) if err = nt.Run(ctl); err != nil { // We used to emit an error here, now we emit a warning--the proper choice diff --git a/internal/cmd/miniooni/main.go b/internal/cmd/miniooni/main.go index 11add9c7e4..f4b0abd7b6 100644 --- a/internal/cmd/miniooni/main.go +++ b/internal/cmd/miniooni/main.go @@ -35,6 +35,7 @@ type Options struct { MaxRuntime int64 NoJSON bool NoCollector bool + NoCredentials bool ProbeServicesURL string Proxy string Random bool @@ -100,6 +101,13 @@ func main() { "do not submit measurements to the OONI collector", ) + flags.BoolVar( + &globalOptions.NoCredentials, + "no-creds", + false, + "submit measurements without an anonymous credential", + ) + flags.StringVar( &globalOptions.ProbeServicesURL, "probe-services", diff --git a/internal/cmd/miniooni/oonirun.go b/internal/cmd/miniooni/oonirun.go index d4c2a2690f..21b96f5459 100644 --- a/internal/cmd/miniooni/oonirun.go +++ b/internal/cmd/miniooni/oonirun.go @@ -26,6 +26,7 @@ func ooniRunMain(ctx context.Context, KVStore: sess.KeyValueStore(), MaxRuntime: currentOptions.MaxRuntime, NoCollector: currentOptions.NoCollector, + NoCredentials: currentOptions.NoCredentials, NoJSON: currentOptions.NoJSON, Random: currentOptions.Random, ReportFile: currentOptions.ReportFile, diff --git a/internal/cmd/miniooni/runx.go b/internal/cmd/miniooni/runx.go index 12bb18518b..91817c3751 100644 --- a/internal/cmd/miniooni/runx.go +++ b/internal/cmd/miniooni/runx.go @@ -22,6 +22,7 @@ func runx(ctx context.Context, sess oonirun.Session, experimentName string, MaxRuntime: currentOptions.MaxRuntime, Name: experimentName, NoCollector: currentOptions.NoCollector, + NoCredentials: currentOptions.NoCredentials, NoJSON: currentOptions.NoJSON, Random: currentOptions.Random, ReportFile: currentOptions.ReportFile, diff --git a/internal/cmd/oonireport/oonireport.go b/internal/cmd/oonireport/oonireport.go index d17953fc99..d016a91845 100644 --- a/internal/cmd/oonireport/oonireport.go +++ b/internal/cmd/oonireport/oonireport.go @@ -88,7 +88,7 @@ func newSession(ctx context.Context) *engine.Session { // new Submitter creates a probe services client and submitter func newSubmitter(sess *engine.Session, ctx context.Context) model.Submitter { - return runtimex.Try1(sess.NewSubmitter(ctx)) + return runtimex.Try1(sess.NewSubmitter(ctx, true)) } // toMeasurement loads an input string as model.Measurement diff --git a/internal/engine/session.go b/internal/engine/session.go index 7f953c5501..0dc1e72156 100644 --- a/internal/engine/session.go +++ b/internal/engine/session.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "math" "net/url" "os" "sync" @@ -19,6 +20,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/probeservices" "github.com/ooni/probe-cli/v3/internal/runtimex" "github.com/ooni/probe-cli/v3/internal/tunnel" + "github.com/ooni/probe-cli/v3/internal/userauth" "github.com/ooni/probe-cli/v3/internal/version" ) @@ -462,12 +464,59 @@ func (s *Session) newProbeServicesClient(ctx context.Context) (*probeservices.Cl } // NewSubmitter creates a new submitter instance. -func (s *Session) NewSubmitter(ctx context.Context) (model.Submitter, error) { +func (s *Session) NewSubmitter(ctx context.Context, useAuth bool) (model.Submitter, error) { psc, err := s.newProbeServicesClient(ctx) if err != nil { return nil, err } - return probeservices.NewSubmitter(psc, s.Logger()), nil + + // Return the probeservices submitter if the caller chooses to submit + // without credentials + if !useAuth { + return psc, nil + } + + // The probe-services client itself submits without a credential; it is + // the baseline and the fallback for the credential path. + var base model.Submitter = psc + + manifest, err := psc.GetManifest(ctx) + if err != nil { + s.Logger().Debugf("userauth: manifest unavailable, submitting without credential: %s", err.Error()) + return base, nil + } + probeCC, probeASN := s.ProbeCC(), s.ProbeASNString() + ageRange, minMeasurementCount, err := probeservices.GetRangesFromPolicy(manifest, probeCC, probeASN) + if err != nil { + s.Logger().Debug("userauth: no usable submission policy, submitting without credential") + return base, nil + } + + cred, err := userauth.NewCredentialSubmitter(ctx, userauth.CredentialSubmitterConfig{ + BaseURL: psc.BaseURL, + ProbeCC: probeCC, + ProbeASN: probeASN, + PublicParams: manifest.Manifest.PublicParameters, + ManifestVersion: manifest.Meta.Version, + AgeRange: userauth.ParamRange{ + Min: ageRange[0], + Max: ageRange[1], + }, + MeasurementCountRange: userauth.ParamRange{ + Min: minMeasurementCount, + Max: math.MaxUint32, + }, + Proxy: s.ProxyURLString(), + Store: userauth.NewCredStore(s.KeyValueStore()), + Logger: s.Logger(), + Fallback: base, + UserAgent: s.SoftwareName(), + }) + if err != nil { + s.Logger().Debugf("credentials unavailable, submitting without credential: %s", err.Error()) + return base, nil + } + return cred, nil } // newOrchestraClient creates a new orchestra client. This client is registered @@ -561,6 +610,15 @@ func (s *Session) ProxyURL() *url.URL { return s.proxyURL } +// ProxyURLString returns the proxy url as a string +func (s *Session) ProxyURLString() string { + proxyURL := s.ProxyURL() + if proxyURL == nil { + return "" + } + return proxyURL.String() +} + // ResolverASNString returns the resolver ASN as a string func (s *Session) ResolverASNString() string { return fmt.Sprintf("AS%d", s.ResolverASN()) diff --git a/internal/engine/session_integration_test.go b/internal/engine/session_integration_test.go index b7e800b0a1..a0828d0953 100644 --- a/internal/engine/session_integration_test.go +++ b/internal/engine/session_integration_test.go @@ -497,7 +497,7 @@ func TestSessionNewSubmitterReturnsNonNilSubmitter(t *testing.T) { } sess := newSessionForTesting(t) - subm, err := sess.NewSubmitter(context.Background()) + subm, err := sess.NewSubmitter(context.Background(), false) if err != nil { t.Fatal(err) } diff --git a/internal/engine/session_internal_test.go b/internal/engine/session_internal_test.go index 65b8b7e873..8922279a5d 100644 --- a/internal/engine/session_internal_test.go +++ b/internal/engine/session_internal_test.go @@ -232,7 +232,7 @@ func TestSessionNewSubmitterWithCancelledContext(t *testing.T) { sess := newSessionForTesting(t) ctx, cancel := context.WithCancel(context.Background()) cancel() // fail immediately - subm, err := sess.NewSubmitter(ctx) + subm, err := sess.NewSubmitter(ctx, false) if !errors.Is(err, context.Canceled) { t.Fatal("not the error we expected", err) } diff --git a/internal/mocks/session.go b/internal/mocks/session.go index 31b8a5bcbb..c42efb2268 100644 --- a/internal/mocks/session.go +++ b/internal/mocks/session.go @@ -57,7 +57,7 @@ type Session struct { MockNewExperimentBuilder func(name string) (model.ExperimentBuilder, error) - MockNewSubmitter func(ctx context.Context) (model.Submitter, error) + MockNewSubmitter func(ctx context.Context, useAuth bool) (model.Submitter, error) MockCheckIn func(ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) @@ -163,8 +163,8 @@ func (sess *Session) NewExperimentBuilder(name string) (model.ExperimentBuilder, return sess.MockNewExperimentBuilder(name) } -func (sess *Session) NewSubmitter(ctx context.Context) (model.Submitter, error) { - return sess.MockNewSubmitter(ctx) +func (sess *Session) NewSubmitter(ctx context.Context, useAuth bool) (model.Submitter, error) { + return sess.MockNewSubmitter(ctx, useAuth) } func (sess *Session) CheckIn(ctx context.Context, diff --git a/internal/mocks/session_test.go b/internal/mocks/session_test.go index 3a98ee5ff9..fd43eccff9 100644 --- a/internal/mocks/session_test.go +++ b/internal/mocks/session_test.go @@ -326,11 +326,11 @@ func TestSession(t *testing.T) { t.Run("NewSubmitter", func(t *testing.T) { expected := errors.New("mocked err") s := &Session{ - MockNewSubmitter: func(ctx context.Context) (model.Submitter, error) { + MockNewSubmitter: func(ctx context.Context, useAuth bool) (model.Submitter, error) { return nil, expected }, } - out, err := s.NewSubmitter(context.Background()) + out, err := s.NewSubmitter(context.Background(), false) if !errors.Is(err, expected) { t.Fatal("unexpected err") } diff --git a/internal/model/measurement.go b/internal/model/measurement.go index 3f13106f5e..6990f5b492 100644 --- a/internal/model/measurement.go +++ b/internal/model/measurement.go @@ -118,8 +118,11 @@ type Measurement struct { // ProbeNetworkName contains the probe network name ProbeNetworkName string `json:"probe_network_name"` - // ReportID contains the report ID - ReportID string `json:"report_id"` + // ProbeID contains the anonymous probe identifier + ProbeID string `json:"probe_id,omitempty"` + + // ReportID contains the report ID. + ReportID string `json:"report_id,omitempty"` // ResolverASN is the ASN of the resolver ResolverASN string `json:"resolver_asn"` diff --git a/internal/model/ooapi.go b/internal/model/ooapi.go index 8a0449568f..96479fdc45 100644 --- a/internal/model/ooapi.go +++ b/internal/model/ooapi.go @@ -313,6 +313,22 @@ type OOAPICollectorUpdateResponse struct { MeasurementUID string `json:"measurement_uid"` } +// OOAPISubmitMeasurementRequest is a request for the submit measurement API. +type OOAPISubmitMeasurementRequest struct { + Format string `json:"format"` + Content string `json:"content"` + Nym *string `json:"nym"` + ZKPRequest *string `json:"zkp_request"` + ManifestVersion *string `json:"manifest_version"` + ProtocolVersion *string `json:"protocol_version"` +} + +// OOAPISubmitMeasurementResponse is the response from the submit measurement API. +type OOAPISubmitMeasurementResponse struct { + // MeasurementUID is the measurement UID. + MeasurementUID string `json:"measurement_uid"` +} + // OOAPILoginCredentials contains the login credentials type OOAPILoginCredentials struct { Username string `json:"username"` @@ -420,3 +436,51 @@ type OOAPIRegisterRequest struct { type OOAPIRegisterResponse struct { ClientID string `json:"client_id"` } + +// OOAPISubmissionPolicyParams holds the credential constraints of a policy. +type OOAPISubmissionPolicyParams struct { + // Age is the [min, max] allowed credential age window. + Age []uint32 `json:"age"` + + // MinMeasurementCount is the minimum allowed measurement count. + MinMeasurementCount uint32 `json:"min_measurement_count"` +} + +// OOAPISubmissionPolicy is a single submission policy in an [OOAPIManifestBody]. +type OOAPISubmissionPolicy struct { + Policy OOAPISubmissionPolicyParams `json:"policy"` + Match OOAPISubmissionPolicyMatch `json:"match"` +} + +// OOAPISubmissionPolicyMatch selects which probes a policy applies to. +type OOAPISubmissionPolicyMatch struct { + ProbeCC string `json:"probe_cc"` + ProbeASN string `json:"probe_asn"` +} + +// OOAPIManifestBody is the body of an [OOAPIManifest]. +type OOAPIManifestBody struct { + // NymScope is the pseudonym scope template, e.g. "ooni.org/{probe_cc}/{probe_asn}". + NymScope string `json:"nym_scope"` + + // SubmissionPolicy is the list of submission policies. + SubmissionPolicy []OOAPISubmissionPolicy `json:"submission_policy"` + + // PublicParameters is the base64-encoded credential public parameters. + PublicParameters string `json:"public_parameters"` +} + +// OOAPIManifestMeta is the metadata of an [OOAPIManifest]. +type OOAPIManifestMeta struct { + Version string `json:"version"` + LastModificationDate string `json:"last_modification_date"` + ManifestURL string `json:"manifest_url"` + LibraryVersion string `json:"library_version"` + ProtocolVersion string `json:"protocol_version"` +} + +// OOAPIManifest is the response from the anonymous-credentials manifest API. +type OOAPIManifest struct { + Manifest OOAPIManifestBody `json:"manifest"` + Meta OOAPIManifestMeta `json:"meta"` +} diff --git a/internal/oonirun/experiment.go b/internal/oonirun/experiment.go index c28f451d71..212aaa6a58 100644 --- a/internal/oonirun/experiment.go +++ b/internal/oonirun/experiment.go @@ -51,6 +51,10 @@ type Experiment struct { // NoCollector OPTIONALLY indicates we should not be using any collector. NoCollector bool + // NoCredentials OPTIONALLY indicates we should submit without an anonymous + // credential (i.e. anonymously). + NoCredentials bool + // NoJSON OPTIONALLY indicates we don't want to save measurements to a JSON file. NoJSON bool @@ -202,6 +206,9 @@ func (ed *Experiment) newSubmitter(ctx context.Context) (model.Submitter, error) Enabled: !ed.NoCollector, Session: ed.Session, Logger: ed.Session.Logger(), + // oonirun is only used by the miniooni CLI, which submits with a + // credential by default; oonimkall does not go through oonirun. + UseAuth: !ed.NoCredentials, }) } diff --git a/internal/oonirun/link.go b/internal/oonirun/link.go index 3321db03b0..e00f65427b 100644 --- a/internal/oonirun/link.go +++ b/internal/oonirun/link.go @@ -36,6 +36,10 @@ type LinkConfig struct { // NoCollector OPTIONALLY indicates we should not be using any collector. NoCollector bool + // NoCredentials OPTIONALLY indicates we should submit without an anonymous + // credential. + NoCredentials bool + // NoJSON OPTIONALLY indicates we don't want to save measurements to a JSON file. NoJSON bool diff --git a/internal/oonirun/submitter.go b/internal/oonirun/submitter.go index 326619ec91..7b81ee6f30 100644 --- a/internal/oonirun/submitter.go +++ b/internal/oonirun/submitter.go @@ -15,7 +15,7 @@ type Submitter = model.Submitter // SubmitterSession is the Submitter's view of the Session. type SubmitterSession interface { // NewSubmitter creates a new probeservices Submitter. - NewSubmitter(ctx context.Context) (Submitter, error) + NewSubmitter(ctx context.Context, useAuth bool) (Submitter, error) } // SubmitterConfig contains settings for NewSubmitter. @@ -28,6 +28,8 @@ type SubmitterConfig struct { // Logger is the logger to be used. Logger model.Logger + + UseAuth bool } // NewSubmitter creates a new submitter instance. Depending on @@ -37,7 +39,7 @@ func NewSubmitter(ctx context.Context, config SubmitterConfig) (Submitter, error if !config.Enabled { return stubSubmitter{}, nil } - subm, err := config.Session.NewSubmitter(ctx) + subm, err := config.Session.NewSubmitter(ctx, config.UseAuth) if err != nil { return nil, err } diff --git a/internal/oonirun/submitter_test.go b/internal/oonirun/submitter_test.go index 133545e0d6..a30b3babd0 100644 --- a/internal/oonirun/submitter_test.go +++ b/internal/oonirun/submitter_test.go @@ -47,7 +47,7 @@ type FakeSubmitterSession struct { Submitter Submitter } -func (fse FakeSubmitterSession) NewSubmitter(ctx context.Context) (Submitter, error) { +func (fse FakeSubmitterSession) NewSubmitter(ctx context.Context, useAuth bool) (Submitter, error) { return fse.Submitter, fse.Error } diff --git a/internal/oonirun/v1.go b/internal/oonirun/v1.go index 8a5edeaecf..7f3e0173bf 100644 --- a/internal/oonirun/v1.go +++ b/internal/oonirun/v1.go @@ -79,6 +79,7 @@ func v1Measure(ctx context.Context, config *LinkConfig, URL string) error { MaxRuntime: config.MaxRuntime, Name: name, NoCollector: config.NoCollector, + NoCredentials: config.NoCredentials, NoJSON: config.NoJSON, Random: config.Random, ReportFile: config.ReportFile, diff --git a/internal/oonirun/v2.go b/internal/oonirun/v2.go index c2a3ca065d..aa71f90235 100644 --- a/internal/oonirun/v2.go +++ b/internal/oonirun/v2.go @@ -198,6 +198,7 @@ func V2MeasureDescriptor(ctx context.Context, config *LinkConfig, desc *V2Descri MaxRuntime: config.MaxRuntime, Name: nettest.TestName, NoCollector: config.NoCollector, + NoCredentials: config.NoCredentials, NoJSON: config.NoJSON, Random: config.Random, ReportFile: config.ReportFile, diff --git a/internal/oonirun/v2_test.go b/internal/oonirun/v2_test.go index d1f11214d0..b8cd4b05b3 100644 --- a/internal/oonirun/v2_test.go +++ b/internal/oonirun/v2_test.go @@ -484,7 +484,7 @@ func TestV2MeasureDescriptor(t *testing.T) { // Note: the convention is that we do not submit experiment results when the // experiment measurement function returns a non-nil error, since such an error // represents a fundamental failure in setting up the experiment - sess.MockNewSubmitter = func(ctx context.Context) (model.Submitter, error) { + sess.MockNewSubmitter = func(ctx context.Context, useAuth bool) (model.Submitter, error) { subm := &mocks.Submitter{ MockSubmit: func(ctx context.Context, m *model.Measurement) (string, error) { panic("should not be called") diff --git a/internal/probeservices/manifest.go b/internal/probeservices/manifest.go new file mode 100644 index 0000000000..fb01462635 --- /dev/null +++ b/internal/probeservices/manifest.go @@ -0,0 +1,54 @@ +package probeservices + +import ( + "context" + "errors" + + "github.com/ooni/probe-cli/v3/internal/httpclientx" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/urlx" +) + +// ErrNoMatchingPolicy +var ErrNoMatchingPolicy = errors.New("no submission policy matches this probe") + +// GetManifest queries the manifest API and returns the parsed manifest +func (c Client) GetManifest(ctx context.Context) (*model.OOAPIManifest, error) { + URL, err := urlx.ResolveReference(c.BaseURL, "/api/v1/manifest", "") + if err != nil { + return nil, err + } + + return httpclientx.GetJSON[*model.OOAPIManifest]( + ctx, + httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), + &httpclientx.Config{ + Client: c.HTTPClient, + Logger: c.Logger, + UserAgent: c.UserAgent, + }) +} + +// matchField reports whether a manifest match value matches a probe value. +func matchField(pattern, value string) bool { + return pattern == "*" || pattern == value +} + +// GetRangesFromPolicy returns the age range and the minimum measurement count +// from the first submission policy in the manifest whose match clause matches +// the given probeCC/probeASN. +func GetRangesFromPolicy( + manifest *model.OOAPIManifest, + probeCC, probeASN string, +) (ageRange [2]uint32, minMeasurementCount uint32, err error) { + for _, item := range manifest.Manifest.SubmissionPolicy { + if !matchField(item.Match.ProbeCC, probeCC) || !matchField(item.Match.ProbeASN, probeASN) { + continue + } + if len(item.Policy.Age) < 2 { + return ageRange, 0, errors.New("probeservices: submission policy has an invalid age range") + } + return [2]uint32{item.Policy.Age[0], item.Policy.Age[1]}, item.Policy.MinMeasurementCount, nil + } + return ageRange, 0, ErrNoMatchingPolicy +} diff --git a/internal/probeservices/manifest_test.go b/internal/probeservices/manifest_test.go new file mode 100644 index 0000000000..0ccacfa20b --- /dev/null +++ b/internal/probeservices/manifest_test.go @@ -0,0 +1,401 @@ +package probeservices + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/url" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/mocks" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/must" + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/testingx" +) + +// sampleManifest +const sampleManifest = `{ + "manifest": { + "nym_scope": "ooni.org/{probe_cc}/{probe_asn}", + "submission_policy": [ + { + "policy": { + "age": [2461110, 2826140], + "min_measurement_count": 0 + }, + "match": { + "probe_cc": "*", + "probe_asn": "*" + } + } + ], + "public_parameters": "AdqzxWc0xFMFlXygX+KfKxRGy6EE" + }, + "meta": { + "version": "TjxIhQyJHRZsqmidU_coSEl2dZUiBGvL", + "protocol_version": "0.1.0" + } +}` + +// newSampleManifest returns the same manifest as [sampleManifest] as a struct, +// so tests can serve it from a local server and diff the round trip. +func newSampleManifest() *model.OOAPIManifest { + return &model.OOAPIManifest{ + Manifest: model.OOAPIManifestBody{ + NymScope: "ooni.org/{probe_cc}/{probe_asn}", + SubmissionPolicy: []model.OOAPISubmissionPolicy{{ + Policy: model.OOAPISubmissionPolicyParams{ + Age: []uint32{2461110, 2826140}, + MinMeasurementCount: 0, + }, + Match: model.OOAPISubmissionPolicyMatch{ + ProbeCC: "*", + ProbeASN: "*", + }, + }}, + PublicParameters: "AdqzxWc0xFMFlXygX+KfKxRGy6EE", + }, + Meta: model.OOAPIManifestMeta{ + Version: "TjxIhQyJHRZsqmidU_coSEl2dZUiBGvL", + ProtocolVersion: "0.1.0", + }, + } +} + +func parseSampleManifest(t *testing.T) *model.OOAPIManifest { + var manifest model.OOAPIManifest + if err := json.Unmarshal([]byte(sampleManifest), &manifest); err != nil { + t.Fatal(err) + } + return &manifest +} + +func TestManifestParsing(t *testing.T) { + manifest := parseSampleManifest(t) + if manifest.Meta.Version != "TjxIhQyJHRZsqmidU_coSEl2dZUiBGvL" { + t.Fatal("unexpected version", manifest.Meta.Version) + } + if manifest.Manifest.PublicParameters == "" { + t.Fatal("missing public parameters") + } + if len(manifest.Manifest.SubmissionPolicy) != 1 { + t.Fatal("unexpected policy count") + } + if got := manifest.Manifest.SubmissionPolicy[0].Policy.Age; len(got) != 2 { + t.Fatal("unexpected age length", got) + } +} + +func TestGetManifest(t *testing.T) { + t.Run("is working as intended with the real backend", func(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + + // create client + client := newclient() + + // issue the request + manifest, err := client.GetManifest(context.Background()) + + // we do not expect an error here + if err != nil { + t.Fatal(err) + } + + // we expect a usable manifest + if manifest.Meta.Version == "" { + t.Fatal("expected a non-empty manifest version") + } + if manifest.Manifest.PublicParameters == "" { + t.Fatal("expected non-empty public parameters") + } + if len(manifest.Manifest.SubmissionPolicy) < 1 { + t.Fatal("expected at least one submission policy") + } + }) + + t.Run("is working as intended with a local test server", func(t *testing.T) { + // this is what we expect to receive + expect := newSampleManifest() + + // create quick and dirty server to serve the response + srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + runtimex.Assert(r.Method == http.MethodGet, "invalid method") + runtimex.Assert(r.URL.Path == "/api/v1/manifest", "invalid URL path") + w.Write(must.MarshalJSON(expect)) + })) + defer srv.Close() + + // create a probeservices client + client := newclient() + + // override the HTTP client + client.HTTPClient = &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + URL := runtimex.Try1(url.Parse(srv.URL)) + req.URL.Scheme = URL.Scheme + req.URL.Host = URL.Host + return http.DefaultClient.Do(req) + }, + MockCloseIdleConnections: func() { + http.DefaultClient.CloseIdleConnections() + }, + } + + // issue the GET request + manifest, err := client.GetManifest(context.Background()) + + // we do not expect an error + if err != nil { + t.Fatal(err) + } + + // we expect to see exactly what the server sent + if diff := cmp.Diff(expect, manifest); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("we can use cloudfronting", func(t *testing.T) { + // this is what we expect to receive + expect := newSampleManifest() + + // create quick and dirty server to serve the response + srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + runtimex.Assert(r.Host == "www.cloudfront.com", "invalid r.Host") + runtimex.Assert(r.Method == http.MethodGet, "invalid method") + runtimex.Assert(r.URL.Path == "/api/v1/manifest", "invalid URL path") + w.Write(must.MarshalJSON(expect)) + })) + defer srv.Close() + + // create a probeservices client + client := newclient() + + // make sure we're using cloudfronting + client.Host = "www.cloudfront.com" + + // override the HTTP client + client.HTTPClient = &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + URL := runtimex.Try1(url.Parse(srv.URL)) + req.URL.Scheme = URL.Scheme + req.URL.Host = URL.Host + return http.DefaultClient.Do(req) + }, + MockCloseIdleConnections: func() { + http.DefaultClient.CloseIdleConnections() + }, + } + + // issue the GET request + manifest, err := client.GetManifest(context.Background()) + + // we do not expect an error + if err != nil { + t.Fatal(err) + } + + // we expect to see exactly what the server sent + if diff := cmp.Diff(expect, manifest); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("reports an error when the connection is reset", func(t *testing.T) { + // create quick and dirty server to serve the response + srv := testingx.MustNewHTTPServer(testingx.HTTPHandlerReset()) + defer srv.Close() + + // create a probeservices client + client := newclient() + + // override the HTTP client + client.HTTPClient = &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + URL := runtimex.Try1(url.Parse(srv.URL)) + req.URL.Scheme = URL.Scheme + req.URL.Host = URL.Host + return http.DefaultClient.Do(req) + }, + MockCloseIdleConnections: func() { + http.DefaultClient.CloseIdleConnections() + }, + } + + // issue the GET request + manifest, err := client.GetManifest(context.Background()) + + // we do expect an error + if !errors.Is(err, netxlite.ECONNRESET) { + t.Fatal("unexpected error", err) + } + + // we expect a nil manifest + if manifest != nil { + t.Fatal("expected nil manifest") + } + }) + + t.Run("reports an error when the response is not JSON parsable", func(t *testing.T) { + // create quick and dirty server to serve the response + srv := testingx.MustNewHTTPServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{`)) + })) + defer srv.Close() + + // create a probeservices client + client := newclient() + + // override the HTTP client + client.HTTPClient = &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + URL := runtimex.Try1(url.Parse(srv.URL)) + req.URL.Scheme = URL.Scheme + req.URL.Host = URL.Host + return http.DefaultClient.Do(req) + }, + MockCloseIdleConnections: func() { + http.DefaultClient.CloseIdleConnections() + }, + } + + // issue the GET request + manifest, err := client.GetManifest(context.Background()) + + // we do expect an error + if err == nil || err.Error() != "unexpected end of JSON input" { + t.Fatal("unexpected error", err) + } + + // we expect a nil manifest + if manifest != nil { + t.Fatal("expected nil manifest") + } + }) + + t.Run("correctly handles the case where the URL is unparseable", func(t *testing.T) { + // create a probeservices client + client := newclient() + + // override the URL to be unparseable + client.BaseURL = "\t\t\t" + + // issue the GET request + manifest, err := client.GetManifest(context.Background()) + + // we do expect an error + if err == nil || err.Error() != `parse "\t\t\t": net/url: invalid control character in URL` { + t.Fatal("unexpected error", err) + } + + // we expect a nil manifest + if manifest != nil { + t.Fatal("expected nil manifest") + } + }) +} + +func TestGetRangesFromPolicy(t *testing.T) { + + t.Run("matches a wildcard policy for any probe", func(t *testing.T) { + manifest := parseSampleManifest(t) + ageRange, minMeasurementCount, err := GetRangesFromPolicy(manifest, "IT", "AS117") + if err != nil { + t.Fatal(err) + } + if ageRange[0] != 2461110 || ageRange[1] != 2826140 { + t.Fatal("unexpected age range", ageRange) + } + if minMeasurementCount != 0 { + t.Fatal("unexpected min measurement count", minMeasurementCount) + } + }) + + t.Run("matches an exact probe and rejects a different one", func(t *testing.T) { + manifest := parseSampleManifest(t) + manifest.Manifest.SubmissionPolicy[0].Match = model.OOAPISubmissionPolicyMatch{ + ProbeCC: "IT", + ProbeASN: "AS117", + } + + if _, _, err := GetRangesFromPolicy(manifest, "IT", "AS117"); err != nil { + t.Fatal("expected a match for the exact probe", err) + } + + if _, _, err := GetRangesFromPolicy(manifest, "US", "AS117"); !errors.Is(err, ErrNoMatchingPolicy) { + t.Fatalf("not the error we expected: %+v", err) + } + + if _, _, err := GetRangesFromPolicy(manifest, "IT", "AS30722"); !errors.Is(err, ErrNoMatchingPolicy) { + t.Fatalf("not the error we expected: %+v", err) + } + }) + + t.Run("matches when only one field is wildcarded", func(t *testing.T) { + manifest := parseSampleManifest(t) + manifest.Manifest.SubmissionPolicy[0].Match = model.OOAPISubmissionPolicyMatch{ + ProbeCC: "IT", + ProbeASN: "*", + } + if _, _, err := GetRangesFromPolicy(manifest, "IT", "AS999"); err != nil { + t.Fatal("expected a match for CC-exact + ASN-wildcard", err) + } + if _, _, err := GetRangesFromPolicy(manifest, "US", "AS999"); !errors.Is(err, ErrNoMatchingPolicy) { + t.Fatalf("not the error we expected: %+v", err) + } + }) + + t.Run("returns the first matching policy (highest priority first)", func(t *testing.T) { + manifest := parseSampleManifest(t) + // Prepend a more specific policy so it should win over the wildcard one. + specific := model.OOAPISubmissionPolicy{ + Policy: model.OOAPISubmissionPolicyParams{ + Age: []uint32{100, 200}, + MinMeasurementCount: 5, + }, + Match: model.OOAPISubmissionPolicyMatch{ProbeCC: "IT", ProbeASN: "AS117"}, + } + manifest.Manifest.SubmissionPolicy = append( + []model.OOAPISubmissionPolicy{specific}, manifest.Manifest.SubmissionPolicy...) + + ageRange, minMeasurementCount, err := GetRangesFromPolicy(manifest, "IT", "AS117") + if err != nil { + t.Fatal(err) + } + if ageRange[0] != 100 || ageRange[1] != 200 || minMeasurementCount != 5 { + t.Fatal("expected the first (specific) policy to win", ageRange, minMeasurementCount) + } + + // A probe not matching the specific policy falls through to the wildcard. + ageRange, minMeasurementCount, err = GetRangesFromPolicy(manifest, "US", "AS999") + if err != nil { + t.Fatal(err) + } + if ageRange[0] != 2461110 || ageRange[1] != 2826140 || minMeasurementCount != 0 { + t.Fatal("expected the wildcard policy for a non-specific probe", ageRange, minMeasurementCount) + } + }) + + t.Run("returns no matching policy error when the list is empty", func(t *testing.T) { + manifest := parseSampleManifest(t) + manifest.Manifest.SubmissionPolicy = nil + if _, _, err := GetRangesFromPolicy(manifest, "IT", "AS117"); !errors.Is(err, ErrNoMatchingPolicy) { + t.Fatalf("not the error we expected: %+v", err) + } + }) + + t.Run("errors when the matched policy has an invalid age range", func(t *testing.T) { + manifest := parseSampleManifest(t) + manifest.Manifest.SubmissionPolicy[0].Policy.Age = []uint32{123} // too short + _, _, err := GetRangesFromPolicy(manifest, "IT", "AS117") + if err == nil || errors.Is(err, ErrNoMatchingPolicy) { + t.Fatal("expected an invalid-age-range error, got", err) + } + }) +} diff --git a/internal/probeservices/submit.go b/internal/probeservices/submit.go new file mode 100644 index 0000000000..e10056aa20 --- /dev/null +++ b/internal/probeservices/submit.go @@ -0,0 +1,41 @@ +package probeservices + +import ( + "context" + "encoding/json" + + "github.com/ooni/probe-cli/v3/internal/httpclientx" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/urlx" +) + +// Submit submits a measurement to the submit_measurement endpoint without a +// credential and returns the measurement UID. +func (c Client) Submit(ctx context.Context, m *model.Measurement) (string, error) { + content, err := json.Marshal(m) + if err != nil { + return "", err + } + + URL, err := urlx.ResolveReference(c.BaseURL, "/api/v1/submit_measurement", "") + if err != nil { + return "", err + } + + resp, err := httpclientx.PostJSON[*model.OOAPISubmitMeasurementRequest, *model.OOAPISubmitMeasurementResponse]( + ctx, + httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), + &model.OOAPISubmitMeasurementRequest{Format: "json", Content: string(content)}, + &httpclientx.Config{ + Client: c.HTTPClient, + Logger: c.Logger, + UserAgent: c.UserAgent, + }, + ) + if err != nil { + return "", err + } + + c.Logger.Infof("Measurement URL: https://explorer.ooni.org/m/%s", resp.MeasurementUID) + return resp.MeasurementUID, nil +} diff --git a/internal/userauth/.gitignore b/internal/userauth/.gitignore new file mode 100644 index 0000000000..c3af857904 --- /dev/null +++ b/internal/userauth/.gitignore @@ -0,0 +1 @@ +lib/ diff --git a/internal/userauth/credstore.go b/internal/userauth/credstore.go new file mode 100644 index 0000000000..bf9606e386 --- /dev/null +++ b/internal/userauth/credstore.go @@ -0,0 +1,56 @@ +package userauth + +import ( + "encoding/json" + + "github.com/ooni/probe-cli/v3/internal/model" +) + +// credStoreKey is the key-value store key under which we persist the credential. +const credStoreKey = "userauth.state" + +// StoredCredential is the persisted anonymous credential together with the +// manifest metadata it was issued against. +type StoredCredential struct { + // Credential is the base64-encoded credential. + Credential string `json:"credential"` + + // ManifestVersion is the manifest version the credential was issued for. + ManifestVersion string `json:"manifest_version"` + + // PublicParams is the base64-encoded public parameters of that manifest. + PublicParams string `json:"public_params"` +} + +// CredStore persists a [StoredCredential] in a generic key-value store, +// mirroring probeservices.StateFile. +type CredStore struct { + Store model.KeyValueStore +} + +// NewCredStore creates a new CredStore backed by the given key-value store. +func NewCredStore(kvstore model.KeyValueStore) *CredStore { + return &CredStore{Store: kvstore} +} + +// Get returns the stored credential. When there is no credential yet, or the +// store cannot be read, it returns a zero StoredCredential +func (cs *CredStore) Get() (cred StoredCredential) { + value, err := cs.Store.Get(credStoreKey) + if err != nil { + return StoredCredential{} + } + if err := json.Unmarshal(value, &cred); err != nil { + return StoredCredential{} + } + return cred +} + +// Set persists the given credential. +func (cs *CredStore) Set(cred StoredCredential) error { + data, err := json.Marshal(cred) + if err != nil { + return err + } + return cs.Store.Set(credStoreKey, data) +} diff --git a/internal/userauth/credstore_test.go b/internal/userauth/credstore_test.go new file mode 100644 index 0000000000..b755f45a57 --- /dev/null +++ b/internal/userauth/credstore_test.go @@ -0,0 +1,30 @@ +package userauth + +import ( + "testing" + + "github.com/ooni/probe-cli/v3/internal/kvstore" +) + +func TestCredStoreRoundTrip(t *testing.T) { + store := NewCredStore(&kvstore.Memory{}) + + // An empty store returns a zero credential. + if got := store.Get(); got.Credential != "" { + t.Fatal("expected empty credential from empty store, got", got.Credential) + } + + cred := StoredCredential{ + Credential: "base64credential", + ManifestVersion: "v1", + PublicParams: "base64params", + } + if err := store.Set(cred); err != nil { + t.Fatal(err) + } + + got := store.Get() + if got != cred { + t.Fatalf("round-trip mismatch: got %+v want %+v", got, cred) + } +} diff --git a/internal/userauth/ooniprobe_userauth.h b/internal/userauth/ooniprobe_userauth.h deleted file mode 100644 index dc7f8cebed..0000000000 --- a/internal/userauth/ooniprobe_userauth.h +++ /dev/null @@ -1,63 +0,0 @@ -#include -#include -#include -#include - -typedef struct ClientResponse { - char *json; - char *error; -} ClientResponse; - -/** - * Free memory allocated by ClientResponse - * - * # Safety - * This function must be called exactly once for each ClientResponse - * returned by other FFI functions to avoid memory leaks. - */ -void client_response_free(struct ClientResponse response); - -/** - * Perform HTTP GET request - * - * # Safety - * - `url` must be a valid null-terminated C string - * - Caller must call `client_response_free` on the returned value - */ -struct ClientResponse client_get(const char *url); - -/** - * Perform HTTP POST request - * - * # Safety - * - `url` and `payload` must be valid null-terminated C strings - * - Caller must call `client_response_free` on the returned value - */ -struct ClientResponse client_post(const char *url, const char *payload); - -/** - * Register a user and obtain a credential - * - * # Safety - * - All parameters must be valid null-terminated C strings - * - Caller must call `client_response_free` on the returned value - */ -struct ClientResponse userauth_register(const char *url, - const char *public_params, - const char *manifest_version); - -/** - * Submit user credentials with measurement data - * - * # Safety - * - All parameters must be valid null-terminated C strings - * - `credential_b64` must be a valid base64-encoded credential - * - `public_params` must be valid base64 public parameters - * - Caller must call `client_response_free` on the returned value - */ -struct ClientResponse userauth_submit(const char *url, - const char *credential_b64, - const char *public_params, - const char *probe_cc, - const char *probe_asn, - const char *manifest_version); diff --git a/internal/userauth/submitter.go b/internal/userauth/submitter.go new file mode 100644 index 0000000000..c66c28057c --- /dev/null +++ b/internal/userauth/submitter.go @@ -0,0 +1,174 @@ +package userauth + +import ( + "context" + "encoding/json" + "errors" + "strings" + + "github.com/ooni/probe-cli/v3/internal/model" +) + +const ( + signCredentialPath = "/api/v1/sign_credential" + submitMeasurementPath = "/api/v1/submit_measurement" +) + +// CredentialSubmitter implements [model.Submitter]. +type CredentialSubmitter struct { + registerURL string + submitURL string + probeCC string + probeASN string + publicParams string + manifestVersion string + ageRange ParamRange + measurementCountRange ParamRange + proxy string + timeout float32 + store *CredStore + logger model.Logger + fallback model.Submitter + userAgent string +} + +var _ model.Submitter = &CredentialSubmitter{} + +// CredentialSubmitterConfig carries the dependencies for [NewCredentialSubmitter]. +type CredentialSubmitterConfig struct { + // BaseURL is the backend base URL (e.g. https://api.ooni.io). The register + // and submit endpoints are resolved relative to it. + BaseURL string + + // ProbeCC and ProbeASN identify the probe (the latter in "ASxxxx" form). + ProbeCC string + ProbeASN string + + // PublicParams, ManifestVersion, AgeRange, MeasurementCountRange come from + // the manifest and its matched submission policy. + PublicParams string + ManifestVersion string + AgeRange ParamRange + MeasurementCountRange ParamRange + + // Proxy is the proxy URL to route FFI requests through ("" means none). + Proxy string + + // Timeout is the per-request timeout in seconds (<= 0 means the default). + Timeout float32 + + // Store persists the credential across runs. + Store *CredStore + + // Logger is the logger to use. + Logger model.Logger + + // Fallback is the legacy submitter used when credential submission fails. + Fallback model.Submitter + + // UserAgent is the user-agent string used with by the http client + UserAgent string +} + +// NewCredentialSubmitter builds a [CredentialSubmitter], registering for a fresh +// credential when there is no usable stored one. +func NewCredentialSubmitter( + ctx context.Context, config CredentialSubmitterConfig) (*CredentialSubmitter, error) { + baseURL := strings.TrimRight(config.BaseURL, "/") + cs := &CredentialSubmitter{ + registerURL: baseURL + signCredentialPath, + submitURL: baseURL + submitMeasurementPath, + probeCC: config.ProbeCC, + probeASN: config.ProbeASN, + publicParams: config.PublicParams, + manifestVersion: config.ManifestVersion, + ageRange: config.AgeRange, + measurementCountRange: config.MeasurementCountRange, + proxy: config.Proxy, + timeout: config.Timeout, + store: config.Store, + logger: config.Logger, + fallback: config.Fallback, + userAgent: config.UserAgent, + } + + // Reuse the stored credential when it matches the current manifest version; + // otherwise register for a new one. + stored := cs.store.Get() + if stored.Credential != "" && stored.ManifestVersion == cs.manifestVersion { + return cs, nil + } + + cs.logger.Info("userauth: registering for a new anonymous credential") + credential, err := Register(cs.registerURL, cs.publicParams, cs.manifestVersion, cs.proxy, cs.userAgent, cs.timeout) + if err != nil { + return nil, err + } + if err := cs.store.Set(StoredCredential{ + Credential: credential, + ManifestVersion: cs.manifestVersion, + PublicParams: cs.publicParams, + }); err != nil { + return nil, err + } + return cs, nil +} + +// submitWithCredential performs the authenticated submission and credential +// rotation. It returns the measurement UID on success. +func (cs *CredentialSubmitter) submitWithCredential(m *model.Measurement) (string, error) { + stored := cs.store.Get() + if stored.Credential == "" { + return "", errors.New("userauth: no stored credential") + } + + // Stamp the measurement with the probe id derived from the credential, so the + // uploaded measurement carries the anonymous identity. + probeID, err := ProbeID(stored.Credential, cs.probeASN, cs.probeCC) + if err != nil { + return "", err + } + m.ProbeID = probeID + + content, err := json.Marshal(m) + if err != nil { + return "", err + } + + config := &CredentialConfig{ + Credential: stored.Credential, + PublicParams: cs.publicParams, + ManifestVersion: cs.manifestVersion, + AgeRange: cs.ageRange, + MeasurementCountRange: cs.measurementCountRange, + } + + result, err := Submit(cs.submitURL, string(content), cs.probeCC, cs.probeASN, + cs.proxy, cs.userAgent, cs.timeout, config) + if err != nil { + return "", err + } + + // Persist the rotated credential for the next submission. + if err := cs.store.Set(StoredCredential{ + Credential: result.Credential, + ManifestVersion: cs.manifestVersion, + PublicParams: cs.publicParams, + }); err != nil { + return "", err + } + + cs.logger.Infof("Measurement URL: https://explorer.ooni.org/m/%s", result.MeasurementUID) + return result.MeasurementUID, nil +} + +// Submit implements [model.Submitter]. It submits the measurement using the +// stored credential and persists the rotated credential. +func (cs *CredentialSubmitter) Submit(ctx context.Context, m *model.Measurement) (string, error) { + uid, err := cs.submitWithCredential(m) + if err != nil { + cs.logger.Warnf("userauth: credential submission failed, falling back to collector: %s", err.Error()) + return cs.fallback.Submit(ctx, m) + } + return uid, nil +} diff --git a/internal/userauth/userauth.go b/internal/userauth/userauth.go index ab5d8c37c4..bc9fcdacb8 100644 --- a/internal/userauth/userauth.go +++ b/internal/userauth/userauth.go @@ -1,134 +1,40 @@ -//go:build ooni_userauth - package userauth -// #cgo CFLAGS: -I${SRCDIR}/ffi -// #cgo linux,amd64 LDFLAGS: -L${SRCDIR}/lib/linux/amd64 -looniprobe_userauth -ldl -lm -lpthread -// #cgo linux,arm64 LDFLAGS: -L${SRCDIR}/lib/linux/arm64 -looniprobe_userauth -ldl -lm -lpthread -// #cgo darwin,amd64 LDFLAGS: -L${SRCDIR}/lib/darwin/amd64 -looniprobe_userauth -framework CoreFoundation -framework Security -// #cgo darwin,arm64 LDFLAGS: -L${SRCDIR}/lib/darwin/arm64 -looniprobe_userauth -framework CoreFoundation -framework Security -// #cgo windows,amd64 LDFLAGS: -L${SRCDIR}/lib/windows/amd64 -looniprobe_userauth -lws2_32 -luserenv -lbcrypt -// #include -// #include "ooniprobe_userauth.h" -import "C" - -import ( - "encoding/json" - "errors" - "unsafe" -) - -// RegistrationResponse represents the result of user registration -type RegistrationResponse struct { - Credential string `json:"credential"` - EmissionDay int16 `json:"emission_day"` -} - -// SubmitResponse represents the result of submitting measurement -type SubmitResponse struct { - MeasurementUID string `json:"measurement_uid,omitempty"` - IsVerified bool `json:"is_verified"` - SubmitResponse string `json:"submit_response"` -} - -// HTTPGet performs an HTTP GET request -func HTTPGet(url string) (string, error) { - cURL := C.CString(url) - defer C.free(unsafe.Pointer(cURL)) - - resp := C.client_get(cURL) - defer C.client_response_free(resp) - - return parseResponse(resp) -} - -// HTTPPost performs an HTTP POST request -func HTTPPost(url, payload string) (string, error) { - cURL := C.CString(url) - defer C.free(unsafe.Pointer(cURL)) - - cPayload := C.CString(payload) - defer C.free(unsafe.Pointer(cPayload)) - - resp := C.client_post(cURL, cPayload) - defer C.client_response_free(resp) +import "errors" - return parseResponse(resp) -} - -// Register registers a new user and obtains a credential -func Register(url, publicParams, manifestVersion string) (*RegistrationResponse, error) { - cURL := C.CString(url) - defer C.free(unsafe.Pointer(cURL)) - - cPublicParams := C.CString(publicParams) - defer C.free(unsafe.Pointer(cPublicParams)) - - cManifestVersion := C.CString(manifestVersion) - defer C.free(unsafe.Pointer(cManifestVersion)) - - resp := C.userauth_register(cURL, cPublicParams, cManifestVersion) - defer C.client_response_free(resp) - - jsonStr, err := parseResponse(resp) - if err != nil { - return nil, err - } - - var result RegistrationResponse - if err := json.Unmarshal([]byte(jsonStr), &result); err != nil { - return nil, errors.New("failed to parse registration response: " + err.Error()) - } +// ErrUnavailable +var ErrUnavailable = errors.New("userauth: anonymous-credential FFI unavailable in this build") - return &result, nil +// ParamRange +type ParamRange struct { + Min uint32 `json:"min"` + Max uint32 `json:"max"` } -// Submit submits user credentials with measurement data -func Submit(url, credentialB64, publicParams, probeCC, probeASN, manifestVersion string) (*SubmitResponse, error) { - cURL := C.CString(url) - defer C.free(unsafe.Pointer(cURL)) +// CredentialConfig carries the stored credential together with the +// manifest-derived parameters +type CredentialConfig struct { + // Credential is the base64-encoded credential to authenticate with. + Credential string `json:"credential"` - cCredentialB64 := C.CString(credentialB64) - defer C.free(unsafe.Pointer(cCredentialB64)) + // PublicParams is the base64-encoded public parameters from the manifest. + PublicParams string `json:"public_params"` - cPublicParams := C.CString(publicParams) - defer C.free(unsafe.Pointer(cPublicParams)) + // ManifestVersion is the manifest version the credential belongs to. + ManifestVersion string `json:"manifest_version"` - cProbeCC := C.CString(probeCC) - defer C.free(unsafe.Pointer(cProbeCC)) + // AgeRange is the allowed credential age window. + AgeRange ParamRange `json:"age_range"` - cProbeASN := C.CString(probeASN) - defer C.free(unsafe.Pointer(cProbeASN)) - - cManifestVersion := C.CString(manifestVersion) - defer C.free(unsafe.Pointer(cManifestVersion)) - - resp := C.userauth_submit(cURL, cCredentialB64, cPublicParams, cProbeCC, cProbeASN, cManifestVersion) - defer C.client_response_free(resp) - - jsonStr, err := parseResponse(resp) - if err != nil { - return nil, err - } - - var result SubmitResponse - if err := json.Unmarshal([]byte(jsonStr), &result); err != nil { - return nil, errors.New("failed to parse submit response: " + err.Error()) - } - - return &result, nil + // MeasurementCountRange is the allowed measurement-count window. + MeasurementCountRange ParamRange `json:"measurement_count_range"` } -// parseResponse converts a C ClientResponse to Go types -func parseResponse(resp C.ClientResponse) (string, error) { - if resp.error != nil { - errStr := C.GoString(resp.error) - return "", errors.New(errStr) - } - - if resp.json != nil { - return C.GoString(resp.json), nil - } +// RotatedCredential is the outcome of a successful authenticated submission. +type RotatedCredential struct { + // Credential is the updated credential to persist for the next submission. + Credential string - return "", errors.New("empty response from FFI") + // MeasurementUID is the collector-assigned measurement identifier. + MeasurementUID string } diff --git a/internal/userauth/userauth_cgo.go b/internal/userauth/userauth_cgo.go new file mode 100644 index 0000000000..49b5a0181e --- /dev/null +++ b/internal/userauth/userauth_cgo.go @@ -0,0 +1,204 @@ +//go:build (cgo && linux && !android && (amd64 || arm64 || arm || 386)) || (cgo && darwin && !ios && (amd64 || arm64)) || (cgo && windows && (amd64 || 386)) + +package userauth + +// #cgo CFLAGS: -I${SRCDIR}/lib/include +// #cgo linux,amd64 LDFLAGS: -L${SRCDIR}/lib/linux/x86_64 -luniffi_ooniprobe -ldl -lm -lpthread +// #cgo linux,arm64 LDFLAGS: -L${SRCDIR}/lib/linux/aarch64 -luniffi_ooniprobe -ldl -lm -lpthread +// #cgo linux,arm LDFLAGS: -L${SRCDIR}/lib/linux/arm -luniffi_ooniprobe -ldl -lm -lpthread +// #cgo linux,386 LDFLAGS: -L${SRCDIR}/lib/linux/x86 -luniffi_ooniprobe -ldl -lm -lpthread +// #cgo darwin,amd64 LDFLAGS: -L${SRCDIR}/lib/macos/x86_64 -luniffi_ooniprobe -framework CoreFoundation -framework Security +// #cgo darwin,arm64 LDFLAGS: -L${SRCDIR}/lib/macos/aarch64 -luniffi_ooniprobe -framework CoreFoundation -framework Security +// #cgo windows,amd64 LDFLAGS: -L${SRCDIR}/lib/windows/x86_64 -luniffi_ooniprobe -lws2_32 -luserenv -lbcrypt +// #cgo windows,386 LDFLAGS: -L${SRCDIR}/lib/windows/x86 -luniffi_ooniprobe -lws2_32 -luserenv -lbcrypt +// #include +// #include "ooniprobe_userauth.h" +import "C" + +import ( + "encoding/json" + "errors" + "fmt" + "unsafe" +) + +// registerResult mirrors the JSON returned by the C `userauth_register`. +type registerResult struct { + Credential string `json:"credential"` + StatusCode int `json:"status_code"` + Body string `json:"body"` +} + +// submitResult mirrors the JSON returned by the C `userauth_submit`. The +// measurement UID is carried inside the raw collector response body. +type submitResult struct { + Credential string `json:"credential"` + StatusCode int `json:"status_code"` + Body string `json:"body"` +} + +// submitBody is the subset of the collector response body we care about. +type submitBody struct { + MeasurementUID string `json:"measurement_uid"` +} + +func httpStatusOK(code int) bool { + return code >= 200 && code < 300 +} + +// optionalCString returns a C string for s, or nil when s is empty. +// NOTE: The caller must free a non-nil return with C.free. +func optionalCString(s string) *C.char { + if s == "" { + return nil + } + return C.CString(s) +} + +// parseResponse converts a C ClientResponse into a Go (string, error). +func parseResponse(resp C.ClientResponse) (string, error) { + if resp.error != nil { + return "", errors.New(C.GoString(resp.error)) + } + if resp.json != nil { + return C.GoString(resp.json), nil + } + return "", errors.New("userauth: empty response from FFI") +} + +// Register registers a new user and returns the base64-encoded credential. +func Register(url, publicParams, manifestVersion, proxy, userAgent string, timeout float32) (string, error) { + cURL := C.CString(url) + defer C.free(unsafe.Pointer(cURL)) + + cPublicParams := C.CString(publicParams) + defer C.free(unsafe.Pointer(cPublicParams)) + + cManifestVersion := C.CString(manifestVersion) + defer C.free(unsafe.Pointer(cManifestVersion)) + + cProxy := optionalCString(proxy) + if cProxy != nil { + defer C.free(unsafe.Pointer(cProxy)) + } + + cUserAgent := optionalCString(userAgent) + if cUserAgent != nil { + defer C.free(unsafe.Pointer(cUserAgent)) + } + + resp := C.userauth_register(cURL, cPublicParams, cManifestVersion, cProxy, C.float(timeout), cUserAgent) + defer C.client_response_free(resp) + + jsonStr, err := parseResponse(resp) + if err != nil { + return "", err + } + + var result registerResult + if err := json.Unmarshal([]byte(jsonStr), &result); err != nil { + return "", fmt.Errorf("userauth: cannot parse registration response: %w", err) + } + if !httpStatusOK(result.StatusCode) || result.Credential == "" { + return "", fmt.Errorf("userauth: registration failed (status %d): %s", + result.StatusCode, result.Body) + } + return result.Credential, nil +} + +// Submit submits a measurement. +func Submit(url, content, probeCC, probeASN, proxy, userAgent string, timeout float32, + cfg *CredentialConfig) (RotatedCredential, error) { + cURL := C.CString(url) + defer C.free(unsafe.Pointer(cURL)) + + cContent := C.CString(content) + defer C.free(unsafe.Pointer(cContent)) + + cProbeCC := C.CString(probeCC) + defer C.free(unsafe.Pointer(cProbeCC)) + + cProbeASN := C.CString(probeASN) + defer C.free(unsafe.Pointer(cProbeASN)) + + cProxy := optionalCString(proxy) + if cProxy != nil { + defer C.free(unsafe.Pointer(cProxy)) + } + + cUserAgent := optionalCString(userAgent) + if cUserAgent != nil { + defer C.free(unsafe.Pointer(cUserAgent)) + } + + // A nil cConfig tells the FFI to use the anonymous submission path. + var cConfig *C.char + if cfg != nil { + raw, err := json.Marshal(cfg) + if err != nil { + return RotatedCredential{}, fmt.Errorf("userauth: cannot marshal credential config: %w", err) + } + cConfig = C.CString(string(raw)) + defer C.free(unsafe.Pointer(cConfig)) + } + + resp := C.userauth_submit(cURL, cContent, cProbeCC, cProbeASN, cProxy, C.float(timeout), cUserAgent, cConfig) + defer C.client_response_free(resp) + + jsonStr, err := parseResponse(resp) + if err != nil { + return RotatedCredential{}, err + } + + var result submitResult + if err := json.Unmarshal([]byte(jsonStr), &result); err != nil { + return RotatedCredential{}, fmt.Errorf("userauth: cannot parse submit response: %w", err) + } + if !httpStatusOK(result.StatusCode) { + return RotatedCredential{}, fmt.Errorf("userauth: submission failed (status %d): %s", + result.StatusCode, result.Body) + } + // For an authenticated submission we expect a rotated credential back. + if cfg != nil && result.Credential == "" { + return RotatedCredential{}, fmt.Errorf("userauth: authenticated submission returned no credential: %s", + result.Body) + } + + // The measurement UID lives in the collector response body; a parse failure + // here is non-fatal (the submission itself already succeeded). + var body submitBody + _ = json.Unmarshal([]byte(result.Body), &body) + + return RotatedCredential{ + Credential: result.Credential, + MeasurementUID: body.MeasurementUID, + }, nil +} + +// ProbeID derives the hex-encoded probe id for the given credential. +func ProbeID(credentialB64, probeASN, probeCC string) (string, error) { + cCredential := C.CString(credentialB64) + defer C.free(unsafe.Pointer(cCredential)) + + cProbeASN := C.CString(probeASN) + defer C.free(unsafe.Pointer(cProbeASN)) + + cProbeCC := C.CString(probeCC) + defer C.free(unsafe.Pointer(cProbeCC)) + + resp := C.get_probe_id(cCredential, cProbeASN, cProbeCC) + defer C.client_response_free(resp) + + jsonStr, err := parseResponse(resp) + if err != nil { + return "", err + } + + var result struct { + ProbeID string `json:"probe_id"` + } + if err := json.Unmarshal([]byte(jsonStr), &result); err != nil { + return "", fmt.Errorf("userauth: cannot parse probe id response: %w", err) + } + return result.ProbeID, nil +} diff --git a/internal/userauth/userauth_stub.go b/internal/userauth/userauth_stub.go new file mode 100644 index 0000000000..dfc7fd36d5 --- /dev/null +++ b/internal/userauth/userauth_stub.go @@ -0,0 +1,21 @@ +//go:build !((cgo && linux && !android && (amd64 || arm64 || arm || 386)) || (cgo && darwin && !ios && (amd64 || arm64)) || (cgo && windows && (amd64 || 386))) + +package userauth + +// This file provides pure-Go stubs for builds without cgo (CGO_ENABLED=0) + +// Register is unavailable without cgo. +func Register(url, publicParams, manifestVersion, proxy, userAgent string, timeout float32) (string, error) { + return "", ErrUnavailable +} + +// Submit is unavailable without cgo. +func Submit(url, content, probeCC, probeASN, proxy, userAgent string, timeout float32, + cfg *CredentialConfig) (RotatedCredential, error) { + return RotatedCredential{}, ErrUnavailable +} + +// ProbeID is unavailable without cgo. +func ProbeID(credentialB64, probeASN, probeCC string) (string, error) { + return "", ErrUnavailable +} diff --git a/pkg/oonimkall/session.go b/pkg/oonimkall/session.go index c4fc92ce7d..2d8d34562c 100644 --- a/pkg/oonimkall/session.go +++ b/pkg/oonimkall/session.go @@ -324,7 +324,7 @@ func (sess *Session) Submit(ctx *Context, measurement string) (*SubmitMeasuremen sess.mtx.Lock() defer sess.mtx.Unlock() if sess.submitter == nil { - submitter, err := sess.sessp.NewSubmitter(ctx.ctx) + submitter, err := sess.sessp.NewSubmitter(ctx.ctx, false) if err != nil { return nil, err }