diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29c1f5c..556a5f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: lint: timeout-minutes: 10 name: lint - runs-on: ${{ github.repository == 'stainless-sdks/context.dev-cli' && 'depot-ubuntu-24.04' || 'ubuntu-latest' }} + runs-on: ${{ startsWith(github.repository, 'stainless-sdks/') && 'depot-ubuntu-24.04' || 'ubuntu-latest' }} if: (github.event_name == 'push' || github.event.pull_request.head.repo.fork) && (github.event_name != 'push' || github.event.head_commit.message != 'codegen metadata') steps: @@ -48,7 +48,7 @@ jobs: permissions: contents: read id-token: write - runs-on: ${{ github.repository == 'stainless-sdks/context.dev-cli' && 'depot-ubuntu-24.04' || 'ubuntu-latest' }} + runs-on: ${{ startsWith(github.repository, 'stainless-sdks/') && 'depot-ubuntu-24.04' || 'ubuntu-latest' }} if: (github.event_name == 'push' || github.event.pull_request.head.repo.fork) && (github.event_name != 'push' || github.event.head_commit.message != 'codegen metadata') steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -95,7 +95,7 @@ jobs: test: timeout-minutes: 10 name: test - runs-on: ${{ github.repository == 'stainless-sdks/context.dev-cli' && 'depot-ubuntu-24.04' || 'ubuntu-latest' }} + runs-on: ${{ startsWith(github.repository, 'stainless-sdks/') && 'depot-ubuntu-24.04' || 'ubuntu-latest' }} if: github.event_name == 'push' || github.event.pull_request.head.repo.fork steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/.release-please-manifest.json b/.release-please-manifest.json index fea3454..2601677 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.0.0" + ".": "1.1.0" } \ No newline at end of file diff --git a/.stats.yml b/.stats.yml index 7cfdbba..0ac1eb3 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ -configured_endpoints: 30 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-fb56935a194e69348fecd985f7cf8b249795b46af2fb32f6c5c8ef648cf10c15.yml -openapi_spec_hash: 7260a560474283b7ad6ac5d426058ac9 -config_hash: daabb160675d86b354711da1e77e5129 +configured_endpoints: 32 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-242450ea46eb8c3e843fd6c4bf87e73192b5f62f6da697cd091d13c6aa7a991b.yml +openapi_spec_hash: c1c561976de1abcacede71fd5ab9b3d9 +config_hash: 70e7e80b5e87f94981bee396c6cd41e8 diff --git a/CHANGELOG.md b/CHANGELOG.md index 090d810..fdf7d1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## 1.1.0 (2026-07-22) + +Full Changelog: [v1.0.0...v1.1.0](https://github.com/context-dot-dev/context-dev-cli/compare/v1.0.0...v1.1.0) + +### Features + +* **api:** api update ([cd4503b](https://github.com/context-dot-dev/context-dev-cli/commit/cd4503b7722a10614512c2d1ea9c3dffc670f434)) +* **api:** api update ([af90e28](https://github.com/context-dot-dev/context-dev-cli/commit/af90e280f6fe278065c1d873be594fae7f9d61a4)) +* **api:** api update ([5f2e61b](https://github.com/context-dot-dev/context-dev-cli/commit/5f2e61b15cc331ec398111cac0d43b409826da40)) +* **api:** api update ([e21218b](https://github.com/context-dot-dev/context-dev-cli/commit/e21218b5cb90979c8a2ef32539859d671a33967a)) +* **api:** api update ([21c81e4](https://github.com/context-dot-dev/context-dev-cli/commit/21c81e4faf5385a5c02c21585524d8d04794f598)) +* **api:** api update ([0468ad1](https://github.com/context-dot-dev/context-dev-cli/commit/0468ad177e1bfb23732f4bda41ad34fb37f972b4)) +* **api:** api update ([349f90c](https://github.com/context-dot-dev/context-dev-cli/commit/349f90c23ba4ac35c8c5dee81060a495475435c3)) +* **api:** manual updates ([e02d02e](https://github.com/context-dot-dev/context-dev-cli/commit/e02d02e69c215116d5d3e1aec5cab2be279da032)) +* **stlc:** configurable CI runner and private-production-repo support in workflow templates ([d55d529](https://github.com/context-dot-dev/context-dev-cli/commit/d55d5297e92d05dc7bc507f2ac9f3c02f1f9d178)) + ## 1.0.0 (2026-07-12) Full Changelog: [v0.11.0...v1.0.0](https://github.com/context-dot-dev/context-dev-cli/compare/v0.11.0...v1.0.0) diff --git a/go.mod b/go.mod index a6da4f9..a000854 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/charmbracelet/bubbletea v1.3.6 github.com/charmbracelet/lipgloss v1.1.0 github.com/charmbracelet/x/term v0.2.1 - github.com/context-dot-dev/context-go-sdk/v2 v2.3.0 + github.com/context-dot-dev/context-go-sdk/v2 v2.4.0 github.com/goccy/go-yaml v1.18.0 github.com/itchyny/json2yaml v0.1.4 github.com/muesli/reflow v0.3.0 diff --git a/go.sum b/go.sum index 42e69ea..e9ff7cd 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,8 @@ github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payR github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= -github.com/context-dot-dev/context-go-sdk/v2 v2.3.0 h1:5EJ87AXgixxMYsmwd5mlz/4NeW3qDmdPsKCDOAg58Ik= -github.com/context-dot-dev/context-go-sdk/v2 v2.3.0/go.mod h1:CvRANWLP0FHj5XyUwzEVX79ujdwr/2Jy375UVzKNQm8= +github.com/context-dot-dev/context-go-sdk/v2 v2.4.0 h1:mehuyGuUzgflR7rQpSlPaH7lNj2A8hdtf1+WG4U4h1I= +github.com/context-dot-dev/context-go-sdk/v2 v2.4.0/go.mod h1:CvRANWLP0FHj5XyUwzEVX79ujdwr/2Jy375UVzKNQm8= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/pkg/cmd/ai.go b/pkg/cmd/ai.go index 138afcf..34a436a 100644 --- a/pkg/cmd/ai.go +++ b/pkg/cmd/ai.go @@ -31,6 +31,11 @@ var aiExtractProduct = cli.Command{ Default: 604800000, BodyPath: "maxAgeMs", }, + &requestflag.Flag[[]string]{ + Name: "tag", + Usage: "Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters.", + BodyPath: "tags", + }, &requestflag.Flag[int64]{ Name: "timeout-ms", Usage: "Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).", @@ -62,6 +67,11 @@ var aiExtractProducts = cli.Command{ Usage: "Maximum number of products to extract.", BodyPath: "maxProducts", }, + &requestflag.Flag[[]string]{ + Name: "tag", + Usage: "Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters.", + BodyPath: "tags", + }, &requestflag.Flag[int64]{ Name: "timeout-ms", Usage: "Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).", diff --git a/pkg/cmd/ai_test.go b/pkg/cmd/ai_test.go index d690bef..e6ed136 100644 --- a/pkg/cmd/ai_test.go +++ b/pkg/cmd/ai_test.go @@ -17,6 +17,8 @@ func TestAIExtractProduct(t *testing.T) { "ai", "extract-product", "--url", "https://example.com", "--max-age-ms", "0", + "--tag", "production", + "--tag", "team-alpha", "--timeout-ms", "1000", ) }) @@ -26,6 +28,9 @@ func TestAIExtractProduct(t *testing.T) { pipeData := []byte("" + "url: https://example.com\n" + "maxAgeMs: 0\n" + + "tags:\n" + + " - production\n" + + " - team-alpha\n" + "timeoutMS: 1000\n") mocktest.TestRunMockTestWithPipeAndFlags( t, pipeData, @@ -45,6 +50,8 @@ func TestAIExtractProducts(t *testing.T) { "--domain", "domain", "--max-age-ms", "0", "--max-products", "1", + "--tag", "production", + "--tag", "team-alpha", "--timeout-ms", "1000", ) }) @@ -55,6 +62,9 @@ func TestAIExtractProducts(t *testing.T) { "domain: domain\n" + "maxAgeMs: 0\n" + "maxProducts: 1\n" + + "tags:\n" + + " - production\n" + + " - team-alpha\n" + "timeoutMS: 1000\n") mocktest.TestRunMockTestWithPipeAndFlags( t, pipeData, diff --git a/pkg/cmd/brand.go b/pkg/cmd/brand.go index ea4e594..c90f30d 100644 --- a/pkg/cmd/brand.go +++ b/pkg/cmd/brand.go @@ -30,7 +30,7 @@ var brandRetrieve = cli.Command{ Required: true, BodyPath: "type", }, - &requestflag.Flag[string]{ + &requestflag.Flag[*string]{ Name: "force-language", Usage: `Allowed values: "afrikaans", "albanian", "amharic", "arabic", "armenian", "assamese", "aymara", "azeri", "basque", "belarusian", "bengali", "bosnian", "bulgarian", "burmese", "cantonese", "catalan", "cebuano", "chinese", "corsican", "croatian", "czech", "danish", "dutch", "english", "esperanto", "estonian", "farsi", "fijian", "finnish", "french", "galician", "georgian", "german", "greek", "guarani", "gujarati", "haitian-creole", "hausa", "hawaiian", "hebrew", "hindi", "hmong", "hungarian", "icelandic", "igbo", "indonesian", "irish", "italian", "japanese", "javanese", "kannada", "kazakh", "khmer", "kinyarwanda", "korean", "kurdish", "kyrgyz", "lao", "latin", "latvian", "lingala", "lithuanian", "luxembourgish", "macedonian", "malagasy", "malay", "malayalam", "maltese", "maori", "marathi", "mongolian", "nepali", "norwegian", "odia", "oromo", "pashto", "pidgin", "polish", "portuguese", "punjabi", "quechua", "romanian", "russian", "samoan", "scottish-gaelic", "serbian", "sesotho", "shona", "sindhi", "sinhala", "slovak", "slovene", "somali", "spanish", "sundanese", "swahili", "swedish", "tagalog", "tajik", "tamil", "tatar", "telugu", "thai", "tibetan", "tigrinya", "tongan", "tswana", "turkish", "turkmen", "ukrainian", "urdu", "uyghur", "uzbek", "vietnamese", "welsh", "wolof", "xhosa", "yiddish", "yoruba", "zulu".`, BodyPath: "force_language", @@ -45,6 +45,11 @@ var brandRetrieve = cli.Command{ Usage: "Optional parameter to optimize the API call for maximum speed. When set to true, the API will skip time-consuming operations for faster response at the cost of less comprehensive data.", BodyPath: "maxSpeed", }, + &requestflag.Flag[[]string]{ + Name: "tag", + Usage: "Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters.", + BodyPath: "tags", + }, &requestflag.Flag[int64]{ Name: "timeout-ms", Usage: "Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).", @@ -95,12 +100,12 @@ var brandRetrieve = cli.Command{ Usage: "When set to true, the API performs additional verification to ensure the identified brand matches the transaction with high confidence.", BodyPath: "high_confidence_only", }, - &requestflag.Flag[int64]{ + &requestflag.Flag[any]{ Name: "mcc", Usage: "Optional Merchant Category Code (MCC) to help identify the business category or industry.", BodyPath: "mcc", }, - &requestflag.Flag[float64]{ + &requestflag.Flag[any]{ Name: "phone", Usage: "Optional phone number from the transaction to help verify brand match.", BodyPath: "phone", @@ -121,12 +126,22 @@ var brandRetrieveSimplified = cli.Command{ Required: true, QueryPath: "domain", }, - &requestflag.Flag[int64]{ + &requestflag.Flag[*int64]{ Name: "max-age-ms", Usage: "Maximum age in milliseconds for cached brand data before the API performs a hard refresh. Defaults to 3 months (7776000000 ms). Values below 1 day (86400000 ms) are clamped to 1 day; values above 1 year (31536000000 ms) are clamped to 1 year.", - Default: 7776000000, + Default: requestflag.Ptr[int64](7776000000), QueryPath: "maxAgeMs", }, + &requestflag.Flag[[]string]{ + Name: "tag", + Usage: "Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters.", + QueryPath: "tags", + }, + &requestflag.Flag[string]{ + Name: "theme", + Usage: "Optional theme preference used when selecting brand assets.", + QueryPath: "theme", + }, &requestflag.Flag[int64]{ Name: "timeout-ms", Usage: "Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).", diff --git a/pkg/cmd/brand_test.go b/pkg/cmd/brand_test.go index e3a79c7..55c691d 100644 --- a/pkg/cmd/brand_test.go +++ b/pkg/cmd/brand_test.go @@ -15,11 +15,13 @@ func TestBrandRetrieve(t *testing.T) { t, "--api-key", "string", "brand", "retrieve", - "--domain", "domain", + "--domain", "xxx", "--type", "by_domain", "--force-language", "afrikaans", "--max-age-ms", "0", "--max-speed=true", + "--tag", "production", + "--tag", "team-alpha", "--timeout-ms", "1000", ) }) @@ -27,11 +29,14 @@ func TestBrandRetrieve(t *testing.T) { t.Run("piping data", func(t *testing.T) { // Test piping YAML data over stdin pipeData := []byte("" + - "domain: domain\n" + + "domain: xxx\n" + "type: by_domain\n" + "force_language: afrikaans\n" + "maxAgeMs: 0\n" + "maxSpeed: true\n" + + "tags:\n" + + " - production\n" + + " - team-alpha\n" + "timeoutMS: 1000\n") mocktest.TestRunMockTestWithPipeAndFlags( t, pipeData, @@ -48,8 +53,11 @@ func TestBrandRetrieveSimplified(t *testing.T) { t, "--api-key", "string", "brand", "retrieve-simplified", - "--domain", "domain", - "--max-age-ms", "86400000", + "--domain", "xxx", + "--max-age-ms", "0", + "--tag", "production", + "--tag", "team-alpha", + "--theme", "light", "--timeout-ms", "1000", ) }) diff --git a/pkg/cmd/cmd.go b/pkg/cmd/cmd.go index 841a1e6..c17a6f6 100644 --- a/pkg/cmd/cmd.go +++ b/pkg/cmd/cmd.go @@ -155,6 +155,8 @@ func init() { &monitorsUpdate, &monitorsList, &monitorsDelete, + &monitorsGetCreditUsage, + &monitorsGetLimits, &monitorsListAccountChanges, &monitorsListAccountRuns, &monitorsListChanges, diff --git a/pkg/cmd/industry.go b/pkg/cmd/industry.go index ed317cf..4fd74f4 100644 --- a/pkg/cmd/industry.go +++ b/pkg/cmd/industry.go @@ -37,6 +37,11 @@ var industryRetrieveNaics = cli.Command{ Default: 1, QueryPath: "minResults", }, + &requestflag.Flag[[]string]{ + Name: "tag", + Usage: "Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters.", + QueryPath: "tags", + }, &requestflag.Flag[int64]{ Name: "timeout-ms", Usage: "Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).", @@ -70,6 +75,11 @@ var industryRetrieveSic = cli.Command{ Default: 1, QueryPath: "minResults", }, + &requestflag.Flag[[]string]{ + Name: "tag", + Usage: "Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters.", + QueryPath: "tags", + }, &requestflag.Flag[int64]{ Name: "timeout-ms", Usage: "Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).", diff --git a/pkg/cmd/industry_test.go b/pkg/cmd/industry_test.go index d74383e..b7f6329 100644 --- a/pkg/cmd/industry_test.go +++ b/pkg/cmd/industry_test.go @@ -15,9 +15,11 @@ func TestIndustryRetrieveNaics(t *testing.T) { t, "--api-key", "string", "industry", "retrieve-naics", - "--input", "input", + "--input", "xxxx", "--max-results", "1", "--min-results", "1", + "--tag", "production", + "--tag", "team-alpha", "--timeout-ms", "1000", ) }) @@ -30,9 +32,11 @@ func TestIndustryRetrieveSic(t *testing.T) { t, "--api-key", "string", "industry", "retrieve-sic", - "--input", "input", + "--input", "xxxx", "--max-results", "1", "--min-results", "1", + "--tag", "production", + "--tag", "team-alpha", "--timeout-ms", "1000", "--type", "original_sic", ) diff --git a/pkg/cmd/monitor.go b/pkg/cmd/monitor.go index 5729a84..1842db3 100644 --- a/pkg/cmd/monitor.go +++ b/pkg/cmd/monitor.go @@ -49,7 +49,7 @@ var monitorsCreate = requestflag.WithInnerFlags(cli.Command{ }, &requestflag.Flag[[]string]{ Name: "tag", - Usage: "User-defined tags for grouping and filtering monitors and their changes.", + Usage: "User-defined tags for grouping and filtering monitors and their changes. Duplicates are removed.", BodyPath: "tags", }, &requestflag.Flag[map[string]any]{ @@ -142,7 +142,7 @@ var monitorsUpdate = requestflag.WithInnerFlags(cli.Command{ }, &requestflag.Flag[[]string]{ Name: "tag", - Usage: "User-defined tags for grouping and filtering monitors and their changes.", + Usage: "User-defined tags for grouping and filtering monitors and their changes. Duplicates are removed.", BodyPath: "tags", }, &requestflag.Flag[map[string]any]{ @@ -202,16 +202,17 @@ var monitorsList = cli.Command{ Flags: []cli.Flag{ &requestflag.Flag[string]{ Name: "change-detection-type", - Usage: `Allowed values: "exact", "semantic".`, + Usage: "Filter by change detection type.", QueryPath: "change_detection_type", }, &requestflag.Flag[string]{ Name: "cursor", + Usage: "Opaque pagination cursor from a previous response.", QueryPath: "cursor", }, &requestflag.Flag[int64]{ Name: "limit", - Default: 25, + Usage: "Maximum number of items to return per page (1-100). Defaults to 25.", QueryPath: "limit", }, &requestflag.Flag[string]{ @@ -219,7 +220,7 @@ var monitorsList = cli.Command{ Usage: "Free-text search term, matched against the fields named in `search_by`.", QueryPath: "q", }, - &requestflag.Flag[[]string]{ + &requestflag.Flag[any]{ Name: "search-by", Usage: "Comma-separated fields to search with `q`. Defaults to all of them. Note `instructions` only exists on extract monitors.", QueryPath: "search_by", @@ -227,12 +228,11 @@ var monitorsList = cli.Command{ &requestflag.Flag[string]{ Name: "search-type", Usage: "`prefix` for as-you-type prefix matching (default), `exact` for full-token matching.", - Default: "prefix", QueryPath: "search_type", }, &requestflag.Flag[string]{ Name: "status", - Usage: "Monitor lifecycle status. `failed` means the most recent run failed (see the monitor's `last_error`); failed monitors keep running on schedule and flip back to `active` on the next successful run. Monitors are auto-`paused` after repeated consecutive failures or insufficient-credit skips; resume by PATCHing status to `active`.", + Usage: "Filter monitors by lifecycle status.", QueryPath: "status", }, &requestflag.Flag[string]{ @@ -240,14 +240,14 @@ var monitorsList = cli.Command{ Usage: "Filter to items that have this tag.", QueryPath: "tag", }, - &requestflag.Flag[[]string]{ + &requestflag.Flag[any]{ Name: "tag", Usage: "Comma-separated list of tags to filter by (matches monitors having any of them).", QueryPath: "tags", }, &requestflag.Flag[string]{ Name: "target-type", - Usage: `Allowed values: "page", "sitemap", "extract".`, + Usage: "Filter by target type.", QueryPath: "target_type", }, }, @@ -270,6 +270,35 @@ var monitorsDelete = cli.Command{ HideHelpCommand: true, } +var monitorsGetCreditUsage = cli.Command{ + Name: "get-credit-usage", + Usage: "Returns credits charged per monitor over an optional [since, until] window,\nnewest spenders first.", + Suggest: true, + Flags: []cli.Flag{ + &requestflag.Flag[any]{ + Name: "since", + Usage: "Only include items at or after this ISO 8601 timestamp.", + QueryPath: "since", + }, + &requestflag.Flag[any]{ + Name: "until", + Usage: "Only include items before this ISO 8601 timestamp.", + QueryPath: "until", + }, + }, + Action: handleMonitorsGetCreditUsage, + HideHelpCommand: true, +} + +var monitorsGetLimits = cli.Command{ + Name: "get-limits", + Usage: "Returns how many monitors the account has and the maximum it allows.", + Suggest: true, + Flags: []cli.Flag{}, + Action: handleMonitorsGetLimits, + HideHelpCommand: true, +} + var monitorsListAccountChanges = cli.Command{ Name: "list-account-changes", Usage: "Returns an account-wide feed of detected changes across monitors.", @@ -277,24 +306,27 @@ var monitorsListAccountChanges = cli.Command{ Flags: []cli.Flag{ &requestflag.Flag[string]{ Name: "change-detection-type", - Usage: `Allowed values: "exact", "semantic".`, + Usage: "Filter by change detection type.", QueryPath: "change_detection_type", }, &requestflag.Flag[string]{ Name: "cursor", + Usage: "Opaque pagination cursor from a previous response.", QueryPath: "cursor", }, &requestflag.Flag[int64]{ Name: "limit", - Default: 25, + Usage: "Maximum number of items to return per page (1-100). Defaults to 25.", QueryPath: "limit", }, &requestflag.Flag[string]{ Name: "monitor-id", + Usage: "Filter changes to a single monitor.", QueryPath: "monitor_id", }, &requestflag.Flag[any]{ Name: "since", + Usage: "Only include items at or after this ISO 8601 timestamp.", QueryPath: "since", }, &requestflag.Flag[string]{ @@ -304,11 +336,12 @@ var monitorsListAccountChanges = cli.Command{ }, &requestflag.Flag[string]{ Name: "target-type", - Usage: `Allowed values: "page", "sitemap", "extract".`, + Usage: "Filter by target type.", QueryPath: "target_type", }, &requestflag.Flag[any]{ Name: "until", + Usage: "Only include items before this ISO 8601 timestamp.", QueryPath: "until", }, }, @@ -323,16 +356,17 @@ var monitorsListAccountRuns = cli.Command{ Flags: []cli.Flag{ &requestflag.Flag[string]{ Name: "cursor", + Usage: "Opaque pagination cursor from a previous response.", QueryPath: "cursor", }, &requestflag.Flag[int64]{ Name: "limit", - Default: 25, + Usage: "Maximum number of items to return per page (1-100). Defaults to 25.", QueryPath: "limit", }, &requestflag.Flag[string]{ Name: "status", - Usage: "Lifecycle status of a run. `skipped` runs never executed — see `skip_reason` (insufficient credits, monitor paused, or superseded by a concurrent run).", + Usage: "Filter runs by lifecycle status.", QueryPath: "status", }, }, @@ -352,15 +386,17 @@ var monitorsListChanges = cli.Command{ }, &requestflag.Flag[string]{ Name: "cursor", + Usage: "Opaque pagination cursor from a previous response.", QueryPath: "cursor", }, &requestflag.Flag[int64]{ Name: "limit", - Default: 25, + Usage: "Maximum number of items to return per page (1-100). Defaults to 25.", QueryPath: "limit", }, &requestflag.Flag[any]{ Name: "since", + Usage: "Only include items at or after this ISO 8601 timestamp.", QueryPath: "since", }, &requestflag.Flag[string]{ @@ -370,6 +406,7 @@ var monitorsListChanges = cli.Command{ }, &requestflag.Flag[any]{ Name: "until", + Usage: "Only include items before this ISO 8601 timestamp.", QueryPath: "until", }, }, @@ -389,16 +426,17 @@ var monitorsListRuns = cli.Command{ }, &requestflag.Flag[string]{ Name: "cursor", + Usage: "Opaque pagination cursor from a previous response.", QueryPath: "cursor", }, &requestflag.Flag[int64]{ Name: "limit", - Default: 25, + Usage: "Maximum number of items to return per page (1-100). Defaults to 25.", QueryPath: "limit", }, &requestflag.Flag[string]{ Name: "status", - Usage: "Lifecycle status of a run. `skipped` runs never executed — see `skip_reason` (insufficient credits, monitor paused, or superseded by a concurrent run).", + Usage: "Filter runs by lifecycle status.", QueryPath: "status", }, }, @@ -651,6 +689,86 @@ func handleMonitorsDelete(ctx context.Context, cmd *cli.Command) error { }) } +func handleMonitorsGetCreditUsage(ctx context.Context, cmd *cli.Command) error { + client := contextdev.NewClient(getDefaultRequestOptions(cmd)...) + unusedArgs := cmd.Args().Slice() + + if len(unusedArgs) > 0 { + return fmt.Errorf("Unexpected extra arguments: %v", unusedArgs) + } + + options, err := flagOptions( + cmd, + apiquery.NestedQueryFormatBrackets, + apiquery.ArrayQueryFormatComma, + EmptyBody, + false, + ) + if err != nil { + return err + } + + params := contextdev.MonitorGetCreditUsageParams{} + + var res []byte + options = append(options, option.WithResponseBodyInto(&res)) + _, err = client.Monitors.GetCreditUsage(ctx, params, options...) + if err != nil { + return err + } + + obj := gjson.ParseBytes(res) + format := cmd.Root().String("format") + explicitFormat := cmd.Root().IsSet("format") + transform := cmd.Root().String("transform") + return ShowJSON(obj, ShowJSONOpts{ + ExplicitFormat: explicitFormat, + Format: format, + RawOutput: cmd.Root().Bool("raw-output"), + Title: "monitors get-credit-usage", + Transform: transform, + }) +} + +func handleMonitorsGetLimits(ctx context.Context, cmd *cli.Command) error { + client := contextdev.NewClient(getDefaultRequestOptions(cmd)...) + unusedArgs := cmd.Args().Slice() + + if len(unusedArgs) > 0 { + return fmt.Errorf("Unexpected extra arguments: %v", unusedArgs) + } + + options, err := flagOptions( + cmd, + apiquery.NestedQueryFormatBrackets, + apiquery.ArrayQueryFormatComma, + EmptyBody, + false, + ) + if err != nil { + return err + } + + var res []byte + options = append(options, option.WithResponseBodyInto(&res)) + _, err = client.Monitors.GetLimits(ctx, options...) + if err != nil { + return err + } + + obj := gjson.ParseBytes(res) + format := cmd.Root().String("format") + explicitFormat := cmd.Root().IsSet("format") + transform := cmd.Root().String("transform") + return ShowJSON(obj, ShowJSONOpts{ + ExplicitFormat: explicitFormat, + Format: format, + RawOutput: cmd.Root().Bool("raw-output"), + Title: "monitors get-limits", + Transform: transform, + }) +} + func handleMonitorsListAccountChanges(ctx context.Context, cmd *cli.Command) error { client := contextdev.NewClient(getDefaultRequestOptions(cmd)...) unusedArgs := cmd.Args().Slice() diff --git a/pkg/cmd/monitor_test.go b/pkg/cmd/monitor_test.go index a307b8c..40e2c3c 100644 --- a/pkg/cmd/monitor_test.go +++ b/pkg/cmd/monitor_test.go @@ -178,12 +178,12 @@ func TestMonitorsList(t *testing.T) { "--change-detection-type", "exact", "--cursor", "cursor", "--limit", "1", - "--q", "q", - "--search-by", "name", + "--q", "pricing", + "--search-by", "[name]", "--search-type", "exact", "--status", "active", - "--tag", "tag", - "--tag", "string", + "--tag", "pricing", + "--tag", "[x]", "--target-type", "page", ) }) @@ -201,6 +201,30 @@ func TestMonitorsDelete(t *testing.T) { }) } +func TestMonitorsGetCreditUsage(t *testing.T) { + t.Skip("Mock server tests are disabled") + t.Run("regular flags", func(t *testing.T) { + mocktest.TestRunMockTestWithFlags( + t, + "--api-key", "string", + "monitors", "get-credit-usage", + "--since", "'2026-06-01T00:00:00Z'", + "--until", "'2026-06-28T00:00:00Z'", + ) + }) +} + +func TestMonitorsGetLimits(t *testing.T) { + t.Skip("Mock server tests are disabled") + t.Run("regular flags", func(t *testing.T) { + mocktest.TestRunMockTestWithFlags( + t, + "--api-key", "string", + "monitors", "get-limits", + ) + }) +} + func TestMonitorsListAccountChanges(t *testing.T) { t.Skip("Mock server tests are disabled") t.Run("regular flags", func(t *testing.T) { @@ -211,11 +235,11 @@ func TestMonitorsListAccountChanges(t *testing.T) { "--change-detection-type", "exact", "--cursor", "cursor", "--limit", "1", - "--monitor-id", "monitor_id", - "--since", "'2019-12-27T18:11:19.117Z'", - "--tag", "tag", + "--monitor-id", "mon_123", + "--since", "'2026-06-01T00:00:00Z'", + "--tag", "pricing", "--target-type", "page", - "--until", "'2019-12-27T18:11:19.117Z'", + "--until", "'2026-06-28T00:00:00Z'", ) }) } @@ -244,9 +268,9 @@ func TestMonitorsListChanges(t *testing.T) { "--monitor-id", "mon_123", "--cursor", "cursor", "--limit", "1", - "--since", "'2019-12-27T18:11:19.117Z'", - "--tag", "tag", - "--until", "'2019-12-27T18:11:19.117Z'", + "--since", "'2026-06-01T00:00:00Z'", + "--tag", "pricing", + "--until", "'2026-06-28T00:00:00Z'", ) }) } diff --git a/pkg/cmd/parse.go b/pkg/cmd/parse.go index 9cc7a65..5403297 100644 --- a/pkg/cmd/parse.go +++ b/pkg/cmd/parse.go @@ -18,7 +18,7 @@ import ( var parseHandle = requestflag.WithInnerFlags(cli.Command{ Name: "handle", - Usage: "Converts raw text, source code, web/data, PDF, Microsoft Office, and image bytes\ninto LLM-usable Markdown. The base request costs 1 credit. When OCR runs\n(requires ocr=true), the entire call costs 5 credits; ocr=true requests where no\nOCR ends up running still cost 1 credit.", + Usage: "Converts raw text, source code, web/data, PDF, Microsoft Office, and image bytes\ninto LLM-usable Markdown.", Suggest: true, Flags: []cli.Flag{ &requestflag.Flag[string]{ @@ -27,47 +27,63 @@ var parseHandle = requestflag.WithInnerFlags(cli.Command{ BodyRoot: true, FileInput: true, }, + &requestflag.Flag[string]{ + Name: "client", + Usage: "Optional client identifier used for usage attribution.", + QueryPath: "client", + }, &requestflag.Flag[string]{ Name: "extension", - Usage: `Optional file extension hint. Case-insensitive; a leading dot is accepted (e.g. ".pdf").`, + Usage: "Optional file extension hint, such as pdf, docx, xlsx, pptx, html, json, csv, md, py, rtf, jpg, png, or txt.", QueryPath: "extension", }, - &requestflag.Flag[bool]{ + &requestflag.Flag[any]{ Name: "include-images", Usage: "Include image references in Markdown output", Default: false, QueryPath: "includeImages", }, - &requestflag.Flag[bool]{ + &requestflag.Flag[any]{ Name: "include-links", Usage: "Preserve hyperlinks in Markdown output", Default: true, QueryPath: "includeLinks", }, - &requestflag.Flag[bool]{ + &requestflag.Flag[any]{ Name: "ocr", - Usage: "Gates all OCR. When true, PDFs get embedded-image OCR (recognized text inserted at each image's position in page reading order, preserving the text layer; pdf.start/pdf.end limit the page range), scanned PDFs with no text layer get full-document OCR, and raster images get their visible text transcribed. When false, no OCR runs: scanned PDFs may yield no content and images return only format/dimension metadata. Calls where OCR actually runs cost 5 credits instead of 1.", + Usage: "When true for PDF inputs, detect and OCR images embedded in the selected pages, inserting recognized text at each image's position in page reading order while preserving the PDF text layer. pdf.start/pdf.end limit the inclusive page range. When false, all OCR is disabled, including the automatic scanned-PDF fallback.", Default: false, QueryPath: "ocr", }, &requestflag.Flag[map[string]any]{ Name: "pdf", - Usage: "PDF page-range controls. Use start/end to limit parsing (and OCR when ocr=true) to an inclusive 1-based page range.", + Usage: `PDF page-range options as a JSON object, e.g. {"start": 2, "end": 5}.`, Default: map[string]any{}, QueryPath: "pdf", }, - &requestflag.Flag[bool]{ + &requestflag.Flag[any]{ Name: "shorten-base64-images", Usage: "Shorten base64-encoded image data in the Markdown output", Default: true, QueryPath: "shortenBase64Images", }, - &requestflag.Flag[bool]{ + &requestflag.Flag[[]string]{ + Name: "tag", + Usage: "Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters.", + QueryPath: "tags", + }, + &requestflag.Flag[any]{ Name: "use-main-content-only", Usage: "Extract only the main content from HTML-like inputs", Default: false, QueryPath: "useMainContentOnly", }, + &requestflag.Flag[string]{ + Name: "zdr", + Usage: "Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true.", + Default: "disabled", + QueryPath: "zdr", + }, }, Action: handleParseHandle, HideHelpCommand: true, diff --git a/pkg/cmd/parse_test.go b/pkg/cmd/parse_test.go index 11b4e77..8177b5d 100644 --- a/pkg/cmd/parse_test.go +++ b/pkg/cmd/parse_test.go @@ -17,13 +17,17 @@ func TestParseHandle(t *testing.T) { "--api-key", "string", "parse", "handle", "--body", mocktest.TestFile(t, "Example data"), + "--client", "x", "--extension", "txt", - "--include-images=true", - "--include-links=true", - "--ocr=true", + "--include-images", "'true'", + "--include-links", "'true'", + "--ocr", "'true'", "--pdf", "{end: 1, start: 1}", - "--shorten-base64-images=true", - "--use-main-content-only=true", + "--shorten-base64-images", "'true'", + "--tag", "production", + "--tag", "team-alpha", + "--use-main-content-only", "'true'", + "--zdr", "enabled", ) }) @@ -37,14 +41,18 @@ func TestParseHandle(t *testing.T) { "--api-key", "string", "parse", "handle", "--body", mocktest.TestFile(t, "Example data"), + "--client", "x", "--extension", "txt", - "--include-images=true", - "--include-links=true", - "--ocr=true", + "--include-images", "'true'", + "--include-links", "'true'", + "--ocr", "'true'", "--pdf.end", "1", "--pdf.start", "1", - "--shorten-base64-images=true", - "--use-main-content-only=true", + "--shorten-base64-images", "'true'", + "--tag", "production", + "--tag", "team-alpha", + "--use-main-content-only", "'true'", + "--zdr", "enabled", ) }) @@ -55,13 +63,17 @@ func TestParseHandle(t *testing.T) { t, pipeData, "--api-key", "string", "parse", "handle", + "--client", "x", "--extension", "txt", - "--include-images=true", - "--include-links=true", - "--ocr=true", + "--include-images", "'true'", + "--include-links", "'true'", + "--ocr", "'true'", "--pdf", "{end: 1, start: 1}", - "--shorten-base64-images=true", - "--use-main-content-only=true", + "--shorten-base64-images", "'true'", + "--tag", "production", + "--tag", "team-alpha", + "--use-main-content-only", "'true'", + "--zdr", "enabled", ) }) } diff --git a/pkg/cmd/utility.go b/pkg/cmd/utility.go index ac0a997..5376d1b 100644 --- a/pkg/cmd/utility.go +++ b/pkg/cmd/utility.go @@ -31,6 +31,11 @@ var utilityPrefetch = cli.Command{ Required: true, BodyPath: "type", }, + &requestflag.Flag[[]string]{ + Name: "tag", + Usage: "Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters.", + BodyPath: "tags", + }, &requestflag.Flag[int64]{ Name: "timeout-ms", Usage: "Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).", diff --git a/pkg/cmd/utility_test.go b/pkg/cmd/utility_test.go index f9d5b35..8f43739 100644 --- a/pkg/cmd/utility_test.go +++ b/pkg/cmd/utility_test.go @@ -15,8 +15,10 @@ func TestUtilityPrefetch(t *testing.T) { t, "--api-key", "string", "utility", "prefetch", - "--identifier", "{domain: domain}", + "--identifier", "{domain: xxx}", "--type", "brand", + "--tag", "production", + "--tag", "team-alpha", "--timeout-ms", "1000", ) }) @@ -25,8 +27,11 @@ func TestUtilityPrefetch(t *testing.T) { // Test piping YAML data over stdin pipeData := []byte("" + "identifier:\n" + - " domain: domain\n" + + " domain: xxx\n" + "type: brand\n" + + "tags:\n" + + " - production\n" + + " - team-alpha\n" + "timeoutMS: 1000\n") mocktest.TestRunMockTestWithPipeAndFlags( t, pipeData, diff --git a/pkg/cmd/version.go b/pkg/cmd/version.go index d47a190..a47345e 100644 --- a/pkg/cmd/version.go +++ b/pkg/cmd/version.go @@ -2,4 +2,4 @@ package cmd -const Version = "1.0.0" // x-release-please-version +const Version = "1.1.0" // x-release-please-version diff --git a/pkg/cmd/web.go b/pkg/cmd/web.go index 4d16c84..483c012 100644 --- a/pkg/cmd/web.go +++ b/pkg/cmd/web.go @@ -76,12 +76,23 @@ var webExtract = requestflag.WithInnerFlags(cli.Command{ Default: map[string]any{"shouldParse": true}, BodyPath: "pdf", }, + &requestflag.Flag[bool]{ + Name: "settle-animations", + Usage: "When true, waits briefly for CSS and transition animations to settle before extracting each crawled page. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages.", + Default: false, + BodyPath: "settleAnimations", + }, &requestflag.Flag[int64]{ Name: "stop-after-ms", Usage: "Soft time budget for the crawl in milliseconds. Min: 10000 (10s). Max: 110000 (110s). Default: 80000 (80s).", Default: 80000, BodyPath: "stopAfterMs", }, + &requestflag.Flag[[]string]{ + Name: "tag", + Usage: "Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters.", + BodyPath: "tags", + }, &requestflag.Flag[int64]{ Name: "timeout-ms", Usage: "Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).", @@ -132,6 +143,11 @@ var webExtractCompetitors = cli.Command{ Default: 5, QueryPath: "numCompetitors", }, + &requestflag.Flag[[]string]{ + Name: "tag", + Usage: "Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters.", + QueryPath: "tags", + }, &requestflag.Flag[int64]{ Name: "timeout-ms", Usage: "Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).", @@ -157,12 +173,17 @@ var webExtractFonts = cli.Command{ Usage: "Domain name to extract fonts from (e.g., 'example.com', 'google.com'). The domain will be automatically normalized and validated. You must provide either 'domain' or 'directUrl', but not both.", QueryPath: "domain", }, - &requestflag.Flag[int64]{ + &requestflag.Flag[*int64]{ Name: "max-age-ms", - Usage: "Maximum age in milliseconds for cached data before the API performs a hard refresh. Defaults to 3 months (7776000000 ms). Values below 1 day (86400000 ms) are clamped to 1 day; values above 1 year (31536000000 ms) are clamped to 1 year.", - Default: 7776000000, + Usage: "Maximum age in milliseconds for cached brand data before the API performs a hard refresh. Defaults to 3 months (7776000000 ms). Values below 1 day (86400000 ms) are clamped to 1 day; values above 1 year (31536000000 ms) are clamped to 1 year.", + Default: requestflag.Ptr[int64](7776000000), QueryPath: "maxAgeMs", }, + &requestflag.Flag[[]string]{ + Name: "tag", + Usage: "Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters.", + QueryPath: "tags", + }, &requestflag.Flag[int64]{ Name: "timeout-ms", Usage: "Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).", @@ -193,12 +214,17 @@ var webExtractStyleguide = cli.Command{ Usage: "Domain name to extract styleguide from (e.g., 'example.com', 'google.com'). The domain will be automatically normalized and validated. You must provide either 'domain' or 'directUrl', but not both.", QueryPath: "domain", }, - &requestflag.Flag[int64]{ + &requestflag.Flag[*int64]{ Name: "max-age-ms", - Usage: "Maximum age in milliseconds for cached data before the API performs a hard refresh. Defaults to 3 months (7776000000 ms). Values below 1 day (86400000 ms) are clamped to 1 day; values above 1 year (31536000000 ms) are clamped to 1 year.", - Default: 7776000000, + Usage: "Maximum age in milliseconds for cached brand data before the API performs a hard refresh. Defaults to 3 months (7776000000 ms). Values below 1 day (86400000 ms) are clamped to 1 day; values above 1 year (31536000000 ms) are clamped to 1 year.", + Default: requestflag.Ptr[int64](7776000000), QueryPath: "maxAgeMs", }, + &requestflag.Flag[[]string]{ + Name: "tag", + Usage: "Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters.", + QueryPath: "tags", + }, &requestflag.Flag[int64]{ Name: "timeout-ms", Usage: "Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).", @@ -221,7 +247,7 @@ var webScreenshot = requestflag.WithInnerFlags(cli.Command{ }, &requestflag.Flag[string]{ Name: "country", - Usage: "Two-letter ISO 3166-1 alpha-2 country code for the website request location. When provided, Context.dev fetches the target page from that country.", + Usage: "Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev residential proxy exit location. Must be one of Context.dev's supported countries. When provided, Context.dev fetches the target page from that country.", QueryPath: "country", }, &requestflag.Flag[string]{ @@ -239,16 +265,16 @@ var webScreenshot = requestflag.WithInnerFlags(cli.Command{ Usage: "Optional parameter to determine screenshot type. If 'true', takes a full page screenshot capturing all content. If 'false' or not provided, takes a viewport screenshot (standard browser view).", QueryPath: "fullScreenshot", }, - &requestflag.Flag[string]{ + &requestflag.Flag[any]{ Name: "handle-cookie-popup", Usage: "Optional parameter to control cookie/consent popup handling. If 'true', we dismiss cookie banner before capture. If 'false' or not provided, captures the page without that step.", - Default: "false", + Default: false, QueryPath: "handleCookiePopup", }, - &requestflag.Flag[int64]{ + &requestflag.Flag[*int64]{ Name: "max-age-ms", Usage: "Return a cached screenshot if a prior screenshot for the same parameters exists and is younger than this many milliseconds. Defaults to 1 day (86400000 ms) when omitted. Max is 30 days (2592000000 ms). Set to 0 to always capture fresh.", - Default: 86400000, + Default: requestflag.Ptr[int64](86400000), QueryPath: "maxAgeMs", }, &requestflag.Flag[string]{ @@ -256,11 +282,16 @@ var webScreenshot = requestflag.WithInnerFlags(cli.Command{ Usage: "Optional parameter to specify which page type to screenshot. If provided, the system will scrape the domain's links and use heuristics to find the most appropriate URL for the specified page type (30 supported languages). If not provided, screenshots the main domain landing page. Only applicable when using 'domain', not 'directUrl'.", QueryPath: "page", }, - &requestflag.Flag[int64]{ + &requestflag.Flag[*int64]{ Name: "scroll-offset", Usage: "Optional vertical scroll offset in pixels for capturing a long page in viewport-sized chunks. When provided, the full page is captured once and the returned image is the viewport-sized slice that begins at this Y offset (e.g. request scrollOffset=0, then 1080, then 2160 to walk a 1920x1080 landing page top to bottom). The final slice may be shorter than the viewport height. Takes precedence over fullScreenshot. Max: 100000.", QueryPath: "scrollOffset", }, + &requestflag.Flag[[]string]{ + Name: "tag", + Usage: "Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters.", + QueryPath: "tags", + }, &requestflag.Flag[int64]{ Name: "timeout-ms", Usage: "Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).", @@ -272,12 +303,18 @@ var webScreenshot = requestflag.WithInnerFlags(cli.Command{ Default: map[string]any{"width": 1920, "height": 1080}, QueryPath: "viewport", }, - &requestflag.Flag[int64]{ + &requestflag.Flag[*int64]{ Name: "wait-for-ms", - Usage: "Optional browser wait time in milliseconds after initial page load before taking the screenshot. Min: 0. Max: 30000 (30 seconds). Defaults to 3000 ms when omitted.", - Default: 3000, + Usage: "Optional browser wait time in milliseconds after initial page load before taking the screenshot. Min: 0. Max: 30000 (30 seconds). Defaults to 3000 ms when omitted.", + Default: requestflag.Ptr[int64](3000), QueryPath: "waitForMs", }, + &requestflag.Flag[string]{ + Name: "zdr", + Usage: "Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true.", + Default: "disabled", + QueryPath: "zdr", + }, }, Action: handleWebScreenshot, HideHelpCommand: true, @@ -343,6 +380,11 @@ var webSearch = requestflag.WithInnerFlags(cli.Command{ Usage: "Expand the query into multiple parallel variants for broader recall.", BodyPath: "queryFanout", }, + &requestflag.Flag[[]string]{ + Name: "tag", + Usage: "Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters.", + BodyPath: "tags", + }, &requestflag.Flag[int64]{ Name: "timeout-ms", Usage: "Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).", @@ -497,6 +539,11 @@ var webWebCrawlMd = requestflag.WithInnerFlags(cli.Command{ Default: 80000, BodyPath: "stopAfterMs", }, + &requestflag.Flag[[]string]{ + Name: "tag", + Usage: "Optional tags for tracking usage. Up to 20 tags, each 1 to 50 characters.", + BodyPath: "tags", + }, &requestflag.Flag[int64]{ Name: "timeout-ms", Usage: "Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).", @@ -518,6 +565,12 @@ var webWebCrawlMd = requestflag.WithInnerFlags(cli.Command{ Usage: "Optional browser wait time in milliseconds after initial page load for each crawled page. Min: 0. Max: 30000 (30 seconds). ", BodyPath: "waitForMs", }, + &requestflag.Flag[string]{ + Name: "zdr", + Usage: "Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true.", + Default: "disabled", + BodyPath: "zdr", + }, }, Action: handleWebWebCrawlMd, HideHelpCommand: true, @@ -548,7 +601,7 @@ var webWebCrawlMd = requestflag.WithInnerFlags(cli.Command{ var webWebScrapeHTML = requestflag.WithInnerFlags(cli.Command{ Name: "web-scrape-html", - Usage: "Scrapes the given URL and returns the raw HTML content of the page.", + Usage: "Scrapes the given URL and returns the raw HTML content of the page. The base\nrequest costs 1 credit; requests with browser actions cost 2 credits.", Suggest: true, Flags: []cli.Flag{ &requestflag.Flag[string]{ @@ -557,12 +610,17 @@ var webWebScrapeHTML = requestflag.WithInnerFlags(cli.Command{ Required: true, QueryPath: "url", }, + &requestflag.Flag[any]{ + Name: "action", + Usage: "Optional browser actions executed in array order after the page loads and before content is captured. Requires a paid plan. Send a JSON array in the query parameter. Maximum: 5 actions.", + QueryPath: "actions", + }, &requestflag.Flag[string]{ Name: "country", - Usage: "Two-letter ISO 3166-1 alpha-2 country code for the website request location. When provided, Context.dev fetches the target page from that country.", + Usage: "Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev residential proxy exit location. Must be one of Context.dev's supported countries. When provided, Context.dev fetches the target page from that country.", QueryPath: "country", }, - &requestflag.Flag[[]string]{ + &requestflag.Flag[any]{ Name: "exclude-selector", Usage: `CSS selectors to remove from the result. Applied after includeSelectors. Exclusion takes precedence: an element matching both is removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]".`, QueryPath: "excludeSelectors", @@ -572,21 +630,21 @@ var webWebScrapeHTML = requestflag.WithInnerFlags(cli.Command{ Usage: "Optional outbound HTTP headers forwarded only to the target URL, sent as deep-object query params such as headers[X-Custom]=value. When provided, caching is bypassed: the result is neither read from nor written to cache.", QueryPath: "headers", }, - &requestflag.Flag[bool]{ + &requestflag.Flag[any]{ Name: "include-frames", Usage: "When true, iframes are rendered inline into the returned HTML.", Default: false, QueryPath: "includeFrames", }, - &requestflag.Flag[[]string]{ + &requestflag.Flag[any]{ Name: "include-selector", Usage: `CSS selectors. When provided, only matching subtrees (and their descendants) are kept and everything else is dropped. When omitted, the entire document is kept. Examples: "article.main", "#content", "[role=main]".`, QueryPath: "includeSelectors", }, - &requestflag.Flag[int64]{ + &requestflag.Flag[*int64]{ Name: "max-age-ms", Usage: "Return a cached result if a prior scrape for the same parameters exists and is younger than this many milliseconds. Defaults to 1 day (86400000 ms) when omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.", - Default: 86400000, + Default: requestflag.Ptr[int64](86400000), QueryPath: "maxAgeMs", }, &requestflag.Flag[map[string]any]{ @@ -595,28 +653,39 @@ var webWebScrapeHTML = requestflag.WithInnerFlags(cli.Command{ Default: map[string]any{"shouldParse": true, "ocr": false}, QueryPath: "pdf", }, - &requestflag.Flag[bool]{ + &requestflag.Flag[any]{ Name: "settle-animations", Usage: "When true, waits briefly for CSS and transition animations to settle before extracting HTML. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages.", Default: false, QueryPath: "settleAnimations", }, + &requestflag.Flag[[]string]{ + Name: "tag", + Usage: "Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters.", + QueryPath: "tags", + }, &requestflag.Flag[int64]{ Name: "timeout-ms", Usage: "Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).", QueryPath: "timeoutMS", }, - &requestflag.Flag[bool]{ + &requestflag.Flag[any]{ Name: "use-main-content-only", Usage: "When true, return only the page's main content in the HTML response, excluding headers, footers, sidebars, and navigation when detectable.", Default: false, QueryPath: "useMainContentOnly", }, - &requestflag.Flag[int64]{ + &requestflag.Flag[*int64]{ Name: "wait-for-ms", - Usage: "Optional browser wait time in milliseconds after initial page load. Min: 0. Max: 30000 (30 seconds). ", + Usage: "Optional browser wait time in milliseconds after initial page load. Min: 0. Max: 30000 (30 seconds).", QueryPath: "waitForMs", }, + &requestflag.Flag[string]{ + Name: "zdr", + Usage: "Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true.", + Default: "disabled", + QueryPath: "zdr", + }, }, Action: handleWebWebScrapeHTML, HideHelpCommand: true, @@ -627,12 +696,12 @@ var webWebScrapeHTML = requestflag.WithInnerFlags(cli.Command{ Usage: "Last 1-based PDF page to parse. When omitted, parsing ends at the last page. Must be greater than or equal to start when both are provided.", InnerField: "end", }, - &requestflag.InnerFlag[bool]{ + &requestflag.InnerFlag[any]{ Name: "pdf.ocr", Usage: "When true, detect and OCR images embedded in the selected PDF pages, inserting recognized text at each image's position in page reading order while preserving the PDF text layer. This is separate from automatic scanned-PDF OCR fallback.", InnerField: "ocr", }, - &requestflag.InnerFlag[bool]{ + &requestflag.InnerFlag[any]{ Name: "pdf.should-parse", Usage: "When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and a 400 WEBSITE_ACCESS_ERROR is returned.", InnerField: "shouldParse", @@ -647,7 +716,7 @@ var webWebScrapeHTML = requestflag.WithInnerFlags(cli.Command{ var webWebScrapeImages = requestflag.WithInnerFlags(cli.Command{ Name: "web-scrape-images", - Usage: "Extract image assets from a web page, including standard URLs, inline SVGs, data\nURIs, responsive image sources, metadata, CSS backgrounds, video posters, and\nembeds. The base request costs 1 credit. When enrichment is enabled, the entire\ncall costs 5 credits.", + Usage: "Extract image assets from a web page, including standard URLs, inline SVGs, data\nURIs, responsive image sources, metadata, CSS backgrounds, video posters, and\nembeds. The base request costs 1 credit, or 2 credits with browser actions. When\nenrichment is enabled, the entire call costs 5 credits, including requests that\nalso use actions.", Suggest: true, Flags: []cli.Flag{ &requestflag.Flag[string]{ @@ -656,7 +725,12 @@ var webWebScrapeImages = requestflag.WithInnerFlags(cli.Command{ Required: true, QueryPath: "url", }, - &requestflag.Flag[bool]{ + &requestflag.Flag[any]{ + Name: "action", + Usage: "Optional browser actions executed in array order after the page loads and before content is captured. Requires a paid plan. Send a JSON array in the query parameter. Maximum: 5 actions.", + QueryPath: "actions", + }, + &requestflag.Flag[any]{ Name: "dedupe", Usage: "When true, visually duplicate images are removed: every image is loaded and perceptually hashed, and only the highest-resolution copy of each duplicate group is kept. Images that cannot be downloaded or hashed are kept. Default: false.", Default: false, @@ -672,20 +746,25 @@ var webWebScrapeImages = requestflag.WithInnerFlags(cli.Command{ Usage: "Optional outbound HTTP headers forwarded only to the target URL, sent as deep-object query params such as headers[X-Custom]=value. When provided, caching is bypassed: the result is neither read from nor written to cache.", QueryPath: "headers", }, - &requestflag.Flag[int64]{ + &requestflag.Flag[*int64]{ Name: "max-age-ms", Usage: "Reuse a cached result this many milliseconds old or newer. Default: 86400000 (1 day). Set to 0 to bypass cache. Maximum: 2592000000 (30 days).", - Default: 86400000, + Default: requestflag.Ptr[int64](86400000), QueryPath: "maxAgeMs", }, + &requestflag.Flag[[]string]{ + Name: "tag", + Usage: "Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters.", + QueryPath: "tags", + }, &requestflag.Flag[int64]{ Name: "timeout-ms", Usage: "Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).", QueryPath: "timeoutMS", }, - &requestflag.Flag[int64]{ + &requestflag.Flag[*int64]{ Name: "wait-for-ms", - Usage: "Optional browser wait time in milliseconds after initial page load before collecting images. Min: 0. Max: 30000 (30 seconds). ", + Usage: "Optional browser wait time in milliseconds after initial page load before collecting images. Min: 0. Max: 30000 (30 seconds).", QueryPath: "waitForMs", }, }, @@ -693,12 +772,12 @@ var webWebScrapeImages = requestflag.WithInnerFlags(cli.Command{ HideHelpCommand: true, }, map[string][]requestflag.HasOuterFlag{ "enrichment": { - &requestflag.InnerFlag[bool]{ + &requestflag.InnerFlag[any]{ Name: "enrichment.classification", Usage: "Classify each image by visual asset type.", InnerField: "classification", }, - &requestflag.InnerFlag[bool]{ + &requestflag.InnerFlag[any]{ Name: "enrichment.hosted-url", Usage: "Host materializable images on the Brand.dev CDN and return their URL and MIME type.", InnerField: "hostedUrl", @@ -708,7 +787,7 @@ var webWebScrapeImages = requestflag.WithInnerFlags(cli.Command{ Usage: "Per-image enrichment timeout in milliseconds. Default: 30000. Maximum: 60000.", InnerField: "maxTimePerMs", }, - &requestflag.InnerFlag[bool]{ + &requestflag.InnerFlag[any]{ Name: "enrichment.resolution", Usage: "Measure image width and height when possible.", InnerField: "resolution", @@ -727,12 +806,17 @@ var webWebScrapeMd = requestflag.WithInnerFlags(cli.Command{ Required: true, QueryPath: "url", }, + &requestflag.Flag[any]{ + Name: "action", + Usage: "Optional browser actions executed in array order after the page loads and before content is captured. Requires a paid plan. Send a JSON array in the query parameter. Maximum: 5 actions.", + QueryPath: "actions", + }, &requestflag.Flag[string]{ Name: "country", - Usage: "Two-letter ISO 3166-1 alpha-2 country code for the website request location. When provided, Context.dev fetches the target page from that country.", + Usage: "Two-letter ISO 3166-1 alpha-2 country code identifying a supported Context.dev residential proxy exit location. Must be one of Context.dev's supported countries. When provided, Context.dev fetches the target page from that country.", QueryPath: "country", }, - &requestflag.Flag[[]string]{ + &requestflag.Flag[any]{ Name: "exclude-selector", Usage: `CSS selectors to remove before conversion to Markdown. Applied after includeSelectors. Exclusion takes precedence: an element matching both is removed. Examples: "nav", "footer", ".ad-banner", "[aria-hidden=true]".`, QueryPath: "excludeSelectors", @@ -742,33 +826,33 @@ var webWebScrapeMd = requestflag.WithInnerFlags(cli.Command{ Usage: "Optional outbound HTTP headers forwarded only to the target URL, sent as deep-object query params such as headers[X-Custom]=value. When provided, caching is bypassed: the result is neither read from nor written to cache.", QueryPath: "headers", }, - &requestflag.Flag[bool]{ + &requestflag.Flag[any]{ Name: "include-frames", Usage: "When true, the contents of iframes are rendered to Markdown.", Default: false, QueryPath: "includeFrames", }, - &requestflag.Flag[bool]{ + &requestflag.Flag[any]{ Name: "include-images", Usage: "Include image references in Markdown output", Default: false, QueryPath: "includeImages", }, - &requestflag.Flag[bool]{ + &requestflag.Flag[any]{ Name: "include-links", Usage: "Preserve hyperlinks in Markdown output", Default: true, QueryPath: "includeLinks", }, - &requestflag.Flag[[]string]{ + &requestflag.Flag[any]{ Name: "include-selector", Usage: `CSS selectors. When provided, only matching HTML subtrees (and their descendants) are kept before conversion to Markdown. When omitted, the entire document is kept. Examples: "article.main", "#content", "[role=main]".`, QueryPath: "includeSelectors", }, - &requestflag.Flag[int64]{ + &requestflag.Flag[*int64]{ Name: "max-age-ms", Usage: "Return a cached result if a prior scrape for the same parameters exists and is younger than this many milliseconds. Defaults to 1 day (86400000 ms) when omitted. Max is 30 days (2592000000 ms). Set to 0 to always scrape fresh.", - Default: 86400000, + Default: requestflag.Ptr[int64](86400000), QueryPath: "maxAgeMs", }, &requestflag.Flag[map[string]any]{ @@ -777,34 +861,45 @@ var webWebScrapeMd = requestflag.WithInnerFlags(cli.Command{ Default: map[string]any{"shouldParse": true, "ocr": false}, QueryPath: "pdf", }, - &requestflag.Flag[bool]{ + &requestflag.Flag[any]{ Name: "settle-animations", Usage: "When true, waits briefly for CSS and transition animations to settle before converting to Markdown. Defaults to false. This adds a bit of latency in exchange for more stable output on animated pages.", Default: false, QueryPath: "settleAnimations", }, - &requestflag.Flag[bool]{ + &requestflag.Flag[any]{ Name: "shorten-base64-images", Usage: "Shorten base64-encoded image data in the Markdown output", Default: true, QueryPath: "shortenBase64Images", }, + &requestflag.Flag[[]string]{ + Name: "tag", + Usage: "Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters.", + QueryPath: "tags", + }, &requestflag.Flag[int64]{ Name: "timeout-ms", Usage: "Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).", QueryPath: "timeoutMS", }, - &requestflag.Flag[bool]{ + &requestflag.Flag[any]{ Name: "use-main-content-only", Usage: "Extract only the main content of the page, excluding headers, footers, sidebars, and navigation", Default: false, QueryPath: "useMainContentOnly", }, - &requestflag.Flag[int64]{ + &requestflag.Flag[*int64]{ Name: "wait-for-ms", - Usage: "Optional browser wait time in milliseconds after initial page load before converting the page to Markdown. Min: 0. Max: 30000 (30 seconds). ", + Usage: "Optional browser wait time in milliseconds after initial page load before converting the page to Markdown. Min: 0. Max: 30000 (30 seconds).", QueryPath: "waitForMs", }, + &requestflag.Flag[string]{ + Name: "zdr", + Usage: "Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true.", + Default: "disabled", + QueryPath: "zdr", + }, }, Action: handleWebWebScrapeMd, HideHelpCommand: true, @@ -815,12 +910,12 @@ var webWebScrapeMd = requestflag.WithInnerFlags(cli.Command{ Usage: "Last 1-based PDF page to parse. When omitted, parsing ends at the last page. Must be greater than or equal to start when both are provided.", InnerField: "end", }, - &requestflag.InnerFlag[bool]{ + &requestflag.InnerFlag[any]{ Name: "pdf.ocr", Usage: "When true, detect and OCR images embedded in the selected PDF pages, inserting recognized text at each image's position in page reading order while preserving the PDF text layer. This is separate from automatic scanned-PDF OCR fallback.", InnerField: "ocr", }, - &requestflag.InnerFlag[bool]{ + &requestflag.InnerFlag[any]{ Name: "pdf.should-parse", Usage: "When true, PDF URLs are fetched and parsed. When false, PDF URLs are skipped and a 400 WEBSITE_ACCESS_ERROR is returned.", InnerField: "shouldParse", @@ -855,6 +950,16 @@ var webWebScrapeSitemap = cli.Command{ Default: 10000, QueryPath: "maxLinks", }, + &requestflag.Flag[string]{ + Name: "sitemap-url", + Usage: "Optional explicit sitemap URL. When provided, exactly this sitemap is crawled instead of discovering the domain's sitemaps.", + QueryPath: "sitemapUrl", + }, + &requestflag.Flag[[]string]{ + Name: "tag", + Usage: "Optional comma-separated caller-defined tags for tracking this request. Tags are recorded on the request's usage log and can be used to filter usage on the dashboard usage page. Up to 20 tags, each 1-50 characters.", + QueryPath: "tags", + }, &requestflag.Flag[int64]{ Name: "timeout-ms", Usage: "Optional timeout in milliseconds for the request. If the request takes longer than this value, it will be aborted with a 408 status code. Maximum allowed value is 300000ms (5 minutes).", @@ -865,6 +970,12 @@ var webWebScrapeSitemap = cli.Command{ Usage: "Optional RE2-compatible regex pattern. Only URLs matching this pattern are returned and counted against maxLinks.", QueryPath: "urlRegex", }, + &requestflag.Flag[string]{ + Name: "zdr", + Usage: "Set to enabled to bypass shared caches and omit request and response content from retained usage logs. Requires zero data retention to be enabled for your organization (contact support@context.dev), otherwise the request fails with ZDR_NOT_ENABLED. Successful ZDR responses include X-Context-ZDR: true.", + Default: "disabled", + QueryPath: "zdr", + }, }, Action: handleWebWebScrapeSitemap, HideHelpCommand: true, diff --git a/pkg/cmd/web_test.go b/pkg/cmd/web_test.go index ab8545d..8eda947 100644 --- a/pkg/cmd/web_test.go +++ b/pkg/cmd/web_test.go @@ -26,7 +26,10 @@ func TestWebExtract(t *testing.T) { "--max-depth", "0", "--max-pages", "1", "--pdf", "{end: 1, shouldParse: true, start: 1}", + "--settle-animations=true", "--stop-after-ms", "10000", + "--tag", "production", + "--tag", "team-alpha", "--timeout-ms", "1000", "--wait-for-ms", "0", ) @@ -53,7 +56,10 @@ func TestWebExtract(t *testing.T) { "--pdf.end", "1", "--pdf.should-parse=true", "--pdf.start", "1", + "--settle-animations=true", "--stop-after-ms", "10000", + "--tag", "production", + "--tag", "team-alpha", "--timeout-ms", "1000", "--wait-for-ms", "0", ) @@ -79,7 +85,11 @@ func TestWebExtract(t *testing.T) { " end: 1\n" + " shouldParse: true\n" + " start: 1\n" + + "settleAnimations: true\n" + "stopAfterMs: 10000\n" + + "tags:\n" + + " - production\n" + + " - team-alpha\n" + "timeoutMS: 1000\n" + "waitForMs: 0\n") mocktest.TestRunMockTestWithPipeAndFlags( @@ -99,6 +109,8 @@ func TestWebExtractCompetitors(t *testing.T) { "web", "extract-competitors", "--domain", "xxx", "--num-competitors", "1", + "--tag", "production", + "--tag", "team-alpha", "--timeout-ms", "1000", ) }) @@ -112,8 +124,10 @@ func TestWebExtractFonts(t *testing.T) { "--api-key", "string", "web", "extract-fonts", "--direct-url", "https://example.com", - "--domain", "domain", - "--max-age-ms", "86400000", + "--domain", "xxx", + "--max-age-ms", "0", + "--tag", "production", + "--tag", "team-alpha", "--timeout-ms", "1000", ) }) @@ -128,8 +142,10 @@ func TestWebExtractStyleguide(t *testing.T) { "web", "extract-styleguide", "--color-scheme", "light", "--direct-url", "https://example.com", - "--domain", "domain", - "--max-age-ms", "86400000", + "--domain", "xxx", + "--max-age-ms", "0", + "--tag", "production", + "--tag", "team-alpha", "--timeout-ms", "1000", ) }) @@ -145,15 +161,18 @@ func TestWebScreenshot(t *testing.T) { "--color-scheme", "light", "--country", "de", "--direct-url", "https://example.com", - "--domain", "domain", + "--domain", "xxx", "--full-screenshot", "true", - "--handle-cookie-popup", "true", + "--handle-cookie-popup", "'true'", "--max-age-ms", "0", "--page", "login", "--scroll-offset", "0", - "--timeout-ms", "1000", + "--tag", "production", + "--tag", "team-alpha", + "--timeout-ms", "1", "--viewport", "{height: 240, width: 240}", "--wait-for-ms", "0", + "--zdr", "enabled", ) }) @@ -169,16 +188,19 @@ func TestWebScreenshot(t *testing.T) { "--color-scheme", "light", "--country", "de", "--direct-url", "https://example.com", - "--domain", "domain", + "--domain", "xxx", "--full-screenshot", "true", - "--handle-cookie-popup", "true", + "--handle-cookie-popup", "'true'", "--max-age-ms", "0", "--page", "login", "--scroll-offset", "0", - "--timeout-ms", "1000", + "--tag", "production", + "--tag", "team-alpha", + "--timeout-ms", "1", "--viewport.height", "240", "--viewport.width", "240", "--wait-for-ms", "0", + "--zdr", "enabled", ) }) } @@ -198,6 +220,8 @@ func TestWebSearch(t *testing.T) { "--markdown-options", "{enabled: true, includeFrames: true, includeImages: true, includeLinks: true, maxAgeMs: 0, pdf: {end: 1, shouldParse: true, start: 1}, shortenBase64Images: true, timeoutMS: 1000, useMainContentOnly: true, waitForMs: 0}", "--num-results", "10", "--query-fanout=true", + "--tag", "production", + "--tag", "team-alpha", "--timeout-ms", "1000", ) }) @@ -228,6 +252,8 @@ func TestWebSearch(t *testing.T) { "--markdown-options.wait-for-ms", "0", "--num-results", "10", "--query-fanout=true", + "--tag", "production", + "--tag", "team-alpha", "--timeout-ms", "1000", ) }) @@ -258,6 +284,9 @@ func TestWebSearch(t *testing.T) { " waitForMs: 0\n" + "numResults: 10\n" + "queryFanout: true\n" + + "tags:\n" + + " - production\n" + + " - team-alpha\n" + "timeoutMS: 1000\n") mocktest.TestRunMockTestWithPipeAndFlags( t, pipeData, @@ -289,10 +318,13 @@ func TestWebWebCrawlMd(t *testing.T) { "--settle-animations=true", "--shorten-base64-images=true", "--stop-after-ms", "10000", + "--tag", "production", + "--tag", "team-alpha", "--timeout-ms", "1000", "--url-regex", "^https?://[^/]+/blog/", "--use-main-content-only=true", "--wait-for-ms", "0", + "--zdr", "enabled", ) }) @@ -323,10 +355,13 @@ func TestWebWebCrawlMd(t *testing.T) { "--settle-animations=true", "--shorten-base64-images=true", "--stop-after-ms", "10000", + "--tag", "production", + "--tag", "team-alpha", "--timeout-ms", "1000", "--url-regex", "^https?://[^/]+/blog/", "--use-main-content-only=true", "--wait-for-ms", "0", + "--zdr", "enabled", ) }) @@ -354,10 +389,14 @@ func TestWebWebCrawlMd(t *testing.T) { "settleAnimations: true\n" + "shortenBase64Images: true\n" + "stopAfterMs: 10000\n" + + "tags:\n" + + " - production\n" + + " - team-alpha\n" + "timeoutMS: 1000\n" + "urlRegex: ^https?://[^/]+/blog/\n" + "useMainContentOnly: true\n" + - "waitForMs: 0\n") + "waitForMs: 0\n" + + "zdr: enabled\n") mocktest.TestRunMockTestWithPipeAndFlags( t, pipeData, "--api-key", "string", @@ -374,17 +413,21 @@ func TestWebWebScrapeHTML(t *testing.T) { "--api-key", "string", "web", "web-scrape-html", "--url", "https://example.com", + "--action", "[{do: wait, timeMs: 0}]", "--country", "de", - "--exclude-selector", "string", + "--exclude-selector", "[x]", "--headers", "{foo: J!}", - "--include-frames=true", - "--include-selector", "string", + "--include-frames", "'true'", + "--include-selector", "[x]", "--max-age-ms", "0", - "--pdf", "{end: 1, ocr: true, shouldParse: true, start: 1}", - "--settle-animations=true", - "--timeout-ms", "1000", - "--use-main-content-only=true", + "--pdf", "{end: 1, ocr: 'true', shouldParse: 'true', start: 1}", + "--settle-animations", "'true'", + "--tag", "production", + "--tag", "team-alpha", + "--timeout-ms", "1", + "--use-main-content-only", "'true'", "--wait-for-ms", "0", + "--zdr", "enabled", ) }) @@ -398,20 +441,24 @@ func TestWebWebScrapeHTML(t *testing.T) { "--api-key", "string", "web", "web-scrape-html", "--url", "https://example.com", + "--action", "[{do: wait, timeMs: 0}]", "--country", "de", - "--exclude-selector", "string", + "--exclude-selector", "[x]", "--headers", "{foo: J!}", - "--include-frames=true", - "--include-selector", "string", + "--include-frames", "'true'", + "--include-selector", "[x]", "--max-age-ms", "0", "--pdf.end", "1", - "--pdf.ocr=true", - "--pdf.should-parse=true", + "--pdf.ocr", "true", + "--pdf.should-parse", "true", "--pdf.start", "1", - "--settle-animations=true", - "--timeout-ms", "1000", - "--use-main-content-only=true", + "--settle-animations", "'true'", + "--tag", "production", + "--tag", "team-alpha", + "--timeout-ms", "1", + "--use-main-content-only", "'true'", "--wait-for-ms", "0", + "--zdr", "enabled", ) }) } @@ -424,11 +471,14 @@ func TestWebWebScrapeImages(t *testing.T) { "--api-key", "string", "web", "web-scrape-images", "--url", "https://example.com", - "--dedupe=true", - "--enrichment", "{classification: true, hostedUrl: true, maxTimePerMs: 1, resolution: true}", + "--action", "[{do: wait, timeMs: 0}]", + "--dedupe", "'true'", + "--enrichment", "{classification: 'true', hostedUrl: 'true', maxTimePerMs: 1, resolution: 'true'}", "--headers", "{foo: J!}", "--max-age-ms", "0", - "--timeout-ms", "1000", + "--tag", "production", + "--tag", "team-alpha", + "--timeout-ms", "1", "--wait-for-ms", "0", ) }) @@ -443,14 +493,17 @@ func TestWebWebScrapeImages(t *testing.T) { "--api-key", "string", "web", "web-scrape-images", "--url", "https://example.com", - "--dedupe=true", - "--enrichment.classification=true", - "--enrichment.hosted-url=true", + "--action", "[{do: wait, timeMs: 0}]", + "--dedupe", "'true'", + "--enrichment.classification", "true", + "--enrichment.hosted-url", "true", "--enrichment.max-time-per-ms", "1", - "--enrichment.resolution=true", + "--enrichment.resolution", "true", "--headers", "{foo: J!}", "--max-age-ms", "0", - "--timeout-ms", "1000", + "--tag", "production", + "--tag", "team-alpha", + "--timeout-ms", "1", "--wait-for-ms", "0", ) }) @@ -464,20 +517,24 @@ func TestWebWebScrapeMd(t *testing.T) { "--api-key", "string", "web", "web-scrape-md", "--url", "https://example.com", + "--action", "[{do: wait, timeMs: 0}]", "--country", "de", - "--exclude-selector", "string", + "--exclude-selector", "[x]", "--headers", "{foo: J!}", - "--include-frames=true", - "--include-images=true", - "--include-links=true", - "--include-selector", "string", + "--include-frames", "'true'", + "--include-images", "'true'", + "--include-links", "'true'", + "--include-selector", "[x]", "--max-age-ms", "0", - "--pdf", "{end: 1, ocr: true, shouldParse: true, start: 1}", - "--settle-animations=true", - "--shorten-base64-images=true", - "--timeout-ms", "1000", - "--use-main-content-only=true", + "--pdf", "{end: 1, ocr: 'true', shouldParse: 'true', start: 1}", + "--settle-animations", "'true'", + "--shorten-base64-images", "'true'", + "--tag", "production", + "--tag", "team-alpha", + "--timeout-ms", "1", + "--use-main-content-only", "'true'", "--wait-for-ms", "0", + "--zdr", "enabled", ) }) @@ -491,23 +548,27 @@ func TestWebWebScrapeMd(t *testing.T) { "--api-key", "string", "web", "web-scrape-md", "--url", "https://example.com", + "--action", "[{do: wait, timeMs: 0}]", "--country", "de", - "--exclude-selector", "string", + "--exclude-selector", "[x]", "--headers", "{foo: J!}", - "--include-frames=true", - "--include-images=true", - "--include-links=true", - "--include-selector", "string", + "--include-frames", "'true'", + "--include-images", "'true'", + "--include-links", "'true'", + "--include-selector", "[x]", "--max-age-ms", "0", "--pdf.end", "1", - "--pdf.ocr=true", - "--pdf.should-parse=true", + "--pdf.ocr", "true", + "--pdf.should-parse", "true", "--pdf.start", "1", - "--settle-animations=true", - "--shorten-base64-images=true", - "--timeout-ms", "1000", - "--use-main-content-only=true", + "--settle-animations", "'true'", + "--shorten-base64-images", "'true'", + "--tag", "production", + "--tag", "team-alpha", + "--timeout-ms", "1", + "--use-main-content-only", "'true'", "--wait-for-ms", "0", + "--zdr", "enabled", ) }) } @@ -519,11 +580,15 @@ func TestWebWebScrapeSitemap(t *testing.T) { t, "--api-key", "string", "web", "web-scrape-sitemap", - "--domain", "domain", + "--domain", "xxx", "--headers", "{foo: J!}", "--max-links", "1", - "--timeout-ms", "1000", + "--sitemap-url", "https://example.com", + "--tag", "production", + "--tag", "team-alpha", + "--timeout-ms", "1", "--url-regex", "^https?://[^/]+/blog/", + "--zdr", "enabled", ) }) }