Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 18 additions & 15 deletions .github/workflows/idempotency-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,15 @@ jobs:
matrix:
description:
- "oas::petstore"
- "apisguru::twitter.com:current"
- "apisguru::notion.com"
- "apisguru::stripe.com"
- "https://api.twitter.com/2/openapi.json"
- "https://developers.notion.com/openapi.json"
- "https://raw.githubusercontent.com/stripe/openapi/refs/heads/master/latest/openapi.spec3.json"
- "https://raw.githubusercontent.com/googlemaps/openapi-specification/refs/tags/v1.22.5/dist/google-maps-platform-openapi3.yml"
- "apisguru::meraki.com"
- "https://raw.githubusercontent.com/meraki/openapi/refs/heads/master/openapi/spec3.json"
- "https://developers.pipedrive.com/docs/api/v1/openapi.yaml"
- "apisguru::twilio.com:api"
- "apisguru::docusign.net"
- "apisguru::github.com:api.github.com"
- "https://raw.githubusercontent.com/twilio/twilio-oai/refs/heads/main/spec/yaml/twilio_api_v2010.yaml"
- "https://raw.githubusercontent.com/docusign/OpenAPI-Specifications/refs/heads/master/esignature.rest.swagger-v2.1.json"
- "https://raw.githubusercontent.com/github/rest-api-description/refs/heads/main/descriptions/api.github.com/api.github.com.json"
- "apisguru::apis.guru"
steps:
- uses: actions/checkout@v7
Expand All @@ -65,7 +65,10 @@ jobs:
- uses: actions/upload-artifact@v7
with:
name: description-${{ steps.replace_url.outputs.ARTKEY }}
path: it/openapi.yaml
# Output is either openapi.json or openapi.yml. Try to upload the one that is found.
path: |
it/openapi.yaml
it/openapi.json

idempotency:
runs-on: ubuntu-latest
Expand All @@ -86,15 +89,15 @@ jobs:
- "./tests/Kiota.Builder.IntegrationTests/InheritingErrors.yaml"
- "./tests/Kiota.Builder.IntegrationTests/NoUnderscoresInModel.yaml"
- "oas::petstore"
- "apisguru::twitter.com:current"
- "apisguru::notion.com"
- "apisguru::stripe.com"
- "https://api.twitter.com/2/openapi.json"
- "https://developers.notion.com/openapi.json"
- "https://raw.githubusercontent.com/stripe/openapi/refs/heads/master/latest/openapi.spec3.json"
- "https://raw.githubusercontent.com/googlemaps/openapi-specification/refs/tags/v1.22.5/dist/google-maps-platform-openapi3.yml"
- "apisguru::meraki.com"
- "https://raw.githubusercontent.com/meraki/openapi/refs/heads/master/openapi/spec3.json"
- "https://developers.pipedrive.com/docs/api/v1/openapi.yaml"
- "apisguru::twilio.com:api"
- "apisguru::docusign.net"
- "apisguru::github.com:api.github.com"
- "https://raw.githubusercontent.com/twilio/twilio-oai/refs/heads/main/spec/yaml/twilio_api_v2010.yaml"
- "https://raw.githubusercontent.com/docusign/OpenAPI-Specifications/refs/heads/master/esignature.rest.swagger-v2.1.json"
- "https://raw.githubusercontent.com/github/rest-api-description/refs/heads/main/descriptions/api.github.com/api.github.com.json"
- "apisguru::apis.guru"
steps:
- uses: actions/checkout@v7
Expand Down
33 changes: 18 additions & 15 deletions .github/workflows/integration-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,15 @@ jobs:
matrix:
description:
- "oas::petstore"
- "apisguru::twitter.com:current"
- "apisguru::notion.com"
- "apisguru::stripe.com"
- "https://api.twitter.com/2/openapi.json"
- "https://developers.notion.com/openapi.json"
- "https://raw.githubusercontent.com/stripe/openapi/refs/heads/master/latest/openapi.spec3.json"
- "https://raw.githubusercontent.com/googlemaps/openapi-specification/refs/tags/v1.22.5/dist/google-maps-platform-openapi3.yml"
- "apisguru::meraki.com"
- "https://raw.githubusercontent.com/meraki/openapi/refs/heads/master/openapi/spec3.json"
- "https://developers.pipedrive.com/docs/api/v1/openapi.yaml"
- "apisguru::twilio.com:api"
- "apisguru::docusign.net"
- "apisguru::github.com:api.github.com"
- "https://raw.githubusercontent.com/twilio/twilio-oai/refs/heads/main/spec/yaml/twilio_api_v2010.yaml"
- "https://raw.githubusercontent.com/docusign/OpenAPI-Specifications/refs/heads/master/esignature.rest.swagger-v2.1.json"
- "https://raw.githubusercontent.com/github/rest-api-description/refs/heads/main/descriptions/api.github.com/api.github.com.json"
- "apisguru::apis.guru"
- "https://github.com/microsoftgraph/msgraph-metadata/raw/refs/heads/master/openapi/beta/openapi.yaml"
steps:
Expand All @@ -66,7 +66,10 @@ jobs:
- uses: actions/upload-artifact@v7
with:
name: description-${{ steps.replace_url.outputs.ARTKEY }}
path: it/openapi.yaml
# Output is either openapi.json or openapi.yml. Try to upload the one that is found.
path: |
it/openapi.yaml
it/openapi.json

integration:
runs-on: ubuntu-latest
Expand All @@ -92,15 +95,15 @@ jobs:
- "./tests/Kiota.Builder.IntegrationTests/DiscriminatorSample.yaml"
- "./tests/Kiota.Builder.IntegrationTests/ModelWithDefaultValues.json"
- "oas::petstore"
- "apisguru::twitter.com:current"
- "apisguru::notion.com"
- "apisguru::stripe.com"
- "https://api.twitter.com/2/openapi.json"
- "https://developers.notion.com/openapi.json"
- "https://raw.githubusercontent.com/stripe/openapi/refs/heads/master/latest/openapi.spec3.json"
- "https://raw.githubusercontent.com/googlemaps/openapi-specification/refs/tags/v1.22.5/dist/google-maps-platform-openapi3.yml"
- "apisguru::meraki.com"
- "https://raw.githubusercontent.com/meraki/openapi/refs/heads/master/openapi/spec3.json"
- "https://developers.pipedrive.com/docs/api/v1/openapi.yaml"
- "apisguru::twilio.com:api"
- "apisguru::docusign.net"
- "apisguru::github.com:api.github.com"
- "https://raw.githubusercontent.com/twilio/twilio-oai/refs/heads/main/spec/yaml/twilio_api_v2010.yaml"
- "https://raw.githubusercontent.com/docusign/OpenAPI-Specifications/refs/heads/master/esignature.rest.swagger-v2.1.json"
- "https://raw.githubusercontent.com/github/rest-api-description/refs/heads/main/descriptions/api.github.com/api.github.com.json"
- "apisguru::apis.guru"
- "https://github.com/microsoftgraph/msgraph-metadata/raw/refs/heads/master/openapi/beta/openapi.yaml"

Expand Down
120 changes: 89 additions & 31 deletions it/Readme.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
# Running the Integration Tests
# Integration tests

## Running the Integration Tests

Run the following steps to locally run the integration tests.

Expand All @@ -14,53 +16,69 @@ Generate the code:
./it/generate-code.ps1 -descriptionUrl ${FILE/URL} -language ${LANG}
```

Instead of publishing, you could also build the project and use the "-Dev" switch that executes Kiota from "src\kiota\bin\Debug\net8.0":

```bash
./it/generate-code.ps1 -descriptionUrl ${FILE/URL} -language ${LANG} -Dev
```

And finally run the test:

```bash
./it/exec-cmd.ps1 -descriptionUrl ${FILE/URL} -language ${LANG}
```

# Surface area / DOM diff test
## Suppressions

The surface area test guards against **binary/source breaking changes** that a change to
kiota's generation logic could introduce into downstream SDKs (notably the Microsoft Graph
SDKs). It compares the public-API surface (`kiota-dom-export.txt`) produced by the currently
published `Microsoft.OpenApi.Kiota` NuGet tool (baseline) against the surface produced by a
locally built kiota (current changeset).
Some API files do not work for a specific target language. There are two ways to define suppressions:

In CI this runs via `.github/workflows/surface-area-tests.yml` and feeds the resulting patch
to the [`microsoftgraph/kiota-dom-export-diff-tool`](https://github.com/microsoftgraph/kiota-dom-export-diff-tool)
`tool` action with `fail-on-removal: true`: any removed surface line fails the check.
### Suppress the full test

To run it locally, first publish a development build of `kiota`:
This will execute the test in Github CI anyway and maybe an error will happen, but the test will not be marked as "failed" (see "integration-tests.yml" and "get-is-suppressed.ps1")

```bash
dotnet publish ./src/kiota/kiota.csproj -c Release -p:PublishSingleFile=true -p:PublishReadyToRun=true -o ./publish -f net10.0
```

Then generate the baseline/current exports and a diff patch:

```bash
./it/compare-dom-export.ps1 -descriptionUrl ${FILE/URL} -language ${LANG}
"https://www.sample.com/sample-api.json": {
"MockServerITFolder": "sampleapi",
"Suppressions": [
{
"Language": "ruby",
"Rationale": "Feature x in file is not supported for ruby."
}
]
}
```

### Suppress creating certain API paths

Useful options:
This will tell Kiota not to generate the code for the specified endpoint. Use it if something inside this endpoint definition is either invalid OpenAPI
or Kiota cannot handle it.

* `-baselineVersion` — pin a specific published `Microsoft.OpenApi.Kiota` version (defaults to latest stable).
* `-patchPath` — where to write the unified diff (defaults to `./<language>-dom-export.patch`).
* `-kiotaExec` — path to the locally built kiota (defaults to `./publish/kiota`).
* `-additionalArguments` — extra arguments forwarded to both `kiota generate` invocations.
```
"https://www.sample.com/sample-api.json": {
"MockServerITFolder": "sampleapi",
"ExcludePatterns": [
{
"Pattern": "/users/*/gpg_keys",
"Rationale": "invalid data type for argument XYZ"
},
{
"Pattern": "/repos/{owner}/{repo}/releases#POST",
"Rationale": "here is something wrong, too"
},
{
"Language": "typescript",
"Pattern": "/repos/{owner}/{repo}/contents/{path}#GET",
"Rationale": "an error only for typescript"
}
]
}
```

**Interpreting results:** removed lines (`-`) in the patch indicate API removed/changed since
the baseline release — i.e. potential breaking changes. Added lines (`+`) are additive and do
not fail the check. Because the baseline is the last published release, the diff reflects every
change since that release, not just the current changeset.
The second snippet demonstrates that you can suppress also creating a method for just one of the HTTP methods of an endpoint.
The third sample shows a suppression only for a single language.

**Intended breaking changes:** review the explanations produced by the diff tool. If a breaking
change is deliberate, it must be acknowledged through normal PR review (and, where configured, a
required-check override) before merging.

# MockServer tests
## MockServer tests

The OpenAPI description can be published to a mock server, and you can execute tests that call this API.

Expand Down Expand Up @@ -104,3 +122,43 @@ The handling depends on the language:

If you create e.g. a custom "csproj" file for your test (might be necessary if you need additional dependencies), add this file
to the Dependabot config so that dependencies are updated.

# Surface area / DOM diff test

The surface area test guards against **binary/source breaking changes** that a change to
kiota's generation logic could introduce into downstream SDKs (notably the Microsoft Graph
SDKs). It compares the public-API surface (`kiota-dom-export.txt`) produced by the currently
published `Microsoft.OpenApi.Kiota` NuGet tool (baseline) against the surface produced by a
locally built kiota (current changeset).

In CI this runs via `.github/workflows/surface-area-tests.yml` and feeds the resulting patch
to the [`microsoftgraph/kiota-dom-export-diff-tool`](https://github.com/microsoftgraph/kiota-dom-export-diff-tool)
`tool` action with `fail-on-removal: true`: any removed surface line fails the check.

To run it locally, first publish a development build of `kiota`:

```bash
dotnet publish ./src/kiota/kiota.csproj -c Release -p:PublishSingleFile=true -p:PublishReadyToRun=true -o ./publish -f net10.0
```

Then generate the baseline/current exports and a diff patch:

```bash
./it/compare-dom-export.ps1 -descriptionUrl ${FILE/URL} -language ${LANG}
```

Useful options:

* `-baselineVersion` — pin a specific published `Microsoft.OpenApi.Kiota` version (defaults to latest stable).
* `-patchPath` — where to write the unified diff (defaults to `./<language>-dom-export.patch`).
* `-kiotaExec` — path to the locally built kiota (defaults to `./publish/kiota`).
* `-additionalArguments` — extra arguments forwarded to both `kiota generate` invocations.

**Interpreting results:** removed lines (`-`) in the patch indicate API removed/changed since
the baseline release — i.e. potential breaking changes. Added lines (`+`) are additive and do
not fail the check. Because the baseline is the last published release, the diff reflects every
change since that release, not just the current changeset.

**Intended breaking changes:** review the explanations produced by the diff tool. If a breaking
change is deliberate, it must be acknowledged through normal PR review (and, where configured, a
required-check override) before merging.
59 changes: 46 additions & 13 deletions it/compare-generation.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,13 @@ switch ($dev) {
}
}

$targetOpenapiPath = Join-Path -Path $PSScriptRoot -ChildPath "openapi.yaml"
#api description might be a JSON or a yaml file (or a apigurus name, which is hopefully a yaml file).
if ($descriptionUrl.ToLower().EndsWith(".json")) {
$targetOpenapiPath = Join-Path -Path $PSScriptRoot -ChildPath "openapi.json"
}
else {
$targetOpenapiPath = Join-Path -Path $PSScriptRoot -ChildPath "openapi.yaml"
}
if (Test-Path $targetOpenapiPath) {
Remove-Item $targetOpenapiPath
}
Expand Down Expand Up @@ -85,21 +91,14 @@ if ($firstGenerationProcess.ExitCode -ne 0 -or $secondGenerationProcess.ExitCode
exit 1
}

# Remove variable output files
Remove-Item (Join-Path -Path $tmpFolder1 -ChildPath "kiota-lock.json")
if (Test-Path (Join-Path -Path $tmpFolder1 -ChildPath ".kiota.log")) {
Remove-Item -Force (Join-Path -Path $tmpFolder1 -ChildPath ".kiota.log")
}
Remove-Item (Join-Path -Path $tmpFolder2 -ChildPath "kiota-lock.json")
if (Test-Path (Join-Path -Path $tmpFolder2 -ChildPath ".kiota.log")) {
Remove-Item -Force (Join-Path -Path $tmpFolder2 -ChildPath ".kiota.log")
}
# Exclude variable output files
$excludes = @("kiota-lock.json",".kiota.log")

# Compare hashes
$HashString1 = (Get-ChildItem $tmpFolder1 -Recurse | where { ! $_.PSIsContainer } | Get-FileHash -Algorithm MD5).Hash | Out-String
$HashString1 = (Get-ChildItem $tmpFolder1 -Recurse -Exclude $excludes | where { ! $_.PSIsContainer } | Get-FileHash -Algorithm MD5).Hash | Out-String
Get-FileHash -InputStream ([IO.MemoryStream]::new([char[]]$HashString1))

$HashString2 = (Get-ChildItem $tmpFolder2 -Recurse | where { ! $_.PSIsContainer } | Get-FileHash -Algorithm MD5).Hash | Out-String
$HashString2 = (Get-ChildItem $tmpFolder2 -Recurse -Exclude $excludes | where { ! $_.PSIsContainer } | Get-FileHash -Algorithm MD5).Hash | Out-String
Get-FileHash -InputStream ([IO.MemoryStream]::new([char[]]$HashString2))

Write-Output "Folder 1: $tmpFolder1"
Expand All @@ -117,11 +116,45 @@ if ($HashString1 -eq $HashString2) {
}
else {
Write-Host "The content of the folders is NOT identical"

# Print differing files: get hash for each file and compare. Print files the differ.
function Get-FileHashMap {
param([string]$Root)
$map = @{}
Get-ChildItem -Path $Root -File -Recurse -Exclude $excludes | ForEach-Object {
$rel = $_.FullName.Substring($Root.Length).TrimStart('\','/')
$hash = (Get-FileHash -Algorithm MD5 -Path $_.FullName).Hash
$map[$rel] = $hash
}
return $map
}

$map1 = Get-FileHashMap -Root $tmpFolder1
$map2 = Get-FileHashMap -Root $tmpFolder2

$allKeys = ($map1.Keys + $map2.Keys) | Sort-Object -Unique

foreach ($key in $allKeys) {
$in1 = $map1.ContainsKey($key)
$in2 = $map2.ContainsKey($key)

if (-not $in1) {
Write-Host "Missing in ${tmpFolder1}: $key" # Should not happen
} elseif (-not $in2) {
Write-Host "Missing in ${tmpFolder2}: $key" # Should not happen
} else {
if ($map1[$key] -ne $map2[$key]) {
Write-Host "Difference found in: $key"
}
}
}

# Now zip the results in the upload folder:
$resultsFolder = Join-Path -Path $rootPath -ChildPath "idempotency-results"
if (Test-Path $resultsFolder) {
Remove-Item $resultsFolder -Force -Verbose -Recurse
}
New-Item -ItemType Directory -Path $resultsFolder -Force
New-Item -ItemType Directory -Path $resultsFolder -Force | Out-Null

if ($dev -eq $false) {
$archivePath1 = Join-Path $resultsFolder -ChildPath "idempotency-folder1.zip"
Expand Down
Loading
Loading