From 4a45e67c6189375bbcde0d08f25ca0e4ef92e4a9 Mon Sep 17 00:00:00 2001 From: Ravi Shankar Date: Wed, 5 Aug 2026 12:06:26 -0700 Subject: [PATCH 1/2] feat: nscale api change Signed-off-by: Ravi Shankar --- docs/providers/nscale.md | 35 +++-- pkg/providers/nscale/provider.go | 45 +++---- pkg/providers/nscale/provider_sim.go | 2 +- pkg/providers/nscale/provider_test.go | 182 ++++++++++++++++++++------ 4 files changed, 187 insertions(+), 77 deletions(-) diff --git a/docs/providers/nscale.md b/docs/providers/nscale.md index ebfb0766..ed8dc212 100644 --- a/docs/providers/nscale.md +++ b/docs/providers/nscale.md @@ -5,23 +5,24 @@ The `nscale` topology provider reads topology data from the Nscale Radar API and The provider uses two Nscale APIs: - **Radar API**: returns each instance's network path via `GET /v1/topology` -- **Instance API**: returns instance metadata via `GET /v2/instances?organizationID=®ionID=` +- **Placement Servers API**: returns server metadata via `GET /api/v2/placements/{placementID}/servers` -The Radar response supplies the provider instance ID, switch path, and optional block ID. The Instance API response maps provider instance IDs to hostnames using `metadata.id` and `metadata.name`; this is used by the Slurm engine when Topograph discovers Slurm nodes automatically. +The Radar response supplies the provider instance ID, switch path, and optional block ID. The Placement Servers API response maps provider instance IDs to hostnames using `metadata.id` and `metadata.name`; this is used by the Slurm engine when Topograph discovers Slurm nodes automatically. ## When to Use This Provider Use this provider for Nscale environments where Radar is the topology source. It is most commonly used with the Slurm engine to generate `topology.conf` from the current Slurm node list. -If the request payload supplies explicit `nodes`, Topograph uses those instance ID to node name mappings directly. If `nodes` is omitted and the Slurm engine is used, Topograph runs `scontrol show nodes -o`, asks the Nscale Instance API for the instance catalog in the configured region, and keeps entries whose `metadata.name` matches a Slurm node name. +If the request payload supplies explicit `nodes`, Topograph uses those instance ID to node name mappings directly. If `nodes` is omitted and the Slurm engine is used, Topograph runs `scontrol show nodes -o`, asks the Nscale Placement Servers API for the server catalog in the configured placement, and keeps entries whose `metadata.name` matches a Slurm node name. ## Prerequisites - A Radar API endpoint reachable from the Topograph host -- An Instance API endpoint reachable from the Topograph host +- A Placement Servers API endpoint reachable from the Topograph host - An Nscale organization ID -- An API token with permission to read topology and instance metadata +- An API token with permission to read topology and placement server metadata - The Nscale region ID for the cluster +- The Nscale placement ID for the rack-scale group of servers - For Slurm auto-discovery, `scontrol` must be available to the Topograph process ## Credentials @@ -29,8 +30,8 @@ If the request payload supplies explicit `nodes`, Topograph uses those instance | Field | Required | Description | |---|---|---| | `org` | Yes | Nscale organization ID | -| `token` | Yes | Bearer token used for Radar and Instance API requests | -| `region` | Required for Slurm auto-discovery | Nscale region ID used for Instance API lookup and Slurm region assignment | +| `token` | Yes | Bearer token used for Radar and Placement Servers API requests | +| `region` | Required for Slurm auto-discovery | Nscale region ID used for Slurm region assignment | Store credentials in a YAML file: @@ -54,6 +55,7 @@ Credentials can also be supplied directly in the topology request payload under |---|---|---| | `radarApiUrl` | Yes | Base URL for the Radar API, for example `https://radar.example.com` | | `instanceApiUrl` | Yes | Base URL for the Instance API, for example `https://api.example.com` | +| `placementId` | Yes | Nscale placement ID used to scope the Placement Servers API request | | `trimTiers` | No | Number of highest topology tiers to trim from output. Defaults to `0` | The top-level Topograph `pageSize` setting controls pagination for the Radar topology request. @@ -76,6 +78,7 @@ credentialsPath: /etc/topograph/nscale-credentials.yaml providerParams: radarApiUrl: https://radar.example.com instanceApiUrl: https://api.example.com + placementId: engineParams: plugin: topology/tree @@ -95,7 +98,8 @@ Example request payload: }, "params": { "radarApiUrl": "https://radar.example.com", - "instanceApiUrl": "https://api.example.com" + "instanceApiUrl": "https://api.example.com", + "placementId": "" } }, "engine": { @@ -120,7 +124,8 @@ If you already have the instance ID to hostname mapping, you can include it expl }, "params": { "radarApiUrl": "https://radar.example.com", - "instanceApiUrl": "https://api.example.com" + "instanceApiUrl": "https://api.example.com", + "placementId": "" } }, "engine": { @@ -159,28 +164,30 @@ Each returned instance is translated as follows: | `network_node_path[2]` | Leaf tier | | `block_id` | Accelerator / NVLink domain | -For Slurm auto-discovery, the provider also fetches instance metadata: +For Slurm auto-discovery, the provider fetches server metadata from the Placement Servers API: ```text -GET /v2/instances?organizationID=®ionID= +GET /api/v2/placements//servers Authorization: Bearer ``` +The response is an array of placement server objects. The provider extracts `metadata.id` (the server's unique identifier) and `metadata.name` (the hostname) from each entry and builds an instance-ID-to-hostname map. + It builds the same map produced by: ```bash curl -s -H "Authorization: Bearer $TOKEN" \ - "$INSTANCE_API_URL/v2/instances?organizationID=$ORG®ionID=$REGION" \ + "$INSTANCE_API_URL/api/v2/placements/$PLACEMENT_ID/servers" \ | jq -r '.[] | "\(.metadata.id)\t\(.metadata.name)"' ``` ## Verifying the Output -First verify that the Instance API returns the hostnames Slurm knows: +First verify that the Placement Servers API returns the hostnames Slurm knows: ```bash curl -s -H "Authorization: Bearer $TOKEN" \ - "$INSTANCE_API_URL/v2/instances?organizationID=$ORG®ionID=$REGION" \ + "$INSTANCE_API_URL/api/v2/placements/$PLACEMENT_ID/servers" \ | jq -r '.[] | "\(.metadata.id)\t\(.metadata.name)"' ``` diff --git a/pkg/providers/nscale/provider.go b/pkg/providers/nscale/provider.go index 44a0156a..aae6bf31 100644 --- a/pkg/providers/nscale/provider.go +++ b/pkg/providers/nscale/provider.go @@ -22,8 +22,8 @@ import ( const ( NAME = "nscale" - urlTopologyPath = "/v1/topology" - urlInstancesPath = "/v2/instances" + urlTopologyPath = "/v1/topology" + urlPlacementServersPath = "/api/v2/placements/%s/servers" ) type baseProvider struct { @@ -35,6 +35,7 @@ type baseProvider struct { type ProviderParams struct { RadarApiUrl string `mapstructure:"radarApiUrl"` InstanceAPIUrl string `mapstructure:"instanceApiUrl"` + PlacementID string `mapstructure:"placementId"` TrimTiers int `mapstructure:"trimTiers"` } @@ -46,7 +47,7 @@ type Credentials struct { type Client interface { Topology(context.Context, string, int, int) ([]InstanceTopology, error) - Instances(context.Context, string) (map[string]string, error) + PlacementServers(context.Context, string) (map[string]string, error) } // nscaleClient is a topology and instance API client. @@ -69,11 +70,11 @@ type TopologyResult struct { Instances []InstanceTopology `json:"results"` } -type instance struct { - Metadata instanceMetadata `json:"metadata"` +type placementServer struct { + Metadata placementServerMetadata `json:"metadata"` } -type instanceMetadata struct { +type placementServerMetadata struct { ID string `json:"id"` Name string `json:"name"` } @@ -103,32 +104,29 @@ func (c *nscaleClient) Topology(ctx context.Context, region string, pageSize, of return resp.Instances, nil } -func (c *nscaleClient) Instances(ctx context.Context, region string) (map[string]string, error) { +func (c *nscaleClient) PlacementServers(ctx context.Context, placementID string) (map[string]string, error) { headers := map[string]string{ "Authorization": "Bearer " + c.token, } - query := map[string]string{ - "organizationID": c.org, - "regionID": region, - } - f := httpreq.GetRequestFunc(ctx, http.MethodGet, headers, query, nil, c.instanceAPIURL, urlInstancesPath) + path := fmt.Sprintf(urlPlacementServersPath, placementID) + f := httpreq.GetRequestFunc(ctx, http.MethodGet, headers, nil, nil, c.instanceAPIURL, path) body, httpErr := httpreq.DoRequestWithRetries(f, false) if httpErr != nil { return nil, httpErr } - instances := []instance{} - if err := json.Unmarshal(body, &instances); err != nil { + servers := []placementServer{} + if err := json.Unmarshal(body, &servers); err != nil { return nil, httperr.NewError(http.StatusBadGateway, err.Error()) } - i2n := make(map[string]string, len(instances)) - for _, instance := range instances { - if instance.Metadata.ID == "" || instance.Metadata.Name == "" { + i2n := make(map[string]string, len(servers)) + for _, s := range servers { + if s.Metadata.ID == "" || s.Metadata.Name == "" { continue } - i2n[instance.Metadata.ID] = instance.Metadata.Name + i2n[s.Metadata.ID] = s.Metadata.Name } return i2n, nil @@ -178,6 +176,9 @@ func getParams(params map[string]any) (*ProviderParams, error) { if len(p.InstanceAPIUrl) == 0 { return nil, fmt.Errorf("missing 'instanceApiUrl'") } + if len(p.PlacementID) == 0 { + return nil, fmt.Errorf("missing 'placementId'") + } return p, nil } @@ -208,13 +209,13 @@ func (p *baseProvider) GenerateTopologyConfig(ctx context.Context, pageSize *int // Instances2NodeMap implements slurm.instanceMapper func (p *Provider) Instances2NodeMap(ctx context.Context, nodes []string) (map[string]string, error) { - if len(p.creds.Region) == 0 { - return nil, fmt.Errorf("missing 'region'") + if len(p.params.PlacementID) == 0 { + return nil, fmt.Errorf("missing 'placementId'") } - instances, err := p.client.Instances(ctx, p.creds.Region) + instances, err := p.client.PlacementServers(ctx, p.params.PlacementID) if err != nil { - return nil, fmt.Errorf("failed to get instances: %v", err) + return nil, fmt.Errorf("failed to get placement servers: %v", err) } if len(nodes) == 0 { return instances, nil diff --git a/pkg/providers/nscale/provider_sim.go b/pkg/providers/nscale/provider_sim.go index dd0dc14e..bb256c8d 100644 --- a/pkg/providers/nscale/provider_sim.go +++ b/pkg/providers/nscale/provider_sim.go @@ -67,7 +67,7 @@ func (c *simClient) Topology(ctx context.Context, _ string, pageSize, offset int return resp, nil } -func (c *simClient) Instances(_ context.Context, _ string) (map[string]string, error) { +func (c *simClient) PlacementServers(_ context.Context, _ string) (map[string]string, error) { i2n := make(map[string]string, len(c.model.Nodes)) for _, node := range c.model.Nodes { i2n[node.ID] = node.ID diff --git a/pkg/providers/nscale/provider_test.go b/pkg/providers/nscale/provider_test.go index 0ad2746f..d37a2595 100644 --- a/pkg/providers/nscale/provider_test.go +++ b/pkg/providers/nscale/provider_test.go @@ -29,6 +29,7 @@ func TestLoader(t *testing.T) { Params: map[string]any{ "radarApiUrl": "https://radar.test.com", "instanceApiUrl": "https://instances.test.com", + "placementId": "placement-1", }, Creds: map[string]any{ "org": "org", @@ -42,6 +43,7 @@ func TestLoader(t *testing.T) { config: providers.Config{ Params: map[string]any{ "instanceApiUrl": "https://instances.test.com", + "placementId": "placement-1", }, Creds: map[string]any{ "org": "org", @@ -55,6 +57,7 @@ func TestLoader(t *testing.T) { config: providers.Config{ Params: map[string]any{ "radarApiUrl": "https://radar.test.com", + "placementId": "placement-1", }, Creds: map[string]any{ "org": "org", @@ -64,12 +67,27 @@ func TestLoader(t *testing.T) { err: "missing 'instanceApiUrl'", }, { - name: "Case 4: missing org", + name: "Case 4: missing placementId", config: providers.Config{ Params: map[string]any{ "radarApiUrl": "https://radar.test.com", "instanceApiUrl": "https://instances.test.com", }, + Creds: map[string]any{ + "org": "org", + "token": "token", + }, + }, + err: "missing 'placementId'", + }, + { + name: "Case 5: missing org", + config: providers.Config{ + Params: map[string]any{ + "radarApiUrl": "https://radar.test.com", + "instanceApiUrl": "https://instances.test.com", + "placementId": "placement-1", + }, Creds: map[string]any{ "token": "token", }, @@ -77,11 +95,12 @@ func TestLoader(t *testing.T) { err: "missing 'org'", }, { - name: "Case 5: missing token", + name: "Case 6: missing token", config: providers.Config{ Params: map[string]any{ "radarApiUrl": "https://radar.test.com", "instanceApiUrl": "https://instances.test.com", + "placementId": "placement-1", }, Creds: map[string]any{ "org": "org", @@ -108,30 +127,59 @@ func TestLoader(t *testing.T) { } } -func TestInstances2NodeMap(t *testing.T) { - ctx := context.Background() - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - require.Equal(t, http.MethodGet, r.Method) - require.Equal(t, "/v2/instances", r.URL.Path) - require.Equal(t, "org", r.URL.Query().Get("organizationID")) - require.Equal(t, "region", r.URL.Query().Get("regionID")) - require.Equal(t, "Bearer token", r.Header.Get("Authorization")) - - w.Header().Set("Content-Type", "application/json") - _, err := w.Write([]byte(`[ - {"metadata":{"id":"instance-1","name":"node-1"}}, - {"metadata":{"id":"instance-2","name":"node-2"}}, - {"metadata":{"id":"instance-3","name":"outside-node"}} - ]`)) - require.NoError(t, err) - })) - defer server.Close() +const placementServersResponse = `[ + { + "metadata": { + "id": "psrv-7f3d9d5d2a7c4e32", + "name": "training-workers-0", + "organizationId": "9a8c6370-4065-4d4a-9da0-7678df40cd9d", + "projectId": "e36c058a-8eba-4f5b-91f4-f6ffb983795c", + "creationTime": "2026-04-28T11:04:00Z", + "createdBy": "john.doe@example.com", + "provisioningStatus": "provisioned", + "healthStatus": "healthy" + }, + "status": { + "regionId": "c7568e2d-f9ab-453d-9a3a-51375f78426b", + "reservationId": "a64f9269-36e0-4312-b8d1-52d93d569b7b", + "placementId": "b8ce034e-fccb-4d6c-a0e0-af3e3f346715", + "networkId": "61f0ad85-3001-41cb-824a-e6a047668dfe", + "powerState": "Running", + "privateIP": "10.0.0.12", + "publicIP": "203.0.113.12", + "macAddress": "fa:16:3e:7c:11:8a" + } + }, + { + "metadata": { + "id": "psrv-950ab3259a1443da", + "name": "training-workers-1", + "organizationId": "9a8c6370-4065-4d4a-9da0-7678df40cd9d", + "projectId": "e36c058a-8eba-4f5b-91f4-f6ffb983795c", + "creationTime": "2026-04-28T11:04:02Z", + "createdBy": "john.doe@example.com", + "provisioningStatus": "provisioned", + "healthStatus": "healthy" + }, + "status": { + "regionId": "c7568e2d-f9ab-453d-9a3a-51375f78426b", + "reservationId": "a64f9269-36e0-4312-b8d1-52d93d569b7b", + "placementId": "b8ce034e-fccb-4d6c-a0e0-af3e3f346715", + "networkId": "61f0ad85-3001-41cb-824a-e6a047668dfe", + "powerState": "Running", + "privateIP": "10.0.0.13", + "macAddress": "fa:16:3e:11:70:2f" + } + } +]` +func newTestProvider(t *testing.T, ctx context.Context, serverURL string) *Provider { + t.Helper() provider, httpErr := Loader(ctx, providers.Config{ Params: map[string]any{ "radarApiUrl": "https://radar.test.com", - "instanceApiUrl": server.URL, + "instanceApiUrl": serverURL, + "placementId": "placement-1", }, Creds: map[string]any{ "org": "org", @@ -140,34 +188,88 @@ func TestInstances2NodeMap(t *testing.T) { }, }) require.Nil(t, httpErr) + return provider.(*Provider) +} + +func TestInstances2NodeMap(t *testing.T) { + ctx := context.Background() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, http.MethodGet, r.Method) + require.Equal(t, "/api/v2/placements/placement-1/servers", r.URL.Path) + require.Equal(t, "Bearer token", r.Header.Get("Authorization")) - i2n, err := provider.(*Provider).Instances2NodeMap(ctx, []string{"node-1", "node-2"}) + w.Header().Set("Content-Type", "application/json") + _, err := w.Write([]byte(placementServersResponse)) + require.NoError(t, err) + })) + defer server.Close() + + p := newTestProvider(t, ctx, server.URL) + + i2n, err := p.Instances2NodeMap(ctx, []string{"training-workers-0", "training-workers-1"}) require.NoError(t, err) require.Equal(t, map[string]string{ - "instance-1": "node-1", - "instance-2": "node-2", + "psrv-7f3d9d5d2a7c4e32": "training-workers-0", + "psrv-950ab3259a1443da": "training-workers-1", }, i2n) - i2n, err = provider.(*Provider).Instances2NodeMap(ctx, nil) + i2n, err = p.Instances2NodeMap(ctx, nil) require.NoError(t, err) require.Equal(t, map[string]string{ - "instance-1": "node-1", - "instance-2": "node-2", - "instance-3": "outside-node", + "psrv-7f3d9d5d2a7c4e32": "training-workers-0", + "psrv-950ab3259a1443da": "training-workers-1", }, i2n) +} - provider, httpErr = Loader(ctx, providers.Config{ - Params: map[string]any{ - "radarApiUrl": "https://radar.test.com", - "instanceApiUrl": server.URL, +func TestPlacementServersErrors(t *testing.T) { + ctx := context.Background() + + tests := []struct { + name string + statusCode int + body string + }{ + { + name: "400 invalid request", + statusCode: http.StatusBadRequest, + body: `{"error":"invalid_request","error_description":"request body invalid","trace_id":"57bc14d9bd461f0b5a72db830149b67a"}`, }, - Creds: map[string]any{ - "org": "org", - "token": "token", + { + name: "401 authentication failed", + statusCode: http.StatusUnauthorized, + body: `{"error":"access_denied","error_description":"authentication failed","trace_id":"57bc14d9bd461f0b5a72db830149b67a"}`, }, - }) - require.Nil(t, httpErr) + { + name: "403 forbidden", + statusCode: http.StatusForbidden, + body: `{"error":"forbidden","error_description":"user credentials do not have the required privileges","trace_id":"57bc14d9bd461f0b5a72db830149b67a"}`, + }, + { + name: "404 not found", + statusCode: http.StatusNotFound, + body: `{"error":"not_found","error_description":"the requested resource does not exist","trace_id":"57bc14d9bd461f0b5a72db830149b67a"}`, + }, + { + name: "500 server error", + statusCode: http.StatusInternalServerError, + body: `{"error":"server_error","error_description":"failed to token claim","trace_id":"57bc14d9bd461f0b5a72db830149b67a"}`, + }, + } - _, err = provider.(*Provider).Instances2NodeMap(ctx, []string{"node-1"}) - require.EqualError(t, err, "missing 'region'") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(tt.statusCode) + _, err := w.Write([]byte(tt.body)) + require.NoError(t, err) + })) + defer server.Close() + + p := newTestProvider(t, ctx, server.URL) + _, err := p.Instances2NodeMap(ctx, nil) + require.ErrorContains(t, err, "failed to get placement servers") + }) + } } From 8554ce8d1807beccbfe5f0360488468aca0c59e4 Mon Sep 17 00:00:00 2001 From: Ravi Shankar Date: Fri, 21 Aug 2026 15:57:23 -0700 Subject: [PATCH 2/2] Updated to use list placements api to get the placementIds in the cluster Signed-off-by: Ravi Shankar --- CHANGELOG.md | 8 + docs/providers/nscale.md | 88 +++++--- pkg/providers/nscale/provider.go | 67 +++++- pkg/providers/nscale/provider_sim.go | 4 + pkg/providers/nscale/provider_test.go | 297 ++++++++++++++++++++------ 5 files changed, 359 insertions(+), 105 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a5da707..04280cb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +### Changed + +- The `nscale` provider's Slurm auto-discovery no longer requires a statically configured `placementId`. It now lists placements for the credentialed organization and region via the Nscale Placements API, then queries the Placement Servers API for each placement and merges the results into the instance-to-node map. + +### Removed + +- `nscale` provider `placementId` parameter — placements are discovered dynamically instead. `region` credential is now required whenever Slurm auto-discovery (`Instances2NodeMap`) is used. + ### Added - Documentation diagrams for architecture, Kubernetes, Slinky, Slurm topology formats, and engine outputs now ship as community-variant SVG and PNG assets with automatic dark/light mode switching (`` / `prefers-color-scheme` in docs; `#gh-light-mode-only` / `#gh-dark-mode-only` in README). diff --git a/docs/providers/nscale.md b/docs/providers/nscale.md index ed8dc212..0689c43d 100644 --- a/docs/providers/nscale.md +++ b/docs/providers/nscale.md @@ -2,27 +2,27 @@ The `nscale` topology provider reads topology data from the Nscale Radar API and converts it into Topograph's canonical three-tier topology graph. -The provider uses two Nscale APIs: +The provider uses three Nscale APIs: - **Radar API**: returns each instance's network path via `GET /v1/topology` -- **Placement Servers API**: returns server metadata via `GET /api/v2/placements/{placementID}/servers` +- **Placements API**: lists the organization's placements in a region via `GET /api/v2/placements` +- **Placement Servers API**: returns server metadata for a placement via `GET /api/v2/placements/{placementID}/servers` -The Radar response supplies the provider instance ID, switch path, and optional block ID. The Placement Servers API response maps provider instance IDs to hostnames using `metadata.id` and `metadata.name`; this is used by the Slurm engine when Topograph discovers Slurm nodes automatically. +The Radar response supplies the provider instance ID, switch path, and optional block ID. For Slurm auto-discovery, the provider lists every placement for the configured organization and region, then queries the Placement Servers API for each one and merges the results into a single instance-ID-to-hostname map using `metadata.id` and `metadata.name`. ## When to Use This Provider Use this provider for Nscale environments where Radar is the topology source. It is most commonly used with the Slurm engine to generate `topology.conf` from the current Slurm node list. -If the request payload supplies explicit `nodes`, Topograph uses those instance ID to node name mappings directly. If `nodes` is omitted and the Slurm engine is used, Topograph runs `scontrol show nodes -o`, asks the Nscale Placement Servers API for the server catalog in the configured placement, and keeps entries whose `metadata.name` matches a Slurm node name. +If the request payload supplies explicit `nodes`, Topograph uses those instance ID to node name mappings directly. If `nodes` is omitted and the Slurm engine is used, Topograph runs `scontrol show nodes -o`, lists the organization's placements in the configured region via the Nscale Placements API, and asks the Placement Servers API for the server catalog of each placement. When `scontrol` returns a non-empty node list, only entries whose `metadata.name` exactly matches a Slurm node name are kept; if the node list is empty, every placement-server mapping is kept. ## Prerequisites - A Radar API endpoint reachable from the Topograph host -- A Placement Servers API endpoint reachable from the Topograph host +- A Placements / Placement Servers API endpoint reachable from the Topograph host - An Nscale organization ID -- An API token with permission to read topology and placement server metadata +- An API token with permission to read topology, placements, and placement server metadata - The Nscale region ID for the cluster -- The Nscale placement ID for the rack-scale group of servers - For Slurm auto-discovery, `scontrol` must be available to the Topograph process ## Credentials @@ -30,8 +30,8 @@ If the request payload supplies explicit `nodes`, Topograph uses those instance | Field | Required | Description | |---|---|---| | `org` | Yes | Nscale organization ID | -| `token` | Yes | Bearer token used for Radar and Placement Servers API requests | -| `region` | Required for Slurm auto-discovery | Nscale region ID used for Slurm region assignment | +| `token` | Yes | Bearer token used for Radar, Placements, and Placement Servers API requests | +| `region` | Required for Slurm auto-discovery | Nscale region ID used for Slurm region assignment and to scope the Placements API listing | Store credentials in a YAML file: @@ -54,8 +54,7 @@ Credentials can also be supplied directly in the topology request payload under | Field | Required | Description | |---|---|---| | `radarApiUrl` | Yes | Base URL for the Radar API, for example `https://radar.example.com` | -| `instanceApiUrl` | Yes | Base URL for the Instance API, for example `https://api.example.com` | -| `placementId` | Yes | Nscale placement ID used to scope the Placement Servers API request | +| `instanceApiUrl` | Yes | Base URL for the Placements and Placement Servers APIs, for example `https://api.example.com` | | `trimTiers` | No | Number of highest topology tiers to trim from output. Defaults to `0` | The top-level Topograph `pageSize` setting controls pagination for the Radar topology request. @@ -78,7 +77,6 @@ credentialsPath: /etc/topograph/nscale-credentials.yaml providerParams: radarApiUrl: https://radar.example.com instanceApiUrl: https://api.example.com - placementId: engineParams: plugin: topology/tree @@ -98,8 +96,7 @@ Example request payload: }, "params": { "radarApiUrl": "https://radar.example.com", - "instanceApiUrl": "https://api.example.com", - "placementId": "" + "instanceApiUrl": "https://api.example.com" } }, "engine": { @@ -124,8 +121,7 @@ If you already have the instance ID to hostname mapping, you can include it expl }, "params": { "radarApiUrl": "https://radar.example.com", - "instanceApiUrl": "https://api.example.com", - "placementId": "" + "instanceApiUrl": "https://api.example.com" } }, "engine": { @@ -164,33 +160,75 @@ Each returned instance is translated as follows: | `network_node_path[2]` | Leaf tier | | `block_id` | Accelerator / NVLink domain | -For Slurm auto-discovery, the provider fetches server metadata from the Placement Servers API: +For Slurm auto-discovery, the provider first lists the organization's placements in the configured region from the Placements API: + +```text +GET /api/v2/placements?organizationID=®ionID= +Authorization: Bearer +``` + +The response is an array of placement objects; the provider extracts `metadata.id` from each entry. It then fetches server metadata from the Placement Servers API for every placement ID returned: ```text GET /api/v2/placements//servers Authorization: Bearer ``` -The response is an array of placement server objects. The provider extracts `metadata.id` (the server's unique identifier) and `metadata.name` (the hostname) from each entry and builds an instance-ID-to-hostname map. +The response is an array of placement server objects. The provider extracts `metadata.id` (the server's unique identifier) and `metadata.name` (the hostname) from each entry and merges them across all placements into a single instance-ID-to-hostname map. It builds the same map produced by: ```bash -curl -s -H "Authorization: Bearer $TOKEN" \ - "$INSTANCE_API_URL/api/v2/placements/$PLACEMENT_ID/servers" \ - | jq -r '.[] | "\(.metadata.id)\t\(.metadata.name)"' +set -euo pipefail + +placement_ids=$(curl --fail --show-error --silent -H "Authorization: Bearer $TOKEN" \ + "$INSTANCE_API_URL/api/v2/placements?organizationID=$ORG_ID®ionID=$REGION_ID" \ + | jq -er '.[] | select(.metadata.id != "") | .metadata.id') + +for placement_id in $placement_ids; do + curl --fail --show-error --silent -H "Authorization: Bearer $TOKEN" \ + "$INSTANCE_API_URL/api/v2/placements/$placement_id/servers" \ + | jq -er '.[] | select(.metadata.id != "" and .metadata.name != "") | "\(.metadata.id)\t\(.metadata.name)"' +done ``` ## Verifying the Output -First verify that the Placement Servers API returns the hostnames Slurm knows: +When Slurm's node list (`scontrol show nodes -o`) is non-empty, `Instances2NodeMap` +only keeps a Placement Server entry when its `metadata.name` is an exact match for a +Slurm node name — there is no fuzzy or partial matching. If the node list is empty, +no filtering is applied and every placement-server mapping is retained. Before +triggering topology generation, compare the hostnames returned by the Placements and +Placement Servers APIs against Slurm's own node list and fail if they differ: ```bash -curl -s -H "Authorization: Bearer $TOKEN" \ - "$INSTANCE_API_URL/api/v2/placements/$PLACEMENT_ID/servers" \ - | jq -r '.[] | "\(.metadata.id)\t\(.metadata.name)"' +set -euo pipefail + +slurm_nodes=$(scontrol show nodes -o | grep -oE 'NodeName=[^ ]+' | cut -d= -f2 | sort -u) +[ -n "$slurm_nodes" ] || { echo "FAIL: scontrol returned no nodes"; exit 1; } + +placement_ids=$(curl --fail --show-error --silent -H "Authorization: Bearer $TOKEN" \ + "$INSTANCE_API_URL/api/v2/placements?organizationID=$ORG_ID®ionID=$REGION_ID" \ + | jq -er '.[] | select(.metadata.id != "") | .metadata.id') + +placement_hostnames=$(for placement_id in $placement_ids; do + curl --fail --show-error --silent -H "Authorization: Bearer $TOKEN" \ + "$INSTANCE_API_URL/api/v2/placements/$placement_id/servers" \ + | jq -er '.[] | select(.metadata.id != "" and .metadata.name != "") | .metadata.name' +done | sort -u) + +if diff <(printf '%s\n' "$slurm_nodes") <(printf '%s\n' "$placement_hostnames"); then + echo "OK: Placement Server hostnames match Slurm's node list" +else + echo "FAIL: Placement Server hostnames differ from Slurm's node list" + exit 1 +fi ``` +If the two lists differ, `Instances2NodeMap` will silently drop the mismatched nodes +from the generated topology rather than erroring, so this check should be run before +relying on Slurm auto-discovery. + Then trigger topology generation: ```bash diff --git a/pkg/providers/nscale/provider.go b/pkg/providers/nscale/provider.go index aae6bf31..38d4358d 100644 --- a/pkg/providers/nscale/provider.go +++ b/pkg/providers/nscale/provider.go @@ -23,6 +23,7 @@ const ( NAME = "nscale" urlTopologyPath = "/v1/topology" + urlPlacementsPath = "/api/v2/placements" urlPlacementServersPath = "/api/v2/placements/%s/servers" ) @@ -35,7 +36,6 @@ type baseProvider struct { type ProviderParams struct { RadarApiUrl string `mapstructure:"radarApiUrl"` InstanceAPIUrl string `mapstructure:"instanceApiUrl"` - PlacementID string `mapstructure:"placementId"` TrimTiers int `mapstructure:"trimTiers"` } @@ -47,10 +47,11 @@ type Credentials struct { type Client interface { Topology(context.Context, string, int, int) ([]InstanceTopology, error) + ListPlacements(ctx context.Context, org, region string) ([]string, error) PlacementServers(context.Context, string) (map[string]string, error) } -// nscaleClient is a topology and instance API client. +// nscaleClient is a Radar topology, Placements, and Placement Servers API client. type nscaleClient struct { radarAPIURL string instanceAPIURL string @@ -70,6 +71,14 @@ type TopologyResult struct { Instances []InstanceTopology `json:"results"` } +type placement struct { + Metadata placementMetadata `json:"metadata"` +} + +type placementMetadata struct { + ID string `json:"id"` +} + type placementServer struct { Metadata placementServerMetadata `json:"metadata"` } @@ -104,6 +113,37 @@ func (c *nscaleClient) Topology(ctx context.Context, region string, pageSize, of return resp.Instances, nil } +func (c *nscaleClient) ListPlacements(ctx context.Context, org, region string) ([]string, error) { + headers := map[string]string{ + "Authorization": "Bearer " + c.token, + } + query := map[string]string{ + "organizationID": org, + "regionID": region, + } + f := httpreq.GetRequestFunc(ctx, http.MethodGet, headers, query, nil, c.instanceAPIURL, urlPlacementsPath) + + body, httpErr := httpreq.DoRequestWithRetries(f, false) + if httpErr != nil { + return nil, httpErr + } + + placements := []placement{} + if err := json.Unmarshal(body, &placements); err != nil { + return nil, httperr.NewError(http.StatusBadGateway, err.Error()) + } + + ids := make([]string, 0, len(placements)) + for _, p := range placements { + if p.Metadata.ID == "" { + continue + } + ids = append(ids, p.Metadata.ID) + } + + return ids, nil +} + func (c *nscaleClient) PlacementServers(ctx context.Context, placementID string) (map[string]string, error) { headers := map[string]string{ "Authorization": "Bearer " + c.token, @@ -176,9 +216,6 @@ func getParams(params map[string]any) (*ProviderParams, error) { if len(p.InstanceAPIUrl) == 0 { return nil, fmt.Errorf("missing 'instanceApiUrl'") } - if len(p.PlacementID) == 0 { - return nil, fmt.Errorf("missing 'placementId'") - } return p, nil } @@ -209,14 +246,26 @@ func (p *baseProvider) GenerateTopologyConfig(ctx context.Context, pageSize *int // Instances2NodeMap implements slurm.instanceMapper func (p *Provider) Instances2NodeMap(ctx context.Context, nodes []string) (map[string]string, error) { - if len(p.params.PlacementID) == 0 { - return nil, fmt.Errorf("missing 'placementId'") + if len(p.creds.Region) == 0 { + return nil, fmt.Errorf("missing 'region'") } - instances, err := p.client.PlacementServers(ctx, p.params.PlacementID) + placementIDs, err := p.client.ListPlacements(ctx, p.creds.Org, p.creds.Region) if err != nil { - return nil, fmt.Errorf("failed to get placement servers: %v", err) + return nil, fmt.Errorf("failed to list placements: %w", err) + } + + instances := make(map[string]string) + for _, placementID := range placementIDs { + servers, err := p.client.PlacementServers(ctx, placementID) + if err != nil { + return nil, fmt.Errorf("failed to get placement servers for placement %s: %w", placementID, err) + } + for id, node := range servers { + instances[id] = node + } } + if len(nodes) == 0 { return instances, nil } diff --git a/pkg/providers/nscale/provider_sim.go b/pkg/providers/nscale/provider_sim.go index bb256c8d..e55e043c 100644 --- a/pkg/providers/nscale/provider_sim.go +++ b/pkg/providers/nscale/provider_sim.go @@ -67,6 +67,10 @@ func (c *simClient) Topology(ctx context.Context, _ string, pageSize, offset int return resp, nil } +func (c *simClient) ListPlacements(_ context.Context, _, _ string) ([]string, error) { + return []string{"sim"}, nil +} + func (c *simClient) PlacementServers(_ context.Context, _ string) (map[string]string, error) { i2n := make(map[string]string, len(c.model.Nodes)) for _, node := range c.model.Nodes { diff --git a/pkg/providers/nscale/provider_test.go b/pkg/providers/nscale/provider_test.go index d37a2595..892fe445 100644 --- a/pkg/providers/nscale/provider_test.go +++ b/pkg/providers/nscale/provider_test.go @@ -7,10 +7,14 @@ package nscale import ( "context" + "fmt" "net/http" "net/http/httptest" + "sync/atomic" "testing" + "time" + "github.com/NVIDIA/topograph/internal/httperr" "github.com/NVIDIA/topograph/pkg/providers" "github.com/stretchr/testify/require" ) @@ -29,7 +33,6 @@ func TestLoader(t *testing.T) { Params: map[string]any{ "radarApiUrl": "https://radar.test.com", "instanceApiUrl": "https://instances.test.com", - "placementId": "placement-1", }, Creds: map[string]any{ "org": "org", @@ -43,7 +46,6 @@ func TestLoader(t *testing.T) { config: providers.Config{ Params: map[string]any{ "instanceApiUrl": "https://instances.test.com", - "placementId": "placement-1", }, Creds: map[string]any{ "org": "org", @@ -57,7 +59,6 @@ func TestLoader(t *testing.T) { config: providers.Config{ Params: map[string]any{ "radarApiUrl": "https://radar.test.com", - "placementId": "placement-1", }, Creds: map[string]any{ "org": "org", @@ -67,27 +68,12 @@ func TestLoader(t *testing.T) { err: "missing 'instanceApiUrl'", }, { - name: "Case 4: missing placementId", + name: "Case 4: missing org", config: providers.Config{ Params: map[string]any{ "radarApiUrl": "https://radar.test.com", "instanceApiUrl": "https://instances.test.com", }, - Creds: map[string]any{ - "org": "org", - "token": "token", - }, - }, - err: "missing 'placementId'", - }, - { - name: "Case 5: missing org", - config: providers.Config{ - Params: map[string]any{ - "radarApiUrl": "https://radar.test.com", - "instanceApiUrl": "https://instances.test.com", - "placementId": "placement-1", - }, Creds: map[string]any{ "token": "token", }, @@ -95,12 +81,11 @@ func TestLoader(t *testing.T) { err: "missing 'org'", }, { - name: "Case 6: missing token", + name: "Case 5: missing token", config: providers.Config{ Params: map[string]any{ "radarApiUrl": "https://radar.test.com", "instanceApiUrl": "https://instances.test.com", - "placementId": "placement-1", }, Creds: map[string]any{ "org": "org", @@ -127,11 +112,16 @@ func TestLoader(t *testing.T) { } } -const placementServersResponse = `[ +const placementsResponse = `[ + {"metadata": {"id": "placement-1"}}, + {"metadata": {"id": "placement-2"}} +]` + +const placementServersResponseTmpl = `[ { "metadata": { - "id": "psrv-7f3d9d5d2a7c4e32", - "name": "training-workers-0", + "id": "%s", + "name": "%s", "organizationId": "9a8c6370-4065-4d4a-9da0-7678df40cd9d", "projectId": "e36c058a-8eba-4f5b-91f4-f6ffb983795c", "creationTime": "2026-04-28T11:04:00Z", @@ -142,34 +132,12 @@ const placementServersResponse = `[ "status": { "regionId": "c7568e2d-f9ab-453d-9a3a-51375f78426b", "reservationId": "a64f9269-36e0-4312-b8d1-52d93d569b7b", - "placementId": "b8ce034e-fccb-4d6c-a0e0-af3e3f346715", + "placementId": "%s", "networkId": "61f0ad85-3001-41cb-824a-e6a047668dfe", "powerState": "Running", "privateIP": "10.0.0.12", - "publicIP": "203.0.113.12", "macAddress": "fa:16:3e:7c:11:8a" } - }, - { - "metadata": { - "id": "psrv-950ab3259a1443da", - "name": "training-workers-1", - "organizationId": "9a8c6370-4065-4d4a-9da0-7678df40cd9d", - "projectId": "e36c058a-8eba-4f5b-91f4-f6ffb983795c", - "creationTime": "2026-04-28T11:04:02Z", - "createdBy": "john.doe@example.com", - "provisioningStatus": "provisioned", - "healthStatus": "healthy" - }, - "status": { - "regionId": "c7568e2d-f9ab-453d-9a3a-51375f78426b", - "reservationId": "a64f9269-36e0-4312-b8d1-52d93d569b7b", - "placementId": "b8ce034e-fccb-4d6c-a0e0-af3e3f346715", - "networkId": "61f0ad85-3001-41cb-824a-e6a047668dfe", - "powerState": "Running", - "privateIP": "10.0.0.13", - "macAddress": "fa:16:3e:11:70:2f" - } } ]` @@ -179,7 +147,6 @@ func newTestProvider(t *testing.T, ctx context.Context, serverURL string) *Provi Params: map[string]any{ "radarApiUrl": "https://radar.test.com", "instanceApiUrl": serverURL, - "placementId": "placement-1", }, Creds: map[string]any{ "org": "org", @@ -196,12 +163,25 @@ func TestInstances2NodeMap(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { require.Equal(t, http.MethodGet, r.Method) - require.Equal(t, "/api/v2/placements/placement-1/servers", r.URL.Path) require.Equal(t, "Bearer token", r.Header.Get("Authorization")) w.Header().Set("Content-Type", "application/json") - _, err := w.Write([]byte(placementServersResponse)) - require.NoError(t, err) + + switch r.URL.Path { + case "/api/v2/placements": + require.Equal(t, "org", r.URL.Query().Get("organizationID")) + require.Equal(t, "region", r.URL.Query().Get("regionID")) + _, err := w.Write([]byte(placementsResponse)) + require.NoError(t, err) + case "/api/v2/placements/placement-1/servers": + _, err := fmt.Fprintf(w, placementServersResponseTmpl, "psrv-7f3d9d5d2a7c4e32", "training-workers-0", "placement-1") + require.NoError(t, err) + case "/api/v2/placements/placement-2/servers": + _, err := fmt.Fprintf(w, placementServersResponseTmpl, "psrv-950ab3259a1443da", "training-workers-1", "placement-2") + require.NoError(t, err) + default: + t.Fatalf("unexpected request path %q", r.URL.Path) + } })) defer server.Close() @@ -222,38 +202,196 @@ func TestInstances2NodeMap(t *testing.T) { }, i2n) } -func TestPlacementServersErrors(t *testing.T) { +const placementServersMissingFieldsResponse = `[ + {"metadata": {"id": "psrv-1", "name": "training-workers-0"}}, + {"metadata": {"id": "", "name": "training-workers-1"}}, + {"metadata": {"id": "psrv-2", "name": ""}}, + {"metadata": {"id": "psrv-3", "name": "training-workers-2"}} +]` + +func TestPlacementServers(t *testing.T) { + ctx := context.Background() + + t.Run("excludes servers missing id or name", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, err := w.Write([]byte(placementServersMissingFieldsResponse)) + require.NoError(t, err) + })) + defer server.Close() + + c := &nscaleClient{instanceAPIURL: server.URL, token: "token"} + i2n, err := c.PlacementServers(ctx, "placement-1") + require.NoError(t, err) + require.Equal(t, map[string]string{ + "psrv-1": "training-workers-0", + "psrv-3": "training-workers-2", + }, i2n) + }) + + t.Run("malformed JSON returns an error", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, err := w.Write([]byte(`{not valid json`)) + require.NoError(t, err) + })) + defer server.Close() + + c := &nscaleClient{instanceAPIURL: server.URL, token: "token"} + i2n, err := c.PlacementServers(ctx, "placement-1") + require.Error(t, err) + require.Nil(t, i2n) + }) + + t.Run("canceled context returns promptly without retries", func(t *testing.T) { + var requests int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&requests, 1) + w.Header().Set("Content-Type", "application/json") + _, err := w.Write([]byte(placementsResponse)) + require.NoError(t, err) + })) + defer server.Close() + + cancelCtx, cancel := context.WithCancel(ctx) + cancel() + + c := &nscaleClient{instanceAPIURL: server.URL, token: "token"} + + start := time.Now() + i2n, err := c.PlacementServers(cancelCtx, "placement-1") + elapsed := time.Since(start) + + require.Error(t, err) + require.Nil(t, i2n) + require.Less(t, elapsed, time.Second, "canceled context should fail immediately, not retry with backoff") + require.Equal(t, int32(0), atomic.LoadInt32(&requests), "canceled context should not reach the server") + }) +} + +const placementsResponseWithEmptyID = `[ + {"metadata": {"id": "placement-1"}}, + {"metadata": {"id": ""}}, + {"metadata": {"id": "placement-2"}} +]` + +func TestListPlacements(t *testing.T) { + ctx := context.Background() + + t.Run("excludes placements missing id", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, err := w.Write([]byte(placementsResponseWithEmptyID)) + require.NoError(t, err) + })) + defer server.Close() + + c := &nscaleClient{instanceAPIURL: server.URL, token: "token"} + ids, err := c.ListPlacements(ctx, "org", "region") + require.NoError(t, err) + require.Equal(t, []string{"placement-1", "placement-2"}, ids) + }) + + t.Run("empty list returns no ids and no error", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, err := w.Write([]byte(`[]`)) + require.NoError(t, err) + })) + defer server.Close() + + c := &nscaleClient{instanceAPIURL: server.URL, token: "token"} + ids, err := c.ListPlacements(ctx, "org", "region") + require.NoError(t, err) + require.Empty(t, ids) + }) + + t.Run("malformed JSON returns HTTP 502", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, err := w.Write([]byte(`{not valid json`)) + require.NoError(t, err) + })) + defer server.Close() + + c := &nscaleClient{instanceAPIURL: server.URL, token: "token"} + ids, err := c.ListPlacements(ctx, "org", "region") + require.Error(t, err) + require.Nil(t, ids) + + var httpErr *httperr.Error + require.ErrorAs(t, err, &httpErr) + require.Equal(t, http.StatusBadGateway, httpErr.Code()) + }) + + t.Run("canceled context returns without issuing a request", func(t *testing.T) { + var requests int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&requests, 1) + w.Header().Set("Content-Type", "application/json") + _, err := w.Write([]byte(placementsResponse)) + require.NoError(t, err) + })) + defer server.Close() + + cancelCtx, cancel := context.WithCancel(ctx) + cancel() + + c := &nscaleClient{instanceAPIURL: server.URL, token: "token"} + + start := time.Now() + ids, err := c.ListPlacements(cancelCtx, "org", "region") + elapsed := time.Since(start) + + require.Error(t, err) + require.Nil(t, ids) + require.Less(t, elapsed, time.Second, "canceled context should fail immediately, not retry with backoff") + require.Equal(t, int32(0), atomic.LoadInt32(&requests), "canceled context should not reach the server") + }) +} + +func TestInstances2NodeMapErrors(t *testing.T) { ctx := context.Background() tests := []struct { - name string - statusCode int - body string + name string + failPlacements bool + statusCode int + body string + wantErrSubstring string }{ { - name: "400 invalid request", - statusCode: http.StatusBadRequest, - body: `{"error":"invalid_request","error_description":"request body invalid","trace_id":"57bc14d9bd461f0b5a72db830149b67a"}`, + name: "400 invalid request listing placements", + failPlacements: true, + statusCode: http.StatusBadRequest, + body: `{"error":"invalid_request","error_description":"request body invalid","trace_id":"57bc14d9bd461f0b5a72db830149b67a"}`, + wantErrSubstring: "failed to list placements", }, { - name: "401 authentication failed", - statusCode: http.StatusUnauthorized, - body: `{"error":"access_denied","error_description":"authentication failed","trace_id":"57bc14d9bd461f0b5a72db830149b67a"}`, + name: "401 authentication failed listing placements", + failPlacements: true, + statusCode: http.StatusUnauthorized, + body: `{"error":"access_denied","error_description":"authentication failed","trace_id":"57bc14d9bd461f0b5a72db830149b67a"}`, + wantErrSubstring: "failed to list placements", }, { - name: "403 forbidden", - statusCode: http.StatusForbidden, - body: `{"error":"forbidden","error_description":"user credentials do not have the required privileges","trace_id":"57bc14d9bd461f0b5a72db830149b67a"}`, + name: "404 not found listing placements", + failPlacements: true, + statusCode: http.StatusNotFound, + body: `{"error":"not_found","error_description":"the requested resource does not exist","trace_id":"57bc14d9bd461f0b5a72db830149b67a"}`, + wantErrSubstring: "failed to list placements", }, { - name: "404 not found", - statusCode: http.StatusNotFound, - body: `{"error":"not_found","error_description":"the requested resource does not exist","trace_id":"57bc14d9bd461f0b5a72db830149b67a"}`, + name: "403 forbidden fetching placement servers", + statusCode: http.StatusForbidden, + body: `{"error":"forbidden","error_description":"user credentials do not have the required privileges","trace_id":"57bc14d9bd461f0b5a72db830149b67a"}`, + wantErrSubstring: "failed to get placement servers", }, { - name: "500 server error", - statusCode: http.StatusInternalServerError, - body: `{"error":"server_error","error_description":"failed to token claim","trace_id":"57bc14d9bd461f0b5a72db830149b67a"}`, + name: "500 server error fetching placement servers", + statusCode: http.StatusInternalServerError, + body: `{"error":"server_error","error_description":"failed to token claim","trace_id":"57bc14d9bd461f0b5a72db830149b67a"}`, + wantErrSubstring: "failed to get placement servers", }, } @@ -261,6 +399,19 @@ func TestPlacementServersErrors(t *testing.T) { t.Run(tt.name, func(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") + + if r.URL.Path == "/api/v2/placements" { + if tt.failPlacements { + w.WriteHeader(tt.statusCode) + _, err := w.Write([]byte(tt.body)) + require.NoError(t, err) + return + } + _, err := w.Write([]byte(placementsResponse)) + require.NoError(t, err) + return + } + w.WriteHeader(tt.statusCode) _, err := w.Write([]byte(tt.body)) require.NoError(t, err) @@ -269,7 +420,11 @@ func TestPlacementServersErrors(t *testing.T) { p := newTestProvider(t, ctx, server.URL) _, err := p.Instances2NodeMap(ctx, nil) - require.ErrorContains(t, err, "failed to get placement servers") + require.ErrorContains(t, err, tt.wantErrSubstring) + + var httpErr *httperr.Error + require.ErrorAs(t, err, &httpErr) + require.Equal(t, tt.statusCode, httpErr.Code()) }) } }