feature/docgen-enhacements-part-2

.github/workflows/deploy-ci-backend.yml

@@ -145,12 +145,16 @@ jobs:
           echo "🔐 Populating CI Key Vault secrets..."
 
           KEY_VAULT_NAME="docgen-ci-kv"
+          SECRET_DIR="${RUNNER_TEMP}/docgen-ci-secrets"
+          mkdir -p "${SECRET_DIR}"
 
           # Set SF private key (for Node -> Salesforce JWT Bearer flow)
-          echo "${{ secrets.CI_SF_PRIVATE_KEY }}" | az keyvault secret set \
+          printf '%s' "${{ secrets.CI_SF_PRIVATE_KEY }}" > "${SECRET_DIR}/sf-private-key.pem"
+          az keyvault secret set \
             --vault-name ${KEY_VAULT_NAME} \
             --name SF-PRIVATE-KEY \
-            --value @/dev/stdin \
+            --file "${SECRET_DIR}/sf-private-key.pem" \
+            --encoding utf-8 \
             --output none
 
           # Set SF client ID (Connected App)
@@ -177,10 +181,19 @@ jobs:
           # Set SFDX Auth URL if provided (for scratch org authentication)
           if [ -n "${{ inputs.sfdx_auth_url }}" ]; then
             echo "Setting SFDX-AUTH-URL from workflow input..."
-            echo "${{ inputs.sfdx_auth_url }}" | az keyvault secret set \
+            SFDX_AUTH_URL="${{ inputs.sfdx_auth_url }}"
+
+            if [[ "$SFDX_AUTH_URL" != force://* ]]; then
+              echo "❌ Provided SFDX Auth URL does not use the expected force:// scheme"
+              exit 1
+            fi
+
+            printf '%s' "$SFDX_AUTH_URL" > "${SECRET_DIR}/sfdx-auth-url.txt"
+            az keyvault secret set \
               --vault-name ${KEY_VAULT_NAME} \
               --name SFDX-AUTH-URL \
-              --value @/dev/stdin \
+              --file "${SECRET_DIR}/sfdx-auth-url.txt" \
+              --encoding utf-8 \
               --output none
             echo "✅ SFDX-AUTH-URL secret set"
           else
@@ -199,6 +212,7 @@ jobs:
             --output none
 
           echo "✅ CI Key Vault secrets populated"
+          rm -f "${SECRET_DIR}/sf-private-key.pem" "${SECRET_DIR}/sfdx-auth-url.txt"
 
       - name: Restart Container App (if SFDX Auth URL provided)
         if: inputs.sfdx_auth_url != ''

.github/workflows/e2e-tests.yml

@@ -192,12 +192,16 @@ jobs:
           echo "🔐 Populating CI Key Vault secrets..."
 
           KEY_VAULT_NAME="docgen-ci-kv"
+          SECRET_DIR="${RUNNER_TEMP}/docgen-ci-secrets"
+          mkdir -p "${SECRET_DIR}"
 
           # Set SF private key (for Node -> Salesforce JWT Bearer flow)
-          echo "${{ secrets.CI_SF_PRIVATE_KEY }}" | az keyvault secret set \
+          printf '%s' "${{ secrets.CI_SF_PRIVATE_KEY }}" > "${SECRET_DIR}/sf-private-key.pem"
+          az keyvault secret set \
             --vault-name ${KEY_VAULT_NAME} \
             --name SF-PRIVATE-KEY \
-            --value @/dev/stdin \
+            --file "${SECRET_DIR}/sf-private-key.pem" \
+            --encoding utf-8 \
             --output none
 
           # Set SF client ID (Connected App)
@@ -221,18 +225,34 @@ jobs:
             --value "test.salesforce.com" \
             --output none
 
-          # Set SFDX Auth URL (from earlier scratch org creation)
-          SFDX_AUTH_URL=$(sf org display --verbose --json | jq -r '.result.sfdxAuthUrl')
+          # Set direct scratch org access token auth for the Node backend.
+          sf org display \
+            --verbose \
+            --json \
+            --target-org "${{ steps.create-org.outputs.org_alias }}" \
+            > "${SECRET_DIR}/scratch-org-display.json"
+
+          SF_ACCESS_TOKEN=$(jq -r '.result.accessToken // empty' "${SECRET_DIR}/scratch-org-display.json")
+          SF_INSTANCE_URL=$(jq -r '.result.instanceUrl // empty' "${SECRET_DIR}/scratch-org-display.json")
 
-          if [ -z "$SFDX_AUTH_URL" ] || [ "$SFDX_AUTH_URL" = "null" ]; then
-            echo "❌ Failed to extract SFDX Auth URL from org"
+          if [ -z "$SF_ACCESS_TOKEN" ] || [ -z "$SF_INSTANCE_URL" ]; then
+            echo "❌ Failed to extract scratch org access token or instance URL"
+            jq '{status, resultKeys: (.result | keys)}' "${SECRET_DIR}/scratch-org-display.json"
             exit 1
           fi
 
-          echo "$SFDX_AUTH_URL" | az keyvault secret set \
+          printf '%s' "$SF_ACCESS_TOKEN" > "${SECRET_DIR}/sf-access-token.txt"
+          az keyvault secret set \
+            --vault-name ${KEY_VAULT_NAME} \
+            --name SF-ACCESS-TOKEN \
+            --file "${SECRET_DIR}/sf-access-token.txt" \
+            --encoding utf-8 \
+            --output none
+
+          az keyvault secret set \
             --vault-name ${KEY_VAULT_NAME} \
-            --name SFDX-AUTH-URL \
-            --value @/dev/stdin \
+            --name SF-INSTANCE-URL \
+            --value "$SF_INSTANCE_URL" \
             --output none
 
           # Set Azure Monitor connection string
@@ -246,7 +266,8 @@ jobs:
             --value "${APP_INSIGHTS_CONN}" \
             --output none
 
-          echo "✅ CI Key Vault secrets populated (including scratch org SFDX Auth URL)"
+          echo "✅ CI Key Vault secrets populated (including scratch org access token auth)"
+          rm -f "${SECRET_DIR}/sf-private-key.pem" "${SECRET_DIR}/sf-access-token.txt" "${SECRET_DIR}/scratch-org-display.json"
 
       - name: Restart Container App to pick up all secrets
         run: |
@@ -271,16 +292,40 @@ jobs:
         id: org-info
         run: |
           # Get org info (org is already authenticated and set as default)
-          sf org display --json > org-info.json
+          sf org display \
+            --verbose \
+            --json \
+            --target-org "${{ steps.create-org.outputs.org_alias }}" \
+            > org-info.json
 
           # Extract credentials and set as environment variables
           INSTANCE_URL=$(jq -r '.result.instanceUrl' org-info.json)
           ACCESS_TOKEN=$(jq -r '.result.accessToken' org-info.json)
           USERNAME=$(jq -r '.result.username' org-info.json)
+          ORG_ID=$(jq -r '.result.id // empty' org-info.json)
+
+          if [ -z "$INSTANCE_URL" ] || [ "$INSTANCE_URL" = "null" ] || [ "$INSTANCE_URL" = "undefined" ]; then
+            echo "❌ Failed to extract scratch org instance URL"
+            jq '{status, resultKeys: (.result | keys)}' org-info.json
+            exit 1
+          fi
+
+          if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ] || [ "$ACCESS_TOKEN" = "undefined" ]; then
+            echo "❌ Failed to extract scratch org access token"
+            jq '{status, resultKeys: (.result | keys)}' org-info.json
+            exit 1
+          fi
+
+          if [ -z "$USERNAME" ] || [ "$USERNAME" = "null" ] || [ "$USERNAME" = "undefined" ]; then
+            echo "❌ Failed to extract scratch org username"
+            jq '{status, resultKeys: (.result | keys)}' org-info.json
+            exit 1
+          fi
 
           echo "SF_INSTANCE_URL=$INSTANCE_URL" >> $GITHUB_ENV
           echo "SF_ACCESS_TOKEN=$ACCESS_TOKEN" >> $GITHUB_ENV
           echo "SF_USERNAME=$USERNAME" >> $GITHUB_ENV
+          echo "SF_ORG_ID=$ORG_ID" >> $GITHUB_ENV
 
           echo "Instance URL: $INSTANCE_URL"
           echo "Username: $USERNAME"

e2e/utils/scratch-org.ts

@@ -1,4 +1,4 @@
-import { exec } from 'child_process';
+import { exec, execFile } from 'child_process';
 import { promisify } from 'util';
 
 const execAsync = promisify(exec);
@@ -10,18 +10,92 @@ export interface ScratchOrgInfo {
   orgId: string;
 }
 
+function isMissingOrgValue(value: unknown): boolean {
+  if (typeof value !== 'string') {
+    return true;
+  }
+  const normalized = value.trim().toLowerCase();
+  return normalized === '' || normalized === 'undefined' || normalized === 'null';
+}
+
+function shellArg(value: string): string {
+  return `'${value.replace(/'/g, "'\\''")}'`;
+}
+
+function getScratchOrgInfoFromEnv(): ScratchOrgInfo | null {
+  const instanceUrl = process.env.SF_INSTANCE_URL;
+  const accessToken = process.env.SF_ACCESS_TOKEN;
+  const username = process.env.SF_USERNAME;
+
+  if (
+    isMissingOrgValue(instanceUrl) ||
+    isMissingOrgValue(accessToken) ||
+    isMissingOrgValue(username)
+  ) {
+    return null;
+  }
+
+  return {
+    instanceUrl: instanceUrl!.replace(/\/$/, ''),
+    accessToken: accessToken!,
+    username: username!,
+    orgId: process.env.SF_ORG_ID || '',
+  };
+}
+
+interface RestCreateResponse {
+  success?: boolean;
+  id?: string;
+  errors?: unknown[];
+}
+
+function execFileCommand(
+  command: string,
+  args: string[],
+  options: Parameters<typeof execFile>[2]
+): Promise<{ stdout: string; stderr: string }> {
+  return new Promise((resolve, reject) => {
+    execFile(command, args, options, (error, stdout, stderr) => {
+      if (error) {
+        reject(Object.assign(error, { stdout, stderr }));
+        return;
+      }
+
+      resolve({
+        stdout: stdout.toString(),
+        stderr: stderr.toString(),
+      });
+    });
+  });
+}
+
+function getSfCliEnv(): NodeJS.ProcessEnv {
+  const env = { ...process.env };
+  delete env.SF_ACCESS_TOKEN;
+  delete env.SF_INSTANCE_URL;
+  delete env.SF_ORG_ID;
+  env.SF_FORMAT_JSON = 'true';
+  env.SF_DISABLE_COLORS = 'true';
+  return env;
+}
+
 /**
  * Get information about the current default scratch org
  * Uses `sf org display --json` to fetch credentials
  */
 export async function getScratchOrgInfo(): Promise<ScratchOrgInfo> {
   try {
+    const envOrgInfo = getScratchOrgInfoFromEnv();
+    if (envOrgInfo) {
+      return envOrgInfo;
+    }
+
     // Get org info from SFDX CLI
     // Use SF_FORMAT_JSON=true to ensure clean JSON output without colors
     // In CI, use SF_USERNAME env var if available to explicitly target the org
     const targetOrg = process.env.SF_USERNAME;
-    const targetOrgFlag = targetOrg ? ` --target-org ${targetOrg}` : '';
-    const { stdout, stderr } = await execAsync(`sf org display${targetOrgFlag} --json`, {
+    const targetOrgFlag = targetOrg ? ` --target-org ${shellArg(targetOrg)}` : '';
+    const { stdout, stderr } = await execAsync(`sf org display${targetOrgFlag} --verbose --json`, {
       env: { ...process.env, SF_FORMAT_JSON: 'true', SF_DISABLE_COLORS: 'true' }
     });
 
@@ -41,17 +115,21 @@ export async function getScratchOrgInfo(): Promise<ScratchOrgInfo> {
     const result = orgData.result;
 
     // Check if we have the required fields
-    if (!result.instanceUrl || !result.accessToken || !result.username) {
+    if (
+      isMissingOrgValue(result.instanceUrl) ||
+      isMissingOrgValue(result.accessToken) ||
+      isMissingOrgValue(result.username)
+    ) {
       throw new Error(
-        'Missing required org information. Make sure a scratch org is set as default.'
+        `Missing required org information. Result keys: ${Object.keys(result).join(', ')}`
       );
     }
 
     return {
-      instanceUrl: result.instanceUrl,
+      instanceUrl: result.instanceUrl.replace(/\/$/, ''),
       accessToken: result.accessToken,
       username: result.username,
-      orgId: result.id,
+      orgId: result.id || '',
     };
   } catch (error) {
     if (error instanceof Error) {
@@ -122,44 +200,60 @@ async function createRecordViaRestAPI(
   objectType: string,
   fields: Record<string, any>
 ): Promise<string> {
-  const orgInfo = await getScratchOrgInfo();
-
-  // Write JSON to temp file
+  // Write JSON to temp file. Use the Salesforce CLI REST command so auth stays
+  // inside the CLI org connection instead of manually passing a bearer token.
   const fs = require('fs');
-  const tempFile = `/tmp/sf-record-rest-${Date.now()}.json`;
+  const tempFile = `/tmp/sf-record-rest-${Date.now()}-${Math.random().toString(36).substring(7)}.json`;
   fs.writeFileSync(tempFile, JSON.stringify(fields));
 
   try {
-    const url = `${orgInfo.instanceUrl}/services/data/v65.0/sobjects/${objectType}`;
-
-    const command = `curl -X POST "${url}" \\
-      -H "Authorization: Bearer ${orgInfo.accessToken}" \\
-      -H "Content-Type: application/json" \\
-      -d @${tempFile}`;
+    const targetOrg = process.env.SF_USERNAME;
+    const args = [
+      'api',
+      'request',
+      'rest',
+      `/services/data/v65.0/sobjects/${objectType}`,
+      '--body',
+      `@${tempFile}`,
+      '--method',
+      'POST',
+    ];
+
+    if (targetOrg) {
+      args.push('--target-org', targetOrg);
+    }
 
-    const { stdout } = await execAsync(command, {
-      maxBuffer: 10 * 1024 * 1024 // 10MB buffer for large responses
+    const { stdout, stderr } = await execFileCommand('sf', args, {
+      env: getSfCliEnv(),
+      maxBuffer: 10 * 1024 * 1024,
     });
 
-    const result = JSON.parse(stdout);
-
-    // Clean up temp file
-    if (fs.existsSync(tempFile)) {
-      fs.unlinkSync(tempFile);
+    if (stderr) {
+      console.error('SF CLI stderr (REST):', stderr);
     }
 
+    const cleanStdout = stdout.replace(/\x1B\[[0-9;]*[mGKHF]/g, '').trim();
+    const result = JSON.parse(cleanStdout) as RestCreateResponse;
+
     if (!result.success || !result.id) {
       throw new Error(`REST API creation failed: ${JSON.stringify(result)}`);
     }
 
     return result.id;
   } catch (error) {
-    // Clean up temp file on error
-    const fs = require('fs');
+    const stdout = error instanceof Error ? (error as any).stdout || '' : '';
+    const stderr = error instanceof Error ? (error as any).stderr || '' : '';
+    const stderrInfo = stderr ? `\nStderr: ${stderr}` : '';
+    const stdoutInfo = stdout ? `\nStdout: ${stdout}` : '';
+
+    if (error instanceof Error) {
+      throw new Error(`REST API creation failed: ${error.message}${stdoutInfo}${stderrInfo}`);
+    }
+    throw error;
+  } finally {
     if (fs.existsSync(tempFile)) {
       fs.unlinkSync(tempFile);
     }
-    throw error;
   }
 }