fix(security): resolve CVE alerts and fix integration tests | SUMO-285325

AppendBlobReader/README.md

@@ -31,26 +31,66 @@ Make all the code changes in `AppendBlobReader/src` directory, once all the chan
 
 This command copies required files in `AppendBlobReader/target` directory
 
-### Run Integrations Test
+### Run Integration Tests
 
-Integrations tests are in `AppendBlobReader/tests` folder and unit tests are in sumo-`function-utils/tests` folder
+Integration tests are in `AppendBlobReader/tests` folder and unit tests are in `sumo-function-utils/tests` folder.
 
-Modify the run_integration_test.sh file with below parameters
-```console
+#### Service Principal
+
+A shared service principal is available for the team via 1Password in the **"App Content team"** vault. Use the credentials from there to configure `run_integration_test.sh`.
+
+#### Permissions
+
+| Role | Scope | Purpose |
+|------|-------|---------|
+| Contributor | Subscription | Create/deploy resource groups, function apps, storage, Event Hub, Service Bus |
+| User Access Administrator | `sumo-appendblob-integration-test-do-not-delete` | Assign Storage Blob Data Reader role to function app managed identity |
+
+**How to set up permissions:**
+1. Raise a helpdesk ticket to assign **User Access Administrator** scoped to the resource group `sumo-appendblob-integration-test-do-not-delete` (one-time request).
+2. Once the SP has User Access Administrator, it can self-assign **Contributor** at subscription level:
+   ```bash
+   az role assignment create \
+     --assignee <service-principal-app-id> \
+     --role "Contributor" \
+     --scope /subscriptions/<subscription-id>
+   ```
+
+#### One-time setup (admin required)
+
+The resource group `sumo-appendblob-integration-test-do-not-delete` must exist and the SP must have `User Access Administrator` scoped to it. This resource group persists across test runs — **do not delete it**.
 
-AZURE_SUBSCRIPTION_ID=`<Your azure subscription id, to obtain it refer docs https://learn.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-subscription>`
-AZURE_CLIENT_ID=`Your application id which you can get after registering application. Refer https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app#register-an-application`
-AZURE_CLIENT_SECRET=`Generate client secret by referring docs https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app#add-credentials`
-AZURE_TENANT_ID=`You tenant id, to obtain it refer docs https://learn.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-microsoft-entra-tenant`
+```bash
+az group create -n sumo-appendblob-integration-test-do-not-delete -l centralus
+
+az role assignment create \
+  --assignee <service-principal-app-id> \
+  --role "User Access Administrator" \
+  --scope /subscriptions/<subscription-id>/resourceGroups/sumo-appendblob-integration-test-do-not-delete
+```
+
+#### Environment Variables
+
+Modify the `run_integration_test.sh` file with below parameters:
+```console
+AZURE_SUBSCRIPTION_ID=`<Your azure subscription id, refer https://learn.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-azure-subscription>`
+AZURE_CLIENT_ID=`<Your application id, refer https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app#register-an-application>`
+AZURE_CLIENT_SECRET=`<Generate client secret, refer https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app#add-credentials>`
+AZURE_TENANT_ID=`<Your tenant id, refer https://learn.microsoft.com/en-us/azure/azure-portal/get-subscription-tenant-id#find-your-microsoft-entra-tenant>`
 AZURE_DEFAULT_REGION=`eastus`
-SUMO_ACCESS_ID=`<Generate access id and access key https://help.sumologic.com/docs/manage/security/access-keys/#create-your-access-key>`
-SUMO_ACCESS_KEY=`<Generate access id and access key https://help.sumologic.com/docs/manage/security/access-keys/#create-your-access-key>`
-SUMO_DEPLOYMENT=`Enter one of the allowed values au, ca, de, eu, fed, in, jp, us1 or us2. Visit https://help.sumologic.com/APIs/General-API-Information/Sumo-Logic-Endpoints-and-Firewall-Security`
+SUMO_ACCESS_ID=`<Generate access key https://help.sumologic.com/docs/manage/security/access-keys/#create-your-access-key>`
+SUMO_ACCESS_KEY=`<Generate access key https://help.sumologic.com/docs/manage/security/access-keys/#create-your-access-key>`
+SUMO_DEPLOYMENT=`<One of: au, ca, de, eu, fed, in, jp, us1, us2. Refer https://help.sumologic.com/APIs/General-API-Information/Sumo-Logic-Endpoints-and-Firewall-Security>`
+TEST_STORAGE_RESOURCE_GROUP=`sumo-appendblob-integration-test-do-not-delete`
 ```
 
-Execute below command under `AppendBlobReader/tests` directory
+#### Running
 
-`python test_appendblobreader.py`
+```bash
+cd AppendBlobReader/tests
+source .venv/bin/activate
+source run_integration_test.sh
+```
 
 ### Run Unit Tests
 

AppendBlobReader/src/appendblobproducer.js

@@ -134,6 +134,10 @@ function queryFiles(tableQuery, context) {
             }
             return resolve(allentities);
         } catch (error) {
+            if (error.statusCode === 404 && JSON.stringify(error).includes("TableNotFound")) {
+                context.log("FileOffsetMap table does not exist yet, skipping poll.");
+                return resolve([]);
+            }
             context.log.error(`Error while fetching queryFiles: ${JSON.stringify(error)}`);
             return reject(error);
         }

AppendBlobReader/src/appendblobreaderdeploy.json

@@ -519,6 +519,10 @@
                             "name": "Project",
                             "value": "AppendBlobReader/target/producer_build/"
                         },
+                        {
+                            "name": "SCM_COMMAND_IDLE_TIMEOUT",
+                            "value": "600"
+                        },
                         {
                             "name": "WEBSITE_CONTENTSHARE",
                             "value": "[toLower(parameters('sites_BlobReader_name'))]"
@@ -618,6 +622,10 @@
                             "name": "Project",
                             "value": "AppendBlobReader/target/appendblob_producer_build/"
                         },
+                        {
+                            "name": "SCM_COMMAND_IDLE_TIMEOUT",
+                            "value": "600"
+                        },
                         {
                             "name": "WEBSITE_CONTENTSHARE",
                             "value": "[toLower(parameters('sites_appendBlobProcessor_name'))]"
@@ -725,6 +733,10 @@
                             "name": "Project",
                             "value": "AppendBlobReader/target/consumer_build/"
                         },
+                        {
+                            "name": "SCM_COMMAND_IDLE_TIMEOUT",
+                            "value": "600"
+                        },
                         {
                             "name": "WEBSITE_CONTENTSHARE",
                             "value": "[toLower(parameters('sites_blobreaderconsumer_name'))]"

AppendBlobReader/target/appendblob_producer_build/AppendBlobTaskProducer/index.js

@@ -3,7 +3,7 @@
 //////////////////////////////////////////////////////////////////////////////////////////////////////
 
 const { TableClient, TableTransaction } = require("@azure/data-tables");
 const tableClient = TableClient.fromConnectionString(process.env.AzureWebJobsStorage, process.env.TABLE_NAME);
 
 function getTask(entity) {
     return {
@@ -24,7 +24,7 @@
     // rowKey/partitionKey cannot contain "/"
     // lastEnqueLockTime - it denotes the last time when it was enqueued in Event Hub
     // done - it is set to true which means the task is enqueued in Event Hub
     var entity = {
         partitionKey: entity.partitionKey,
         rowKey: entity.rowKey,
         done: true,
@@ -52,7 +52,7 @@
     } else {
         lastEnqueLockTime = entity.lastEnqueLockTime;
     }
     var entity = {
         partitionKey: entity.partitionKey,
         rowKey: entity.rowKey,
         done: false,
@@ -82,7 +82,7 @@
 function isAppendBlobArchived(context, entity) {
 
     if (entity.blobType === "AppendBlob" && entity.offset > 0 && entity.eventdate !== undefined) {
         var maxArchivedHours = parseInt(process.env.APPSETTING_MAX_LOG_FILE_ROLLOVER_HOURS);
 
         var curDate = new Date();
         var fileCreationDate = new Date(entity.eventdate);
@@ -123,7 +123,7 @@
 function queryFiles(tableQuery, context) {
     // Todo: it queries to the whole table break this by per Partitionkey
     var allentities = [];
     return new Promise(async (resolve, reject) => {
         try {
             var entities = tableClient.listEntities({
                 queryOptions: { filter: tableQuery }
@@ -134,6 +134,10 @@
             }
             return resolve(allentities);
         } catch (error) {
+            if (error.statusCode === 404 && JSON.stringify(error).includes("TableNotFound")) {
+                context.log("FileOffsetMap table does not exist yet, skipping poll.");
+                return resolve([]);
+            }
             context.log.error(`Error while fetching queryFiles: ${JSON.stringify(error)}`);
             return reject(error);
         }
@@ -158,8 +162,8 @@
     Object.keys(groupedEntities).forEach(function (groupKey) {
         var entities = groupedEntities[groupKey];
         for (let batchIndex = 0; batchIndex < entities.length; batchIndex += maxBatchItems) {
             (function (batchIndex, groupKey) {
                 batch_promises.push(new Promise(async function (resolve, reject) {
                     var currentBatch = entities.slice(batchIndex, batchIndex + maxBatchItems);
                     var transaction = new TableTransaction();
                     for (let index = 0; index < currentBatch.length; index++) {
@@ -246,7 +250,7 @@
     // based on experiments it takes ~3min to process 8000 tasks with file size of ~5MB
     let maxFileTaskPerInvokePerStorageAccount = 8000;
     var filteredEntities = [];
     let entity = null;
     for (let idx = 0; idx < entities.length; idx += 1) {
         entity = entities[idx];