Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 36 additions & 26 deletions gatsby-node.js
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,11 @@ const {
generateColorsScssFile
} = require("./src/utils/scssUtils");

const { FIFTY_PER_PAGE } = require("./src/utils/build-json/constants");
const { FIFTY_PER_PAGE, BUILD_REQUEST_TIMEOUT_MS, BUILD_PAGE_FETCH_CONCURRENCY } = require("./src/utils/build-json/constants");
const SpeakersAPIRequest = require("./src/utils/build-json/SpeakersAPIRequest");
const getWithRetry = require("./src/utils/build-json/getWithRetry");

axios.defaults.timeout = BUILD_REQUEST_TIMEOUT_MS;

const fileBuildTimes = [];

Expand All @@ -52,30 +55,37 @@ const getAccessToken = async (config, scope) => {
try {
return await client.getToken({ scope });
} catch (error) {
console.log("Access Token error", error);
throw new Error(`Failed to obtain build access token: ${error?.message || error}`, { cause: error });
}
};

const SSR_GetRemainingPages = async (endpoint, params, lastPage) => {
// create an array with remaining pages to perform Promise.All
const pages = [];
for (let i = 2; i <= lastPage; i++) {
pages.push(i);
}

let remainingPages = await Promise.all(pages.map(pageIdx => {
return axios.get(endpoint,
{
params: {
...params,
page: pageIdx
}
}).then(({ data }) => data);
}));
const fetchPage = (pageIdx) =>
getWithRetry(endpoint, { params: { ...params, page: pageIdx } }).then(({ data }) => data);

const remainingPages = [];
for (let i = 0; i < pages.length; i += BUILD_PAGE_FETCH_CONCURRENCY) {
const chunk = pages.slice(i, i + BUILD_PAGE_FETCH_CONCURRENCY);
const chunkResults = await Promise.all(chunk.map(fetchPage));
remainingPages.push(...chunkResults);
}

return remainingPages.sort((a, b,) => a.current_page - b.current_page).map(p => p.data).flat();
}

const SSR_handleError = (e) => {
const status = e?.response?.status;
const statusText = e?.response?.statusText;
const url = e?.config?.url;
const detail = status ? `HTTP ${status} ${statusText || ""}`.trim() : (e?.message || String(e));
throw new Error(`Build API request failed: ${detail}${url ? ` (url: ${url})` : ""}`, { cause: e });
};

const SSR_getMarketingSettings = async (baseUrl, summitId) => {

const endpoint = `${baseUrl}/api/public/v1/config-values/all/shows/${summitId}`;
Expand All @@ -85,15 +95,15 @@ const SSR_getMarketingSettings = async (baseUrl, summitId) => {
page: 1
};

return await axios.get(endpoint, { params }).then(async ({ data }) => {
return await getWithRetry(endpoint, { params }).then(async ({ data }) => {

console.log(`SSR_getMarketingSettings then data.current_page ${data.current_page} data.last_page ${data.last_page} total ${data.total}`)

let remainingPages = await SSR_GetRemainingPages(endpoint, params, data.last_page);

return [...data.data, ...remainingPages];

}).catch(e => console.log("ERROR: ", e));
}).catch(SSR_handleError);
};

const SSR_getEvents = async (baseUrl, summitId, accessToken) => {
Expand All @@ -108,15 +118,15 @@ const SSR_getEvents = async (baseUrl, summitId, accessToken) => {

const params = EventAPIRequest.getParams(apiUrl);

return await axios.get(apiUrlWithParams).then(async ({ data }) => {
return await getWithRetry(apiUrlWithParams).then(async ({ data }) => {

console.log(`SSR_getEvents then data.current_page ${data.current_page} data.last_page ${data.last_page} total ${data.total}`)

let remainingPages = await SSR_GetRemainingPages(apiUrlWithParams, params, data.last_page);

return [...data.data, ...remainingPages];

}).catch(e => console.log("ERROR: ", e));
}).catch(SSR_handleError);
};

const SSR_getSponsors = async (baseUrl, summitId, accessToken) => {
Expand All @@ -131,15 +141,15 @@ const SSR_getSponsors = async (baseUrl, summitId, accessToken) => {
expand: 'company,sponsorship,sponsorship.type',
}

return await axios.get(endpoint, { params }).then(async ({ data }) => {
return await getWithRetry(endpoint, { params }).then(async ({ data }) => {

console.log(`SSR_getSponsors then data.current_page ${data.current_page} data.last_page ${data.last_page} total ${data.total}`)

let remainingPages = await SSR_GetRemainingPages(endpoint, params, data.last_page);

return [...data.data, ...remainingPages];

}).catch(e => console.log('ERROR: ', e));
}).catch(SSR_handleError);
};

const SSR_getSponsorCollections = async (allSponsors, baseUrl, summitId, accessToken) => {
Expand All @@ -150,11 +160,11 @@ const SSR_getSponsorCollections = async (allSponsors, baseUrl, summitId, accessT
page: 1,
}

const getSponsorCollection = async (endpoint, params) => await axios.get(endpoint, { params }).then(async ({ data }) => {
const getSponsorCollection = async (endpoint, params) => await getWithRetry(endpoint, { params }).then(async ({ data }) => {
console.log(`SSR_getSponsorCollection then data.current_page ${data.current_page} data.last_page ${data.last_page} total ${data.total}`)
let remainingPages = await SSR_GetRemainingPages(endpoint, params, data.last_page);
return [...data.data, ...remainingPages];
}).catch(e => console.log('ERROR: ', e));
}).catch(SSR_handleError);

const sponsorsWithCollections = await Promise.all(allSponsors.map(async (sponsor) => {
console.log(`Collections for ${sponsor.company.name}...`);
Expand All @@ -179,15 +189,15 @@ const SSR_getSpeakers = async (baseUrl, summitId, accessToken, filter = null) =>

const params = SpeakersAPIRequest.getParams(apiUrl);

return await axios.get(apiUrlWithParams)
return await getWithRetry(apiUrlWithParams)
.then(async ({ data }) => {
console.log(`SSR_getSpeakers then data.current_page ${data.current_page} data.last_page ${data.last_page} total ${data.total}`)

let remainingPages = await SSR_GetRemainingPages(apiUrlWithParams, params, data.last_page);

return [...data.data, ...remainingPages];
})
.catch(e => console.log("ERROR: ", e));
.catch(SSR_handleError);
};

const SSR_getSummit = async (baseUrl, summitId, accessToken) => {
Expand All @@ -199,11 +209,11 @@ const SSR_getSummit = async (baseUrl, summitId, accessToken) => {

const apiUrlWithParams = SummitAPIRequest.build(apiUrl);

return await axios.get(
return await getWithRetry(
apiUrlWithParams
)
.then(({ data }) => data)
.catch(e => console.log("ERROR: ", e));
.catch(SSR_handleError);
};

const SSR_getVoteablePresentations = async (baseUrl, summitId, accessToken) => {
Expand All @@ -218,7 +228,7 @@ const SSR_getVoteablePresentations = async (baseUrl, summitId, accessToken) => {
expand: "slides,links,videos,media_uploads,type,track,track.allowed_access_levels,location,location.venue,location.floor,speakers,moderator,sponsors,current_attendance,groups,rsvp_template,tags",
};

return await axios.get(endpoint,
return await getWithRetry(endpoint,
{ params }).then(async ({ data }) => {

console.log(`SSR_getVoteablePresentations then data.current_page ${data.current_page} data.last_page ${data.last_page} total ${data.total}`)
Expand All @@ -227,7 +237,7 @@ const SSR_getVoteablePresentations = async (baseUrl, summitId, accessToken) => {

return [...data.data, ...remainingPages];
})
.catch(e => console.log("ERROR: ", e));
.catch(SSR_handleError);
};

exports.onPreBootstrap = async () => {
Expand Down
11 changes: 10 additions & 1 deletion src/utils/build-json/constants.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,14 @@
const FIFTY_PER_PAGE = "50";

const BUILD_REQUEST_TIMEOUT_MS = 30000;
const BUILD_REQUEST_MAX_RETRIES = 2;
const BUILD_REQUEST_RETRY_BASE_BACKOFF_MS = 500;
const BUILD_PAGE_FETCH_CONCURRENCY = 5;

module.exports = {
FIFTY_PER_PAGE
FIFTY_PER_PAGE,
BUILD_REQUEST_TIMEOUT_MS,
BUILD_REQUEST_MAX_RETRIES,
BUILD_REQUEST_RETRY_BASE_BACKOFF_MS,
BUILD_PAGE_FETCH_CONCURRENCY
};
32 changes: 32 additions & 0 deletions src/utils/build-json/getWithRetry.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
const axios = require("axios");
const {
BUILD_REQUEST_MAX_RETRIES,
BUILD_REQUEST_RETRY_BASE_BACKOFF_MS
} = require("./constants");

const RETRIABLE_STATUSES = new Set([502, 503, 504]);

const isRetriable = (error) => {
if (!error?.response) return true;
return RETRIABLE_STATUSES.has(error.response.status);
};

const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

const getWithRetry = async (
url,
options = {},
retriesLeft = BUILD_REQUEST_MAX_RETRIES,
backoffMs = BUILD_REQUEST_RETRY_BASE_BACKOFF_MS
) => {
try {
return await axios.get(url, options);
} catch (error) {
if (retriesLeft <= 0 || !isRetriable(error)) throw error;
console.log(`getWithRetry: ${error?.response?.status || error?.code || error?.message} on ${url}, retrying in ${backoffMs}ms (${retriesLeft} left)`);
await delay(backoffMs);
return getWithRetry(url, options, retriesLeft - 1, backoffMs * 2);
}
};

module.exports = getWithRetry;
Loading