actions/toolkit
2
0
Fork 0
mirror of https://github.com/actions/toolkit.git synced 2026-08-28 00:00:25 +02:00
Code Issues Packages Projects Releases Wiki Activity

Updates to @actions/artifact (#396)

Browse Source 
* Add support for 429s and Exponential backoff

* Refactor status-reporter so it can be used with download and upload

* Extra logs

* Fixes around download & gzip

* Cleanup headers and add extra tests

* Improved Docs

* Spelling bloopers

* Improved error messages

* User http client version 1.0.7
This commit is contained in:
Konrad Pabjan
2020-04-08 16:55:18 +02:00
committed by GitHub
parent 36a4b7df61
commit 1b521c4778
15 changed files with 759 additions and 296 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/artifact/src/internal/download-http-client.ts
+180 -76
View File
@@ -1,30 +1,35 @@
import * as fs from 'fs'
import * as core from '@actions/core'
import * as zlib from 'zlib'
import {
getArtifactUrl,
getRequestOptions,
getDownloadRequestOptions,
isSuccessStatusCode,
isRetryableStatusCode,
createHttpClient
isThrottledStatusCode,
getExponentialRetryTimeInMilliseconds,
tryGetRetryAfterValueTimeInMilliseconds,
displayHttpDiagnostics
} from './utils'
import {URL} from 'url'
import {StatusReporter} from './status-reporter'
import {performance} from 'perf_hooks'
import {ListArtifactsResponse, QueryArtifactResponse} from './contracts'
import {IHttpClientResponse} from '@actions/http-client/interfaces'
import {HttpManager} from './http-manager'
import {DownloadItem} from './download-specification'
import {
getDownloadFileConcurrency,
getRetryWaitTimeInMilliseconds
} from './config-variables'
import {warning} from '@actions/core'
import {getDownloadFileConcurrency, getRetryLimit} from './config-variables'
import {IncomingHttpHeaders} from 'http'
export class DownloadHttpClient {
// http manager is used for concurrent connection when downloading mulitple files at once
// http manager is used for concurrent connections when downloading multiple files at once
private downloadHttpManager: HttpManager
private statusReporter: StatusReporter
constructor() {
this.downloadHttpManager = new HttpManager(getDownloadFileConcurrency())
// downloads are usually significantly faster than uploads so display status information every second
this.statusReporter = new StatusReporter(1000)
}
/**
@@ -32,18 +37,20 @@ export class DownloadHttpClient {
*/
async listArtifacts(): Promise<ListArtifactsResponse> {
const artifactUrl = getArtifactUrl()
// use the first client from the httpManager, `keep-alive` is not used so the connection will close immediatly
const client = this.downloadHttpManager.getClient(0)
const requestOptions = getRequestOptions('application/json')
const rawResponse = await client.get(artifactUrl, requestOptions)
const body: string = await rawResponse.readBody()
if (isSuccessStatusCode(rawResponse.message.statusCode) && body) {
// use the first client from the httpManager, `keep-alive` is not used so the connection will close immediately
const client = this.downloadHttpManager.getClient(0)
const requestOptions = getDownloadRequestOptions('application/json')
const response = await client.get(artifactUrl, requestOptions)
const body: string = await response.readBody()
if (isSuccessStatusCode(response.message.statusCode) && body) {
return JSON.parse(body)
}
// eslint-disable-next-line no-console
console.log(rawResponse)
throw new Error(`Unable to list artifacts for the run`)
displayHttpDiagnostics(response)
throw new Error(
`Unable to list artifacts for the run. Resource Url ${artifactUrl}`
)
}
/**
@@ -59,18 +66,16 @@ export class DownloadHttpClient {
const resourceUrl = new URL(containerUrl)
resourceUrl.searchParams.append('itemPath', artifactName)
// no concurrent calls so a single httpClient without the http-manager is sufficient
const client = createHttpClient()
// use the first client from the httpManager, `keep-alive` is not used so the connection will close immediately
const client = this.downloadHttpManager.getClient(0)
const requestOptions = getDownloadRequestOptions('application/json')
const response = await client.get(resourceUrl.toString(), requestOptions)
const body: string = await response.readBody()
// no keep-alive header, client disposal is not necessary
const requestOptions = getRequestOptions('application/json')
const rawResponse = await client.get(resourceUrl.toString(), requestOptions)
const body: string = await rawResponse.readBody()
if (isSuccessStatusCode(rawResponse.message.statusCode) && body) {
if (isSuccessStatusCode(response.message.statusCode) && body) {
return JSON.parse(body)
}
// eslint-disable-next-line no-console
console.log(rawResponse)
displayHttpDiagnostics(response)
throw new Error(`Unable to get ContainersItems from ${resourceUrl}`)
}
@@ -81,24 +86,53 @@ export class DownloadHttpClient {
async downloadSingleArtifact(downloadItems: DownloadItem[]): Promise<void> {
const DOWNLOAD_CONCURRENCY = getDownloadFileConcurrency()
// limit the number of files downloaded at a single time
core.debug(`Download file concurrency is set to ${DOWNLOAD_CONCURRENCY}`)
const parallelDownloads = [...new Array(DOWNLOAD_CONCURRENCY).keys()]
let currentFile = 0
let downloadedFiles = 0
core.info(
`Total number of files that will be downloaded: ${downloadItems.length}`
)
this.statusReporter.setTotalNumberOfFilesToProcess(downloadItems.length)
this.statusReporter.start()
await Promise.all(
parallelDownloads.map(async index => {
while (downloadedFiles < downloadItems.length) {
const currentFileToDownload = downloadItems[downloadedFiles]
downloadedFiles += 1
while (currentFile < downloadItems.length) {
const currentFileToDownload = downloadItems[currentFile]
currentFile += 1
const startTime = performance.now()
await this.downloadIndividualFile(
index,
currentFileToDownload.sourceLocation,
currentFileToDownload.targetPath
)
if (core.isDebug()) {
core.debug(
`File: ${++downloadedFiles}/${downloadItems.length}. ${
currentFileToDownload.targetPath
} took ${(performance.now() - startTime).toFixed(
3
)} milliseconds to finish downloading`
)
}
this.statusReporter.incrementProcessedCount()
}
})
)
// done downloading, safety dispose all connections
this.downloadHttpManager.disposeAndReplaceAllClients()
.catch(error => {
throw new Error(`Unable to download the artifact: ${error}`)
})
.finally(() => {
this.statusReporter.stop()
// safety dispose all connections
this.downloadHttpManager.disposeAndReplaceAllClients()
})
}
/**
@@ -112,10 +146,20 @@ export class DownloadHttpClient {
artifactLocation: string,
downloadPath: string
): Promise<void> {
const stream = fs.createWriteStream(downloadPath)
const client = this.downloadHttpManager.getClient(httpClientIndex)
const requestOptions = getRequestOptions('application/octet-stream', true)
const response = await client.get(artifactLocation, requestOptions)
let retryCount = 0
const retryLimit = getRetryLimit()
const destinationStream = fs.createWriteStream(downloadPath)
const requestOptions = getDownloadRequestOptions(
'application/json',
true,
true
)
// a single GET request is used to download a file
const makeDownloadRequest = async (): Promise<IHttpClientResponse> => {
const client = this.downloadHttpManager.getClient(httpClientIndex)
return await client.get(artifactLocation, requestOptions)
}
// check the response headers to determine if the file was compressed using gzip
const isGzip = (headers: IncomingHttpHeaders): boolean => {
@@ -124,66 +168,126 @@ export class DownloadHttpClient {
)
}
if (isSuccessStatusCode(response.message.statusCode)) {
await this.pipeResponseToStream(
response,
stream,
isGzip(response.message.headers)
)
} else if (isRetryableStatusCode(response.message.statusCode)) {
warning(
`Received http ${response.message.statusCode} during file download, will retry ${artifactLocation} after 10 seconds`
)
// if an error is encountered, dispose of the http connection, and create a new one
this.downloadHttpManager.disposeAndReplaceClient(httpClientIndex)
await new Promise(resolve =>
setTimeout(resolve, getRetryWaitTimeInMilliseconds())
)
const retryResponse = await client.get(artifactLocation)
if (isSuccessStatusCode(retryResponse.message.statusCode)) {
await this.pipeResponseToStream(
response,
stream,
isGzip(response.message.headers)
// Increments the current retry count and then checks if the retry limit has been reached
// If there have been too many retries, fail so the download stops. If there is a retryAfterValue value provided,
// it will be used
const backOff = async (retryAfterValue?: number): Promise<void> => {
retryCount++
if (retryCount > retryLimit) {
return Promise.reject(
new Error(
`Retry limit has been reached. Unable to download ${artifactLocation}`
)
)
} else {
// eslint-disable-next-line no-console
console.log(retryResponse)
throw new Error(`Unable to download ${artifactLocation}`)
this.downloadHttpManager.disposeAndReplaceClient(httpClientIndex)
if (retryAfterValue) {
// Back off by waiting the specified time denoted by the retry-after header
core.info(
`Backoff due to too many requests, retry #${retryCount}. Waiting for ${retryAfterValue} milliseconds before continuing the download`
)
await new Promise(resolve => setTimeout(resolve, retryAfterValue))
} else {
// Back off using an exponential value that depends on the retry count
const backoffTime = getExponentialRetryTimeInMilliseconds(retryCount)
core.info(
`Exponential backoff for retry #${retryCount}. Waiting for ${backoffTime} milliseconds before continuing the download`
)
await new Promise(resolve => setTimeout(resolve, backoffTime))
}
core.info(
`Finished backoff for retry #${retryCount}, continuing with download`
)
}
}
// keep trying to download a file until a retry limit has been reached
while (retryCount <= retryLimit) {
let response: IHttpClientResponse
try {
response = await makeDownloadRequest()
} catch (error) {
// if an error is caught, it is usually indicative of a timeout so retry the download
core.info('An error occurred while attempting to download a file')
// eslint-disable-next-line no-console
console.log(error)
// increment the retryCount and use exponential backoff to wait before making the next request
await backOff()
continue
}
if (isSuccessStatusCode(response.message.statusCode)) {
// The body contains the contents of the file however calling response.readBody() causes all the content to be converted to a string
// which can cause some gzip encoded data to be lost
// Instead of using response.readBody(), response.message is a readableStream that can be directly used to get the raw body contents
return this.pipeResponseToFile(
response,
destinationStream,
isGzip(response.message.headers)
)
} else if (isRetryableStatusCode(response.message.statusCode)) {
core.info(
`A ${response.message.statusCode} response code has been received while attempting to download an artifact`
)
// if a throttled status code is received, try to get the retryAfter header value, else differ to standard exponential backoff
isThrottledStatusCode(response.message.statusCode)
? await backOff(
tryGetRetryAfterValueTimeInMilliseconds(response.message.headers)
)
: await backOff()
} else {
// Some unexpected response code, fail immediately and stop the download
displayHttpDiagnostics(response)
return Promise.reject(
new Error(
`Unexpected http ${response.message.statusCode} during download for ${artifactLocation}`
)
)
}
} else {
// eslint-disable-next-line no-console
console.log(response)
throw new Error(`Unable to download ${artifactLocation}`)
}
}
/**
* Pipes the response from downloading an individual file to the appropriate stream
* @param response the http response recieved when downloading a file
* @param stream the stream where the file should be written to
* @param isGzip does the response need to be be uncompressed
* Pipes the response from downloading an individual file to the appropriate destination stream while decoding gzip content if necessary
* @param response the http response received when downloading a file
* @param destinationStream the stream where the file should be written to
* @param isGzip a boolean denoting if the content is compressed using gzip and if we need to decode it
*/
private async pipeResponseToStream(
async pipeResponseToFile(
response: IHttpClientResponse,
stream: NodeJS.WritableStream,
destinationStream: fs.WriteStream,
isGzip: boolean
): Promise<void> {
return new Promise(resolve => {
await new Promise((resolve, reject) => {
if (isGzip) {
// pipe the response into gunzip to decompress
const gunzip = zlib.createGunzip()
response.message
.pipe(gunzip)
.pipe(stream)
.pipe(destinationStream)
.on('close', () => {
resolve()
})
.on('error', error => {
core.error(
`An error has been encountered while decompressing and writing a downloaded file to ${destinationStream.path}`
)
reject(error)
})
} else {
response.message.pipe(stream).on('close', () => {
resolve()
})
response.message
.pipe(destinationStream)
.on('close', () => {
resolve()
})
.on('error', error => {
core.error(
`An error has been encountered while writing a downloaded file to ${destinationStream.path}`
)
reject(error)
})
}
})
return
}
}