Artifact upload: support uploading single un-zipped files (#2256)

* Artifact upload: support uploading single un-zipped files

* Fix linters

* Fix lint again

* Fix tests

* Check for 0 sized artifact lists

* Add some more stream tests and handle an upload failure gracefully

* Add CI tests for non-zipped artifacts

* Add an html report to test rendering in the browser

* Fix linting issue

* Artifact: bump the version and add release notes

* Fix Windows tests

* Fix linting

* stream: switch the error details to error type

* Refactor the validation logic in `uploadArtifact` a bit

* Added more details about how the name parameter is handled

packages/artifact/src/internal/shared/interfaces.ts

@@ -50,6 +50,13 @@ export interface UploadArtifactOptions {
    * For large files that are not easily compressed, a value of 0 is recommended for significantly faster uploads.
    */
   compressionLevel?: number
+  /**
+   * If true, the artifact will be uploaded without being archived (zipped).
+   * This is only supported when uploading a single file.
+   * When using this option, the artifact will not be compressed.
+   * When using this option, the name parameter passed to the upload is ignored. Instead, the name of the file is used as the name of the artifact.
+   */
+  skipArchive?: boolean
 }
 
 /**

packages/artifact/src/internal/upload/blob-upload.ts

@@ -1,6 +1,6 @@
 import {BlobClient, BlockBlobUploadStreamOptions} from '@azure/storage-blob'
 import {TransferProgressEvent} from '@azure/core-http-compat'
-import {ZipUploadStream} from './zip.js'
+import {WaterMarkedUploadStream} from './stream.js'
 import {
   getUploadChunkSize,
   getConcurrency,
@@ -23,9 +23,10 @@ export interface BlobUploadResponse {
   sha256Hash?: string
 }
 
-export async function uploadZipToBlobStorage(
+export async function uploadToBlobStorage(
   authenticatedUploadURL: string,
-  zipUploadStream: ZipUploadStream
+  uploadStream: WaterMarkedUploadStream,
+  contentType: string
 ): Promise<BlobUploadResponse> {
   let uploadByteCount = 0
   let lastProgressTime = Date.now()
@@ -51,7 +52,7 @@ export async function uploadZipToBlobStorage(
   const blockBlobClient = blobClient.getBlockBlobClient()
 
   core.debug(
-    `Uploading artifact zip to blob storage with maxConcurrency: ${maxConcurrency}, bufferSize: ${bufferSize}`
+    `Uploading artifact to blob storage with maxConcurrency: ${maxConcurrency}, bufferSize: ${bufferSize}, contentType: ${contentType}`
   )
 
   const uploadCallback = (progress: TransferProgressEvent): void => {
@@ -61,24 +62,24 @@ export async function uploadZipToBlobStorage(
   }
 
   const options: BlockBlobUploadStreamOptions = {
-    blobHTTPHeaders: {blobContentType: 'zip'},
+    blobHTTPHeaders: {blobContentType: contentType},
     onProgress: uploadCallback,
     abortSignal: abortController.signal
   }
 
   let sha256Hash: string | undefined = undefined
-  const uploadStream = new stream.PassThrough()
+  const blobUploadStream = new stream.PassThrough()
   const hashStream = crypto.createHash('sha256')
 
-  zipUploadStream.pipe(uploadStream) // This stream is used for the upload
-  zipUploadStream.pipe(hashStream).setEncoding('hex') // This stream is used to compute a hash of the zip content that gets used. Integrity check
+  uploadStream.pipe(blobUploadStream) // This stream is used for the upload
+  uploadStream.pipe(hashStream).setEncoding('hex') // This stream is used to compute a hash of the content for integrity check
 
   core.info('Beginning upload of artifact content to blob storage')
 
   try {
     await Promise.race([
       blockBlobClient.uploadStream(
-        uploadStream,
+        blobUploadStream,
         bufferSize,
         maxConcurrency,
         options
@@ -98,7 +99,7 @@ export async function uploadZipToBlobStorage(
 
   hashStream.end()
   sha256Hash = hashStream.read() as string
-  core.info(`SHA256 digest of uploaded artifact zip is ${sha256Hash}`)
+  core.info(`SHA256 digest of uploaded artifact is ${sha256Hash}`)
 
   if (uploadByteCount === 0) {
     core.warning(

packages/artifact/src/internal/upload/stream.ts

@@ -0,0 +1,53 @@
+import * as stream from 'stream'
+import * as fs from 'fs'
+import {realpath} from 'fs/promises'
+import * as core from '@actions/core'
+import {getUploadChunkSize} from '../shared/config.js'
+
+// Custom stream transformer so we can set the highWaterMark property
+// See https://github.com/nodejs/node/issues/8855
+export class WaterMarkedUploadStream extends stream.Transform {
+  constructor(bufferSize: number) {
+    super({
+      highWaterMark: bufferSize
+    })
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  _transform(chunk: any, enc: any, cb: any): void {
+    cb(null, chunk)
+  }
+}
+
+export async function createRawFileUploadStream(
+  filePath: string
+): Promise<WaterMarkedUploadStream> {
+  core.debug(`Creating raw file upload stream for: ${filePath}`)
+
+  const bufferSize = getUploadChunkSize()
+  const uploadStream = new WaterMarkedUploadStream(bufferSize)
+
+  // Check if symlink and resolve the source path
+  let sourcePath = filePath
+  const stats = await fs.promises.lstat(filePath)
+  if (stats.isSymbolicLink()) {
+    sourcePath = await realpath(filePath)
+  }
+
+  // Create a read stream from the file and pipe it to the upload stream
+  const fileStream = fs.createReadStream(sourcePath, {
+    highWaterMark: bufferSize
+  })
+
+  fileStream.on('error', error => {
+    core.error('An error has occurred while reading the file for upload')
+    core.error(String(error))
+    uploadStream.destroy(
+      new Error('An error has occurred during file read for the artifact')
+    )
+  })
+
+  fileStream.pipe(uploadStream)
+
+  return uploadStream
+}

packages/artifact/src/internal/upload/types.ts

@@ -0,0 +1,82 @@
+import * as path from 'path'
+
+/**
+ * Maps file extensions to MIME types
+ */
+const mimeTypes: Record<string, string> = {
+  // Text
+  '.txt': 'text/plain',
+  '.html': 'text/html',
+  '.htm': 'text/html',
+  '.css': 'text/css',
+  '.csv': 'text/csv',
+  '.xml': 'text/xml',
+  '.md': 'text/markdown',
+
+  // JavaScript/JSON
+  '.js': 'application/javascript',
+  '.mjs': 'application/javascript',
+  '.json': 'application/json',
+
+  // Images
+  '.png': 'image/png',
+  '.jpg': 'image/jpeg',
+  '.jpeg': 'image/jpeg',
+  '.gif': 'image/gif',
+  '.svg': 'image/svg+xml',
+  '.webp': 'image/webp',
+  '.ico': 'image/x-icon',
+  '.bmp': 'image/bmp',
+  '.tiff': 'image/tiff',
+  '.tif': 'image/tiff',
+
+  // Audio
+  '.mp3': 'audio/mpeg',
+  '.wav': 'audio/wav',
+  '.ogg': 'audio/ogg',
+  '.flac': 'audio/flac',
+
+  // Video
+  '.mp4': 'video/mp4',
+  '.webm': 'video/webm',
+  '.avi': 'video/x-msvideo',
+  '.mov': 'video/quicktime',
+
+  // Documents
+  '.pdf': 'application/pdf',
+  '.doc': 'application/msword',
+  '.docx':
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+  '.xls': 'application/vnd.ms-excel',
+  '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+  '.ppt': 'application/vnd.ms-powerpoint',
+  '.pptx':
+    'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+
+  // Archives
+  '.zip': 'application/zip',
+  '.tar': 'application/x-tar',
+  '.gz': 'application/gzip',
+  '.rar': 'application/vnd.rar',
+  '.7z': 'application/x-7z-compressed',
+
+  // Code/Data
+  '.wasm': 'application/wasm',
+  '.yaml': 'application/x-yaml',
+  '.yml': 'application/x-yaml',
+
+  // Fonts
+  '.woff': 'font/woff',
+  '.woff2': 'font/woff2',
+  '.ttf': 'font/ttf',
+  '.otf': 'font/otf',
+  '.eot': 'application/vnd.ms-fontobject'
+}
+
+/**
+ * Gets the MIME type for a file based on its extension
+ */
+export function getMimeType(filePath: string): string {
+  const ext = path.extname(filePath).toLowerCase()
+  return mimeTypes[ext] || 'application/octet-stream'
+}

packages/artifact/src/internal/upload/upload-artifact.ts

@@ -1,4 +1,6 @@
 import * as core from '@actions/core'
+import * as fs from 'fs'
+import * as path from 'path'
 import {
   UploadArtifactOptions,
   UploadArtifactResponse
@@ -12,14 +14,16 @@ import {
   validateRootDirectory
 } from './upload-zip-specification.js'
 import {getBackendIdsFromToken} from '../shared/util.js'
-import {uploadZipToBlobStorage} from './blob-upload.js'
+import {uploadToBlobStorage} from './blob-upload.js'
 import {createZipUploadStream} from './zip.js'
+import {createRawFileUploadStream, WaterMarkedUploadStream} from './stream.js'
 import {
   CreateArtifactRequest,
   FinalizeArtifactRequest,
   StringValue
 } from '../../generated/index.js'
 import {FilesNotFoundError, InvalidResponseError} from '../shared/errors.js'
+import {getMimeType} from './types.js'
 
 export async function uploadArtifact(
   name: string,
@@ -27,19 +31,43 @@ export async function uploadArtifact(
   rootDirectory: string,
   options?: UploadArtifactOptions | undefined
 ): Promise<UploadArtifactResponse> {
+  let artifactFileName = `${name}.zip`
+  if (options?.skipArchive) {
+    if (files.length === 0) {
+      throw new FilesNotFoundError([])
+    }
+
+    if (files.length > 1) {
+      throw new Error(
+        'skipArchive option is only supported when uploading a single file'
+      )
+    }
+
+    if (!fs.existsSync(files[0])) {
+      throw new FilesNotFoundError(files)
+    }
+
+    artifactFileName = path.basename(files[0])
+    name = artifactFileName
+  }
+
   validateArtifactName(name)
   validateRootDirectory(rootDirectory)
 
-  const zipSpecification: UploadZipSpecification[] = getUploadZipSpecification(
-    files,
-    rootDirectory
-  )
-  if (zipSpecification.length === 0) {
-    throw new FilesNotFoundError(
-      zipSpecification.flatMap(s => (s.sourcePath ? [s.sourcePath] : []))
-    )
+  let zipSpecification: UploadZipSpecification[] = []
+
+  if (!options?.skipArchive) {
+    zipSpecification = getUploadZipSpecification(files, rootDirectory)
+
+    if (zipSpecification.length === 0) {
+      throw new FilesNotFoundError(
+        zipSpecification.flatMap(s => (s.sourcePath ? [s.sourcePath] : []))
+      )
+    }
   }
 
+  const contentType = getMimeType(artifactFileName)
+
   // get the IDs needed for the artifact creation
   const backendIds = getBackendIdsFromToken()
 
@@ -51,7 +79,8 @@ export async function uploadArtifact(
     workflowRunBackendId: backendIds.workflowRunBackendId,
     workflowJobRunBackendId: backendIds.workflowJobRunBackendId,
     name,
-    version: 4
+    mimeType: StringValue.create({value: contentType}),
+    version: 7
   }
 
   // if there is a retention period, add it to the request
@@ -68,15 +97,24 @@ export async function uploadArtifact(
     )
   }
 
-  const zipUploadStream = await createZipUploadStream(
-    zipSpecification,
-    options?.compressionLevel
-  )
+  let stream: WaterMarkedUploadStream
 
-  // Upload zip to blob storage
-  const uploadResult = await uploadZipToBlobStorage(
+  if (options?.skipArchive) {
+    // Upload raw file without archiving
+    stream = await createRawFileUploadStream(files[0])
+  } else {
+    // Create and upload zip archive
+    stream = await createZipUploadStream(
+      zipSpecification,
+      options?.compressionLevel
+    )
+  }
+
+  core.info(`Uploading artifact: ${artifactFileName}`)
+  const uploadResult = await uploadToBlobStorage(
     createArtifactResp.signedUploadUrl,
-    zipUploadStream
+    stream,
+    contentType
   )
 
   // finalize the artifact
@@ -105,7 +143,7 @@ export async function uploadArtifact(
 
   const artifactId = BigInt(finalizeArtifactResp.artifactId)
   core.info(
-    `Artifact ${name}.zip successfully finalized. Artifact ID ${artifactId}`
+    `Artifact ${name} successfully finalized. Artifact ID ${artifactId}`
   )
 
   return {

packages/artifact/src/internal/upload/zip.ts

@@ -1,31 +1,16 @@
-import * as stream from 'stream'
 import {realpath} from 'fs/promises'
 import archiver from 'archiver'
 import * as core from '@actions/core'
 import {UploadZipSpecification} from './upload-zip-specification.js'
 import {getUploadChunkSize} from '../shared/config.js'
+import {WaterMarkedUploadStream} from './stream.js'
 
 export const DEFAULT_COMPRESSION_LEVEL = 6
 
-// Custom stream transformer so we can set the highWaterMark property
-// See https://github.com/nodejs/node/issues/8855
-export class ZipUploadStream extends stream.Transform {
-  constructor(bufferSize: number) {
-    super({
-      highWaterMark: bufferSize
-    })
-  }
-
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  _transform(chunk: any, enc: any, cb: any): void {
-    cb(null, chunk)
-  }
-}
-
 export async function createZipUploadStream(
   uploadSpecification: UploadZipSpecification[],
   compressionLevel: number = DEFAULT_COMPRESSION_LEVEL
-): Promise<ZipUploadStream> {
+): Promise<WaterMarkedUploadStream> {
   core.debug(
     `Creating Artifact archive with compressionLevel: ${compressionLevel}`
   )
@@ -60,7 +45,7 @@ export async function createZipUploadStream(
   }
 
   const bufferSize = getUploadChunkSize()
-  const zipUploadStream = new ZipUploadStream(bufferSize)
+  const zipUploadStream = new WaterMarkedUploadStream(bufferSize)
 
   core.debug(
     `Zip write high watermark value ${zipUploadStream.writableHighWaterMark}`