import { Credentials, S3 } from 'aws-sdk'; import fs from 'fs'; import path from 'path'; import klawSync from 'klaw-sync'; import klaw, { Item } from 'klaw'; import mime from 'mime-types'; import UploadStream from 's3-stream-upload'; import { isEmpty } from 'ramda'; import { createReadStream } from 'fs-extra'; import archiver from 'archiver'; import { utils } from '@serverless/core'; import {} from '../../types'; export const getClients = ( credentials: Credentials, region: string ): { regular: S3; accelerated: S3; } => { const params = { region, credentials, }; // we need two S3 clients because creating/deleting buckets // is not available with the acceleration feature. return { regular: new S3(params), accelerated: new S3({ ...params, endpoint: `s3-accelerate.amazonaws.com` }), }; }; const bucketCreation = async (s3: S3, Bucket: S3.BucketName): Promise => { try { await s3.headBucket({ Bucket }).promise(); } catch (e) { if (e.code === 'NotFound' || e.code === 'NoSuchBucket') { await utils.sleep(2000); return bucketCreation(s3, Bucket); } throw new Error(e); } }; export const ensureBucket = async ( s3: S3, name: string, debug: (message: string) => void ): Promise => { try { debug(`Checking if bucket ${name} exists.`); await s3.headBucket({ Bucket: name }).promise(); } catch (e) { if (e.code === 'NotFound') { debug(`Bucket ${name} does not exist. Creating...`); await s3.createBucket({ Bucket: name }).promise(); // there's a race condition when using acceleration // so we need to sleep for a couple seconds. See this issue: // https://github.com/serverless/components/issues/428 debug(`Bucket ${name} created. Confirming it's ready...`); await bucketCreation(s3, name); debug(`Bucket ${name} creation confirmed.`); } else if (e.code === 'Forbidden' && e.message === null) { throw Error(`Forbidden: Invalid credentials or this AWS S3 bucket name may already be taken`); } else if (e.code === 'Forbidden') { throw Error(`Bucket name "${name}" is already taken.`); } else { throw e; } } }; export const uploadDir = async ( s3: S3, bucketName: string, dirPath: string, cacheControl?: S3.CacheControl, options?: { keyPrefix?: string } ): Promise => { const items: ReadonlyArray = await new Promise((resolve, reject) => { try { resolve(klawSync(dirPath)); } catch (error) { reject(error); } }); const uploadItems: Promise[] = []; items.forEach((item) => { if (item.stats.isDirectory()) { return; } let key = path.relative(dirPath, item.path); if (options?.keyPrefix) { key = path.posix.join(options.keyPrefix, key); } // convert backslashes to forward slashes on windows if (path.sep === '\\') { key = key.replace(/\\/g, '/'); } const itemParams = { Bucket: bucketName, Key: key, Body: fs.readFileSync(item.path), ContentType: mime.lookup(path.basename(item.path)) || 'application/octet-stream', CacheControl: cacheControl, }; uploadItems.push(s3.upload(itemParams).promise()); }); await Promise.all(uploadItems); }; export const packAndUploadDir = async ({ s3, bucketName, dirPath, key, append = [], cacheControl, }: { s3: S3; bucketName: string; dirPath: string; key: string; append?: string[]; cacheControl?: S3.CacheControl; }): Promise => { const ignore = (await utils.readFileIfExists(path.join(dirPath, '.slsignore'))) || []; return new Promise((resolve, reject) => { const archive = archiver('zip', { zlib: { level: 9 }, }); if (!isEmpty(append)) { append.forEach((file) => { const fileStream = createReadStream(file); archive.append(fileStream, { name: path.basename(file) }); }); } archive.glob( '**/*', { cwd: dirPath, ignore, }, {} ); archive .pipe( UploadStream(s3, { Bucket: bucketName, Key: key, CacheControl: cacheControl, }) ) .on('error', (err: Error) => reject(err)) .on('finish', () => resolve()); archive.finalize(); }); }; export const uploadFile = async ({ s3, bucketName, filePath, key, cacheControl, }: { s3: S3; bucketName: string; filePath: string; key: string; cacheControl?: S3.CacheControl; }): Promise => { return new Promise((resolve, reject) => { fs.createReadStream(filePath) .pipe( UploadStream(s3, { Bucket: bucketName, Key: key, ContentType: mime.lookup(filePath) || 'application/octet-stream', CacheControl: cacheControl, }) ) .on('error', (err: Error) => reject(err)) .on('finish', () => resolve()); }); }; /* * Delete Website Bucket */ export const clearBucket = async (s3: S3, bucketName: string): Promise => { try { const data = await s3.listObjects({ Bucket: bucketName }).promise(); const items = data.Contents || []; const promises = []; for (let i = 0; i < items.length; i += 1) { const deleteParams = { Bucket: bucketName, Key: items[i].Key as string }; const delObj = s3.deleteObject(deleteParams).promise(); promises.push(delObj); } await Promise.all(promises); } catch (error) { if (error.code !== 'NoSuchBucket') { throw error; } } }; export const accelerateBucket = async ( s3: S3, bucketName: string, accelerated: boolean ): Promise => { try { await s3 .putBucketAccelerateConfiguration({ AccelerateConfiguration: { Status: accelerated ? 'Enabled' : 'Suspended', }, Bucket: bucketName, }) .promise(); } catch (e) { if (e.code === 'NoSuchBucket') { await utils.sleep(2000); return accelerateBucket(s3, bucketName, accelerated); } throw e; } }; export const deleteBucket = async (s3: S3, bucketName: string): Promise => { try { await s3.deleteBucket({ Bucket: bucketName }).promise(); } catch (error) { if (error.code !== 'NoSuchBucket') { throw error; } } }; export const configureCors = async ( s3: S3, bucketName: string, config: S3.CORSConfiguration ): Promise => { const params = { Bucket: bucketName, CORSConfiguration: config }; try { await s3.putBucketCors(params).promise(); } catch (e) { if (e.code === 'NoSuchBucket') { await utils.sleep(2000); return configureCors(s3, bucketName, config); } throw e; } }; export const pathToPosix = (path: string): string => path.replace(/\\/g, '/'); export const readDirectoryFiles = (directory: string): Promise> => { const items: Item[] = []; return new Promise((resolve, reject) => { klaw(directory.trim()) .on('data', (item) => items.push(item)) .on('end', () => { resolve(items); }) .on('error', reject); }); }; export const filterOutDirectories = (fileItem: Item): boolean => !fileItem.stats.isDirectory(); export const getMimeType = (filePath: string): string => mime.lookup(path.basename(filePath)) || 'application/octet-stream';