Skip to content

Commit

Permalink
fix: env handling in s3 storage
Browse files Browse the repository at this point in the history
  • Loading branch information
frytg committed Dec 18, 2024
1 parent d931304 commit fe2fa54
Show file tree
Hide file tree
Showing 4 changed files with 30 additions and 37 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ permissions:

env:
NODE_ENV: production
JSR_DEPENDENCIES: "@cross/test @std/assert @std/fmt @frytg/logger @frytg/check-required-env"
JSR_DEPENDENCIES: "@cross/test @std/assert @std/fmt @frytg/logger"
# NPM_DEPENDENCIES: "luxon minio sinon"

jobs:
Expand Down
6 changes: 3 additions & 3 deletions storage-s3/s3.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ import sinon from 'sinon'
test('s3 - exports a Minio.Client instance', async () => {
// Setup
const envStub = sinon.stub(process, 'env').value({
S3_ENDPOINT: 'test-endpoint',
MY_SCW_ACCESS_KEY: 'test-access-key',
MY_SCW_SECRET_KEY: 'test-secret-key',
STORE_S3_ENDPOINT: 'test-endpoint',
STORE_S3_ACCESS_KEY: 'test-access-key',
STORE_S3_SECRET_KEY: 'test-secret-key',
})

// load module with stubbed env
Expand Down
13 changes: 9 additions & 4 deletions storage-s3/s3.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,17 @@
// load packages
import { getRequiredEnv } from '@frytg/check-required-env/get'
import process from 'node:process'
// @deno-types="minio/dist/esm/minio.d.mts"
import { Client } from 'minio'

// check environment variables
if (!process.env.STORE_S3_ENDPOINT) throw new Error('Environment variable STORE_S3_ENDPOINT is not defined')
if (!process.env.STORE_S3_ACCESS_KEY) throw new Error('Environment variable STORE_S3_ACCESS_KEY is not defined')
if (!process.env.STORE_S3_SECRET_KEY) throw new Error('Environment variable STORE_S3_SECRET_KEY is not defined')

// create a minio client
export const minioClient = new Client({
useSSL: true,
endPoint: getRequiredEnv('S3_ENDPOINT', false),
accessKey: getRequiredEnv('MY_SCW_ACCESS_KEY', false),
secretKey: getRequiredEnv('MY_SCW_SECRET_KEY', false),
endPoint: process.env.STORE_S3_ENDPOINT,
accessKey: process.env.STORE_S3_ACCESS_KEY,
secretKey: process.env.STORE_S3_SECRET_KEY,
})
46 changes: 17 additions & 29 deletions storage-s3/storage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@

// import packages
import { Buffer } from 'node:buffer'
import { getRequiredEnv } from '@frytg/check-required-env/get'
import type { BucketItem, BucketItemStat, BucketStream, Client } from 'minio/dist/esm/minio.d.mts'

// load utils
Expand All @@ -19,33 +18,33 @@ export const client: Client = minioClient

/**
* Retrieves an object from S3.
* @param {string} bucketName - The name of the bucket to retrieve the object from.
* @param {string} path - The path to the object in S3.
* @param {object} options - The options for the operation.
* @param {boolean} options.parseJson - Whether to parse the object as JSON. Defaults to `false`.
* @param {boolean} options.throwError - Whether to throw an error if the object does not exist. Defaults to `true`.
* @param {string} options.bucketName - The name of the bucket to retrieve the object from. Defaults to env `S3_BUCKET_NAME`.
* @returns {Promise<Buffer | string | null>}
*
* @see https://min.io/docs/minio/linux/developers/javascript/API.html#getObject
*
* @example
* ```ts
* import { getRequiredEnv } from '@frytg/check-required-env/get'
* import { getObject } from '@frytg/storage-s3'
*
* const object = await getObject('path/to/object.json', { parseJson: true })
* const object = await getObject(getRequiredEnv('S3_BUCKET_NAME'), 'path/to/object.json', { parseJson: true })
* console.log(object)
* ```
*/
export const getObject = (
bucketName: string,
path: string,
{
parseJson = false,
throwError = true,
bucketName = getRequiredEnv('S3_BUCKET_NAME'),
}: {
parseJson?: boolean
throwError?: boolean
bucketName?: string
},
): Promise<Buffer | string | object | null> =>
new Promise((resolve, reject) => {
Expand Down Expand Up @@ -79,27 +78,21 @@ export const getObject = (
*
* @example JSON
* ```ts
* import { getRequiredEnv } from '@frytg/check-required-env/get'
* import { uploadObject } from '@frytg/storage-s3'
*
* await uploadObject('path/to/object.json', { foo: 'bar' })
* await uploadObject(getRequiredEnv('S3_BUCKET_NAME'), 'path/to/object.json', { foo: 'bar' })
* ```
*
* @example Buffer
* ```ts
* import { getRequiredEnv } from '@frytg/check-required-env/get'
* import { uploadObject } from '@frytg/storage-s3'
*
* await uploadObject('path/to/object.blob', Buffer.from('foo'))
* await uploadObject(getRequiredEnv('S3_BUCKET_NAME'), 'path/to/object.blob', Buffer.from('foo'))
* ```
*/
export const uploadObject = async (
path: string,
data: Buffer | string,
{
bucketName = getRequiredEnv('S3_BUCKET_NAME'),
}: {
bucketName?: string
},
): Promise<void> => {
export const uploadObject = async (bucketName: string, path: string, data: Buffer | string): Promise<void> => {
// convert data to string if it's an object or array
let dataString = data
if (typeof data === 'object' || Array.isArray(data)) {
Expand All @@ -121,20 +114,14 @@ export const uploadObject = async (
*
* @example
* ```ts
* import { getRequiredEnv } from '@frytg/check-required-env/get'
* import { objectExists } from '@frytg/storage-s3'
*
* const exists = await objectExists('path/to/object.json')
* const exists = await objectExists(getRequiredEnv('S3_BUCKET_NAME'), 'path/to/object.json')
* console.log(exists) // null if it doesn't exist, otherwise the object stat
* ```
*/
export const objectExists = async (
path: string,
{
bucketName = getRequiredEnv('S3_BUCKET_NAME'),
}: {
bucketName?: string
},
): Promise<BucketItemStat | Promise<null>> => {
export const objectExists = async (bucketName: string, path: string): Promise<BucketItemStat | Promise<null>> => {
try {
// explicitly awaiting the result to avoid unhandled promise rejection
const result: BucketItemStat = await minioClient.statObject(bucketName, path)
Expand Down Expand Up @@ -165,28 +152,29 @@ const readableStreamForListObjects = (stream: BucketStream<BucketItem>): Promise
*
* @example
* ```ts
* import { getRequiredEnv } from '@frytg/check-required-env/get'
* import { listObjects } from '@frytg/storage-s3'
*
* const objects = await listObjects('path/to/prefix')
* const objects = await listObjects(getRequiredEnv('S3_BUCKET_NAME'), 'path/to/prefix')
* console.log(objects)
* ```
*
* @example Recursive
* ```ts
* import { getRequiredEnv } from '@frytg/check-required-env/get'
* import { listObjects } from '@frytg/storage-s3'
*
* const objects = await listObjects('path/to/prefix', { recursive: true })
* const objects = await listObjects(getRequiredEnv('S3_BUCKET_NAME'), 'path/to/prefix', { recursive: true })
* console.log(objects)
* ```
*/
export const listObjects = async (
bucketName: string,
prefix: string,
{
recursive = false,
bucketName = getRequiredEnv('S3_BUCKET_NAME'),
}: {
recursive?: boolean
bucketName?: string
},
): Promise<BucketItem[]> => {
const result: BucketStream<BucketItem> = await minioClient.listObjectsV2(bucketName, prefix, recursive)
Expand Down

0 comments on commit fe2fa54

Please sign in to comment.