diff --git a/migrations/tenant/57-unicode-object-names.sql b/migrations/tenant/57-unicode-object-names.sql new file mode 100644 index 00000000..3530f60f --- /dev/null +++ b/migrations/tenant/57-unicode-object-names.sql @@ -0,0 +1,143 @@ +DO $$ +DECLARE + -- SQL_ASCII databases do not have reliable Unicode code point semantics, so + -- the migration needs a byte-pattern branch instead of U&'' character checks. + -- This is the case for OrioleDB and/or --locale=C databases. + server_encoding text := current_setting('server_encoding'); +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint c + JOIN pg_class t ON t.oid = c.conrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE c.conname = 'objects_name_check' + AND n.nspname = 'storage' + AND t.relname = 'objects' + ) THEN + IF server_encoding = 'SQL_ASCII' THEN + EXECUTE $ddl$ + ALTER TABLE "storage"."objects" + ADD CONSTRAINT objects_name_check + CHECK ( + name !~ E'[\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F]' + AND name !~ E'\\xED[\\xA0-\\xBF][\\x80-\\xBF]' + AND name !~ E'\\xEF\\xBF\\xBE|\\xEF\\xBF\\xBF' + ) NOT VALID + $ddl$; + ELSE + EXECUTE $ddl$ + ALTER TABLE "storage"."objects" + ADD CONSTRAINT objects_name_check + CHECK ( + name !~ E'[\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F]' + AND POSITION(U&'\FFFE' IN name) = 0 + AND POSITION(U&'\FFFF' IN name) = 0 + ) NOT VALID + $ddl$; + END IF; + END IF; + + IF EXISTS ( + SELECT 1 + FROM pg_constraint c + JOIN pg_class t ON t.oid = c.conrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE c.conname = 'objects_name_check' + AND n.nspname = 'storage' + AND t.relname = 'objects' + AND c.convalidated = false + ) THEN + EXECUTE 'ALTER TABLE "storage"."objects" VALIDATE CONSTRAINT objects_name_check'; + END IF; + + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint c + JOIN pg_class t ON t.oid = c.conrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE c.conname = 's3_multipart_uploads_key_check' + AND n.nspname = 'storage' + AND t.relname = 's3_multipart_uploads' + ) THEN + IF server_encoding = 'SQL_ASCII' THEN + EXECUTE $ddl$ + ALTER TABLE "storage"."s3_multipart_uploads" + ADD CONSTRAINT s3_multipart_uploads_key_check + CHECK ( + key !~ E'[\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F]' + AND key !~ E'\\xED[\\xA0-\\xBF][\\x80-\\xBF]' + AND key !~ E'\\xEF\\xBF\\xBE|\\xEF\\xBF\\xBF' + ) NOT VALID + $ddl$; + ELSE + EXECUTE $ddl$ + ALTER TABLE "storage"."s3_multipart_uploads" + ADD CONSTRAINT s3_multipart_uploads_key_check + CHECK ( + key !~ E'[\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F]' + AND POSITION(U&'\FFFE' IN key) = 0 + AND POSITION(U&'\FFFF' IN key) = 0 + ) NOT VALID + $ddl$; + END IF; + END IF; + + IF EXISTS ( + SELECT 1 + FROM pg_constraint c + JOIN pg_class t ON t.oid = c.conrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE c.conname = 's3_multipart_uploads_key_check' + AND n.nspname = 'storage' + AND t.relname = 's3_multipart_uploads' + AND c.convalidated = false + ) THEN + EXECUTE 'ALTER TABLE "storage"."s3_multipart_uploads" VALIDATE CONSTRAINT s3_multipart_uploads_key_check'; + END IF; + + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint c + JOIN pg_class t ON t.oid = c.conrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE c.conname = 's3_multipart_uploads_parts_key_check' + AND n.nspname = 'storage' + AND t.relname = 's3_multipart_uploads_parts' + ) THEN + IF server_encoding = 'SQL_ASCII' THEN + EXECUTE $ddl$ + ALTER TABLE "storage"."s3_multipart_uploads_parts" + ADD CONSTRAINT s3_multipart_uploads_parts_key_check + CHECK ( + key !~ E'[\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F]' + AND key !~ E'\\xED[\\xA0-\\xBF][\\x80-\\xBF]' + AND key !~ E'\\xEF\\xBF\\xBE|\\xEF\\xBF\\xBF' + ) NOT VALID + $ddl$; + ELSE + EXECUTE $ddl$ + ALTER TABLE "storage"."s3_multipart_uploads_parts" + ADD CONSTRAINT s3_multipart_uploads_parts_key_check + CHECK ( + key !~ E'[\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F]' + AND POSITION(U&'\FFFE' IN key) = 0 + AND POSITION(U&'\FFFF' IN key) = 0 + ) NOT VALID + $ddl$; + END IF; + END IF; + + IF EXISTS ( + SELECT 1 + FROM pg_constraint c + JOIN pg_class t ON t.oid = c.conrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + WHERE c.conname = 's3_multipart_uploads_parts_key_check' + AND n.nspname = 'storage' + AND t.relname = 's3_multipart_uploads_parts' + AND c.convalidated = false + ) THEN + EXECUTE 'ALTER TABLE "storage"."s3_multipart_uploads_parts" VALIDATE CONSTRAINT s3_multipart_uploads_parts_key_check'; + END IF; +END +$$; diff --git a/src/http/plugins/xml.ts b/src/http/plugins/xml.ts index 0101d925..257f2e77 100644 --- a/src/http/plugins/xml.ts +++ b/src/http/plugins/xml.ts @@ -6,6 +6,38 @@ import xml from 'xml2js' type XmlParserOptions = { disableContentParser?: boolean; parseAsArray?: string[] } type RequestError = Error & { statusCode?: number } +function isValidXmlCodePoint(codePoint: number): boolean { + if (!Number.isInteger(codePoint) || codePoint < 0 || codePoint > 0x10ffff) { + return false + } + + return ( + codePoint === 0x9 || + codePoint === 0xa || + codePoint === 0xd || + (codePoint >= 0x20 && codePoint <= 0xd7ff) || + (codePoint >= 0xe000 && codePoint <= 0xfffd) || + (codePoint >= 0x10000 && codePoint <= 0x10ffff) + ) +} + +function getInvalidXmlNumericEntity(value: string): string | undefined { + const numericEntityPattern = /&#([xX][0-9a-fA-F]{1,6}|[0-9]{1,7});/g + + let match = numericEntityPattern.exec(value) + while (match) { + const rawValue = match[1] + const isHex = rawValue[0].toLowerCase() === 'x' + const codePoint = Number.parseInt(isHex ? rawValue.slice(1) : rawValue, isHex ? 16 : 10) + + if (!isValidXmlCodePoint(codePoint)) { + return match[0] + } + + match = numericEntityPattern.exec(value) + } +} + function forcePathAsArray(node: unknown, pathSegments: string[]): void { if (pathSegments.length === 0 || node === null || node === undefined) { return @@ -52,8 +84,19 @@ export const xmlParser = fastifyPlugin( return } + const xmlBody = typeof body === 'string' ? body : body.toString('utf8') + const invalidNumericEntity = getInvalidXmlNumericEntity(xmlBody) + if (invalidNumericEntity) { + const parseError: RequestError = new Error( + `Invalid XML payload: invalid numeric entity ${invalidNumericEntity}` + ) + parseError.statusCode = 400 + done(parseError) + return + } + xml.parseString( - body, + xmlBody, { explicitArray: false, trim: true, diff --git a/src/http/routes/object/getSignedObject.ts b/src/http/routes/object/getSignedObject.ts index d821741c..3709c41f 100644 --- a/src/http/routes/object/getSignedObject.ts +++ b/src/http/routes/object/getSignedObject.ts @@ -4,6 +4,7 @@ import { getConfig } from '../../../config' import { SignedToken, verifyJWT } from '../../../internal/auth' import { getJwtSecret } from '../../../internal/database' import { ERRORS } from '../../../internal/errors' +import { doesSignedTokenMatchRequestPath } from '../../../internal/http' import { ROUTE_OPERATIONS } from '../operations' const { storageS3Bucket } = getConfig() @@ -71,9 +72,8 @@ export default async function routes(fastify: FastifyInstance) { } const { url, exp } = payload - const path = `${request.params.bucketName}/${request.params['*']}` - if (url !== path) { + if (!doesSignedTokenMatchRequestPath(request.raw.url, '/object/sign', url)) { throw ERRORS.InvalidSignature() } diff --git a/src/http/routes/object/getSignedURL.ts b/src/http/routes/object/getSignedURL.ts index bf0aa841..a5d03fc3 100644 --- a/src/http/routes/object/getSignedURL.ts +++ b/src/http/routes/object/getSignedURL.ts @@ -67,7 +67,6 @@ export default async function routes(fastify: FastifyInstance) { const objectName = request.params['*'] const { expiresIn } = request.body - const urlPath = request.url.split('?').shift() const imageTransformationEnabled = await isImageTransformationEnabled(request.tenantId) const transformationOptions = imageTransformationEnabled @@ -82,7 +81,7 @@ export default async function routes(fastify: FastifyInstance) { const signedURL = await request.storage .from(bucketName) - .signObjectUrl(objectName, urlPath as string, expiresIn, transformationOptions) + .signObjectUrl(objectName, expiresIn, transformationOptions) return response.status(200).send({ signedURL }) } diff --git a/src/http/routes/object/getSignedUploadURL.ts b/src/http/routes/object/getSignedUploadURL.ts index b84c113b..dfb1f74d 100644 --- a/src/http/routes/object/getSignedUploadURL.ts +++ b/src/http/routes/object/getSignedUploadURL.ts @@ -67,11 +67,9 @@ export default async function routes(fastify: FastifyInstance) { const objectName = request.params['*'] const owner = request.owner - const urlPath = `${bucketName}/${objectName}` - const signedUpload = await request.storage .from(bucketName) - .signUploadObjectUrl(objectName, urlPath as string, uploadSignedUrlExpirationTime, owner, { + .signUploadObjectUrl(objectName, uploadSignedUrlExpirationTime, owner, { upsert: request.headers['x-upsert'] === 'true', }) diff --git a/src/http/routes/object/uploadSignedObject.ts b/src/http/routes/object/uploadSignedObject.ts index d0c280eb..6d77a862 100644 --- a/src/http/routes/object/uploadSignedObject.ts +++ b/src/http/routes/object/uploadSignedObject.ts @@ -1,4 +1,8 @@ import fastifyMultipart from '@fastify/multipart' +import { SignedUploadToken, verifyJWT } from '@internal/auth' +import { getJwtSecret } from '@internal/database' +import { ERRORS } from '@internal/errors' +import { doesSignedTokenMatchRequestPath } from '@internal/http' import { FastifyInstance } from 'fastify' import { FromSchema } from 'json-schema-to-ts' import { ROUTE_OPERATIONS } from '../operations' @@ -83,9 +87,21 @@ export default async function routes(fastify: FastifyInstance) { const { bucketName } = request.params const objectName = request.params['*'] - const { owner, upsert } = await request.storage - .from(bucketName) - .verifyObjectSignature(token, objectName) + let payload: SignedUploadToken + const { secret: jwtSecret, jwks } = await getJwtSecret(request.tenantId) + + try { + payload = (await verifyJWT(token, jwtSecret, jwks)) as SignedUploadToken + } catch (e) { + const err = e as Error + throw ERRORS.InvalidJWT(err) + } + + const { owner, upsert, url } = payload + + if (!doesSignedTokenMatchRequestPath(request.raw.url, '/object/upload/sign', url)) { + throw ERRORS.InvalidSignature() + } const { objectMetadata, path } = await request.storage .asSuperUser() diff --git a/src/http/routes/render/renderSignedImage.ts b/src/http/routes/render/renderSignedImage.ts index 3bf51dc0..6b19abb0 100644 --- a/src/http/routes/render/renderSignedImage.ts +++ b/src/http/routes/render/renderSignedImage.ts @@ -1,6 +1,7 @@ import { SignedToken, verifyJWT } from '@internal/auth' import { getJwtSecret, getTenantConfig } from '@internal/database' import { ERRORS } from '@internal/errors' +import { doesSignedTokenMatchRequestPath } from '@internal/http' import { ImageRenderer } from '@storage/renderer' import { FastifyInstance } from 'fastify' import { FromSchema } from 'json-schema-to-ts' @@ -68,10 +69,7 @@ export default async function routes(fastify: FastifyInstance) { } const { url, transformations, exp } = payload - - const path = `${request.params.bucketName}/${request.params['*']}` - - if (url !== path) { + if (!doesSignedTokenMatchRequestPath(request.raw.url, '/render/image/sign', url)) { throw ERRORS.InvalidSignature() } diff --git a/src/internal/database/migrations/types.ts b/src/internal/database/migrations/types.ts index 35bd839c..a568083d 100644 --- a/src/internal/database/migrations/types.ts +++ b/src/internal/database/migrations/types.ts @@ -56,4 +56,5 @@ export const DBMigration = { 'drop-index-object-level': 54, 'prevent-direct-deletes': 55, 'fix-optimized-search-function': 56, -} + 'unicode-object-names': 57, +} as const diff --git a/src/internal/errors/codes.ts b/src/internal/errors/codes.ts index 1902daa3..69f759d4 100644 --- a/src/internal/errors/codes.ts +++ b/src/internal/errors/codes.ts @@ -1,3 +1,4 @@ +import { safeEncodeURIComponent } from '../http' import { StorageBackendError } from './storage-error' export enum ErrorCode { @@ -319,12 +320,20 @@ export const ERRORS = { originalError: e, }), + InvalidObjectName: (e?: Error) => + new StorageBackendError({ + code: ErrorCode.InvalidKey, + httpStatusCode: 400, + message: 'Invalid object name', + originalError: e, + }), + InvalidKey: (key: string, e?: Error) => new StorageBackendError({ code: ErrorCode.InvalidKey, resource: key, httpStatusCode: 400, - message: `Invalid key: ${key}`, + message: `Invalid key: ${safeEncodeURIComponent(key)}`, originalError: e, }), diff --git a/src/internal/http/index.ts b/src/internal/http/index.ts index c9209a59..50cde1ce 100644 --- a/src/internal/http/index.ts +++ b/src/internal/http/index.ts @@ -1 +1,2 @@ export * from './agent' +export * from './url' diff --git a/src/internal/http/url.ts b/src/internal/http/url.ts new file mode 100644 index 00000000..1a7f0bc7 --- /dev/null +++ b/src/internal/http/url.ts @@ -0,0 +1,69 @@ +function toWellFormedString(value: string): string { + const maybeToWellFormed = (value as unknown as { toWellFormed?: () => string }).toWellFormed + if (typeof maybeToWellFormed === 'function') { + return maybeToWellFormed.call(value) + } + + let normalized = '' + for (let i = 0; i < value.length; i++) { + const currentCodeUnit = value.charCodeAt(i) + + if (currentCodeUnit >= 0xd800 && currentCodeUnit <= 0xdbff) { + const nextCodeUnit = value.charCodeAt(i + 1) + if (i + 1 < value.length && nextCodeUnit >= 0xdc00 && nextCodeUnit <= 0xdfff) { + normalized += value[i] + value[i + 1] + i += 1 + } else { + normalized += '\uFFFD' + } + continue + } + + if (currentCodeUnit >= 0xdc00 && currentCodeUnit <= 0xdfff) { + normalized += '\uFFFD' + continue + } + + normalized += value[i] + } + + return normalized +} + +export function safeEncodeURIComponent(value: string): string { + try { + return encodeURIComponent(value) + } catch { + return encodeURIComponent(toWellFormedString(value)) + } +} + +export function encodePathPreservingSeparators(path: string): string { + return path + .split('/') + .map((pathToken) => safeEncodeURIComponent(pathToken)) + .join('/') +} + +export function encodeBucketAndObjectPath(bucket: string, key: string): string { + return `${safeEncodeURIComponent(bucket)}/${encodePathPreservingSeparators(key)}` +} + +function stripQueryString(rawUrl: string): string { + const queryIdx = rawUrl.indexOf('?') + return queryIdx === -1 ? rawUrl : rawUrl.slice(0, queryIdx) +} + +export function doesSignedTokenMatchRequestPath( + rawUrl: string | undefined, + routePrefix: string, + signedObjectPath: string +): boolean { + if (!rawUrl) { + return false + } + + const pathname = stripQueryString(rawUrl) + const expectedPath = `${routePrefix}/${encodePathPreservingSeparators(signedObjectPath)}` + return pathname === expectedPath +} diff --git a/src/scripts/migrations-types.ts b/src/scripts/migrations-types.ts index 37414216..24fdcf9f 100644 --- a/src/scripts/migrations-types.ts +++ b/src/scripts/migrations-types.ts @@ -39,7 +39,7 @@ function main() { const template = `export const DBMigration = { ${migrationsEnum.join('\n')} -} +} as const ` const destinationPath = path.resolve( diff --git a/src/storage/backend/file.ts b/src/storage/backend/file.ts index f2754b1e..b3a98fd6 100644 --- a/src/storage/backend/file.ts +++ b/src/storage/backend/file.ts @@ -342,18 +342,17 @@ export class FileBackend implements StorageBackendAdapter { cacheControl: string ): Promise { const uploadId = randomUUID() - const multiPartFolder = this.resolveSecurePath( - path.join('multiparts', uploadId, bucketName, withOptionalVersion(key, version)) - ) - const multipartFile = this.resolveSecurePath( - path.join( - 'multiparts', - uploadId, - bucketName, - withOptionalVersion(key, version), - 'metadata.json' - ) - ) + const multiPartFolder = this.resolveSecureMultipartPath(uploadId, { + bucketName, + key, + version, + }) + const multipartFile = this.resolveSecureMultipartPath(uploadId, { + bucketName, + key, + version, + suffix: 'metadata.json', + }) await fsExtra.ensureDir(multiPartFolder) await fsExtra.writeFile(multipartFile, JSON.stringify({ contentType, cacheControl })) @@ -368,15 +367,12 @@ export class FileBackend implements StorageBackendAdapter { partNumber: number, body: stream.Readable ): Promise<{ ETag?: string }> { - const partPath = this.resolveSecurePath( - path.join( - 'multiparts', - uploadId, - bucketName, - withOptionalVersion(key, version), - `part-${partNumber}` - ) - ) + const partPath = this.resolveSecureMultipartPath(uploadId, { + bucketName, + key, + version, + suffix: `part-${partNumber}`, + }) const writeStream = fsExtra.createWriteStream(partPath) @@ -404,15 +400,12 @@ export class FileBackend implements StorageBackendAdapter { } > { const partsByEtags = parts.map(async (part) => { - const partFilePath = this.resolveSecurePath( - path.join( - 'multiparts', - uploadId, - bucketName, - withOptionalVersion(key, version), - `part-${part.PartNumber}` - ) - ) + const partFilePath = this.resolveSecureMultipartPath(uploadId, { + bucketName, + key, + version, + suffix: `part-${part.PartNumber}`, + }) const partExists = await fsExtra.pathExists(partFilePath) if (partExists) { @@ -432,15 +425,12 @@ export class FileBackend implements StorageBackendAdapter { const multipartStream = this.mergePartStreams(finalParts) const metadataContent = await fsExtra.readFile( - this.resolveSecurePath( - path.join( - 'multiparts', - uploadId, - bucketName, - withOptionalVersion(key, version), - 'metadata.json' - ) - ), + this.resolveSecureMultipartPath(uploadId, { + bucketName, + key, + version, + suffix: 'metadata.json', + }), 'utf-8' ) @@ -455,7 +445,7 @@ export class FileBackend implements StorageBackendAdapter { metadata.cacheControl ) - fsExtra.remove(this.resolveSecurePath(path.join('multiparts', uploadId))).catch(() => { + fsExtra.remove(this.resolveSecureMultipartPath(uploadId)).catch(() => { // no-op }) @@ -473,7 +463,7 @@ export class FileBackend implements StorageBackendAdapter { uploadId: string, version?: string ): Promise { - const multiPartFolder = this.resolveSecurePath(path.join('multiparts', uploadId)) + const multiPartFolder = this.resolveSecureMultipartPath(uploadId) await fsExtra.remove(multiPartFolder) @@ -495,15 +485,12 @@ export class FileBackend implements StorageBackendAdapter { sourceVersion?: string, rangeBytes?: { fromByte: number; toByte: number } ): Promise<{ eTag?: string; lastModified?: Date }> { - const partFilePath = this.resolveSecurePath( - path.join( - 'multiparts', - UploadId, - storageS3Bucket, - withOptionalVersion(key, version), - `part-${PartNumber}` - ) - ) + const partFilePath = this.resolveSecureMultipartPath(UploadId, { + bucketName: storageS3Bucket, + key, + version, + suffix: `part-${PartNumber}`, + }) const sourceFilePath = this.resolveSecurePath( `${storageS3Bucket}/${withOptionalVersion(sourceKey, sourceVersion)}` ) @@ -692,6 +679,34 @@ export class FileBackend implements StorageBackendAdapter { return normalizedPath } + private resolveSecureMultipartPath( + uploadId: string, + options?: { + bucketName?: string + key?: string + version?: string + suffix?: string + } + ): string { + // Intentionally avoid path.join for attacker-controlled segments so dot segments remain visible + // to resolveSecurePath instead of being normalized away beforehand. + let relativePath = `multiparts/${uploadId}` + + if (typeof options?.bucketName === 'string') { + relativePath += `/${options.bucketName}` + } + + if (typeof options?.key === 'string') { + relativePath += `/${withOptionalVersion(options.key, options.version)}` + } + + if (typeof options?.suffix === 'string') { + relativePath += `/${options.suffix}` + } + + return this.resolveSecurePath(relativePath) + } + private async etag(file: string, stats: fs.Stats): Promise { if (this.etagAlgorithm === 'md5') { const checksum = await this.computeMd5(file) diff --git a/src/storage/backend/s3/adapter.ts b/src/storage/backend/s3/adapter.ts index 5c8fc473..c85ed762 100644 --- a/src/storage/backend/s3/adapter.ts +++ b/src/storage/backend/s3/adapter.ts @@ -19,7 +19,7 @@ import { import { Progress, Upload } from '@aws-sdk/lib-storage' import { getSignedUrl } from '@aws-sdk/s3-request-presigner' import { ERRORS, StorageBackendError } from '@internal/errors' -import { createAgent, InstrumentedAgent } from '@internal/http' +import { createAgent, encodeBucketAndObjectPath, InstrumentedAgent } from '@internal/http' import { monitorStream } from '@internal/streams' import { NodeHttpHandler } from '@smithy/node-http-handler' import { BackupObjectInfo, ObjectBackup } from '@storage/backend/s3/backup' @@ -255,7 +255,7 @@ export class S3Backend implements StorageBackendAdapter { try { const command = new CopyObjectCommand({ Bucket: bucket, - CopySource: encodeURIComponent(`${bucket}/${withOptionalVersion(source, version)}`), + CopySource: encodeBucketAndObjectPath(bucket, withOptionalVersion(source, version)), Key: withOptionalVersion(destination, destinationVersion), CopySourceIfMatch: conditions?.ifMatch, CopySourceIfNoneMatch: conditions?.ifNoneMatch, @@ -568,7 +568,10 @@ export class S3Backend implements StorageBackendAdapter { Key: withOptionalVersion(key, version), UploadId, PartNumber, - CopySource: `${storageS3Bucket}/${withOptionalVersion(sourceKey, sourceKeyVersion)}`, + CopySource: encodeBucketAndObjectPath( + storageS3Bucket, + withOptionalVersion(sourceKey, sourceKeyVersion) + ), CopySourceRange: bytesRange ? `bytes=${bytesRange.fromByte}-${bytesRange.toByte}` : undefined, }) diff --git a/src/storage/backend/s3/backup.ts b/src/storage/backend/s3/backup.ts index a9c65d6d..255b14f7 100644 --- a/src/storage/backend/s3/backup.ts +++ b/src/storage/backend/s3/backup.ts @@ -7,6 +7,7 @@ import { S3Client, UploadPartCopyCommand, } from '@aws-sdk/client-s3' +import { encodeBucketAndObjectPath } from '@internal/http' const FIVE_GB = 5 * 1024 * 1024 * 1024 @@ -69,7 +70,7 @@ export class ObjectBackup { const copyParams = { Bucket: destinationBucket, Key: destinationKey, - CopySource: encodeURIComponent(`/${sourceBucket}/${sourceKey}`), + CopySource: encodeBucketAndObjectPath(sourceBucket, sourceKey), } const copyCommand = new CopyObjectCommand(copyParams) @@ -157,7 +158,7 @@ export class ObjectBackup { Key: destinationKey, PartNumber: partNumber, UploadId: uploadId, - CopySource: encodeURIComponent(`/${sourceBucket}/${sourceKey}`), + CopySource: encodeBucketAndObjectPath(sourceBucket, sourceKey), CopySourceRange: `bytes=${start}-${end}`, }) diff --git a/src/storage/database/knex.ts b/src/storage/database/knex.ts index 54665710..88ddfef8 100644 --- a/src/storage/database/knex.ts +++ b/src/storage/database/knex.ts @@ -1114,6 +1114,12 @@ export class DBError extends StorageBackendError implements RenderableError { } static fromDBError(pgError: DatabaseError, query?: string) { + const objectNameConstraintNames = new Set([ + 'objects_name_check', + 's3_multipart_uploads_key_check', + 's3_multipart_uploads_parts_key_check', + ]) + switch (pgError.code) { case '42501': return ERRORS.AccessDenied( @@ -1145,7 +1151,18 @@ export class DBError extends StorageBackendError implements RenderableError { code: pgError.code, }) default: - return ERRORS.DatabaseError(`database error, code: ${pgError.code}`, pgError).withMetadata({ + if ( + pgError.code === '23514' && + pgError.constraint && + objectNameConstraintNames.has(pgError.constraint) + ) { + return ERRORS.InvalidObjectName(pgError).withMetadata({ + query, + code: pgError.code, + }) + } + + return ERRORS.DatabaseError(pgError.message, pgError).withMetadata({ query, code: pgError.code, }) diff --git a/src/storage/events/lifecycle/webhook-filter.ts b/src/storage/events/lifecycle/webhook-filter.ts new file mode 100644 index 00000000..312995d5 --- /dev/null +++ b/src/storage/events/lifecycle/webhook-filter.ts @@ -0,0 +1,10 @@ +export function shouldDisableWebhookEvent( + disabledEvents: string[], + eventType: string, + payload: { bucketId: string; name: string } +) { + return ( + disabledEvents.includes(`Webhook:${eventType}`) || + disabledEvents.includes(`Webhook:${eventType}:${payload.bucketId}/${payload.name}`) + ) +} diff --git a/src/storage/events/lifecycle/webhook.ts b/src/storage/events/lifecycle/webhook.ts index 782e5045..24a269d4 100644 --- a/src/storage/events/lifecycle/webhook.ts +++ b/src/storage/events/lifecycle/webhook.ts @@ -5,6 +5,7 @@ import axios from 'axios' import { Job, SendOptions, WorkOptions } from 'pg-boss' import { getConfig } from '../../../config' import { BaseEvent } from '../base-event' +import { shouldDisableWebhookEvent } from './webhook-filter' const { isMultitenant, @@ -73,12 +74,7 @@ export class Webhook extends BaseEvent { // Do not send an event if disabled for this specific tenant const tenant = await getTenantConfig(payload.tenant.ref) const disabledEvents = tenant.disableEvents || [] - if ( - disabledEvents.includes(`Webhook:${payload.event.type}`) || - disabledEvents.includes( - `Webhook:${payload.event.type}:${payload.event.payload.bucketId}/${payload.event.payload.name}` - ) - ) { + if (shouldDisableWebhookEvent(disabledEvents, payload.event.type, payload.event.payload)) { return false } } diff --git a/src/storage/limits.ts b/src/storage/limits.ts index 35ca8573..6392bb12 100644 --- a/src/storage/limits.ts +++ b/src/storage/limits.ts @@ -52,9 +52,15 @@ export async function isImageTransformationEnabled(tenantId: string) { * @param key */ export function isValidKey(key: string): boolean { - // only allow s3 safe characters and characters which require special handling for now - // https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html - return key.length > 0 && /^(\w|\/|!|-|\.|\*|'|\(|\)| |&|\$|@|=|;|:|\+|,|\?)*$/.test(key) + // Allow any sequence of Unicode characters with UTF-8 encoding, + // except characters not allowed in XML 1.0. + // See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html + // See: https://www.w3.org/TR/REC-xml/#charsets + // + const regex = + /[\0-\x08\x0B\f\x0E-\x1F\uFFFE\uFFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]/ + + return key.length > 0 && !regex.test(key) } /** diff --git a/src/storage/object.ts b/src/storage/object.ts index 9a8781c5..af26583b 100644 --- a/src/storage/object.ts +++ b/src/storage/object.ts @@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto' import { SignedUploadToken, signJWT, verifyJWT } from '@internal/auth' import { getJwtSecret } from '@internal/database' import { ERRORS } from '@internal/errors' +import { encodeBucketAndObjectPath, encodePathPreservingSeparators } from '@internal/http' import { StorageObjectLocator } from '@storage/locator' import { Obj } from '@storage/schemas' import { FastifyRequest } from 'fastify/types/request' @@ -23,6 +24,7 @@ const { requestUrlLengthLimit } = getConfig() interface CopyObjectParams { sourceKey: string + sourceVersion?: string destinationBucket: string destinationKey: string owner?: string @@ -294,6 +296,7 @@ export class ObjectStorage { */ async copyObject({ sourceKey, + sourceVersion, destinationBucket, destinationKey, owner, @@ -324,6 +327,10 @@ export class ObjectStorage { 'bucket_id,metadata,user_metadata,version' ) + if (sourceVersion && originObject.version !== sourceVersion) { + throw ERRORS.NoSuchKey(sourceKey) + } + const baseMetadata = originObject.metadata || {} const destinationMetadata = copyMetadata ? baseMetadata @@ -343,7 +350,7 @@ export class ObjectStorage { const copyResult = await this.backend.copyObject( this.location.getRootLocation(), s3SourceKey, - originObject.version, + sourceVersion || originObject.version, s3DestinationKey, newVersion, destinationMetadata, @@ -600,7 +607,15 @@ export class ObjectStorage { const prefix = options?.prefix || '' const delimiter = options?.delimiter - const cursor = options?.cursor ? decodeContinuationToken(options.cursor) : undefined + let cursor: ContinuationToken | undefined + if (options?.cursor) { + try { + cursor = decodeContinuationToken(options.cursor) + } catch (error) { + throw ERRORS.InvalidParameter('ContinuationToken', { error: error as Error }) + } + } + let searchResult = await this.db.listObjectsV2(this.bucketId, { prefix: options?.prefix, delimiter: options?.delimiter, @@ -655,7 +670,12 @@ export class ObjectStorage { const name = obj.id === null && !obj.name.endsWith('/') ? obj.name + '/' : obj.name target.push({ ...obj, - name: options?.encodingType === 'url' ? encodeURIComponent(name) : name, + name: + options?.encodingType === 'url' + ? obj.id === null + ? encodePathPreservingSeparators(name) + : encodeURIComponent(name) + : name, }) }) @@ -699,7 +719,6 @@ export class ObjectStorage { */ async signObjectUrl( objectName: string, - url: string, expiresIn: number, metadata?: Record ) { @@ -716,8 +735,7 @@ export class ObjectStorage { // make sure it's never able to specify a role JWT claim delete metadata['role'] - const urlParts = url.split('/') - const urlToSign = decodeURI(urlParts.splice(3).join('/')) + const urlToSign = `${this.bucketId}/${objectName}` const { urlSigningKey } = await getJwtSecret(this.db.tenantId) const token = await signJWT({ url: urlToSign, ...metadata }, urlSigningKey, expiresIn) @@ -727,8 +745,10 @@ export class ObjectStorage { urlPath = 'render/image' } + const encodedUrlToSign = encodeBucketAndObjectPath(this.bucketId, objectName) + // @todo parse the url properly - return `/${urlPath}/sign/${urlToSign}?token=${token}` + return `/${urlPath}/sign/${encodedUrlToSign}?token=${token}` } /** @@ -764,7 +784,8 @@ export class ObjectStorage { if (nameSet.has(path)) { const urlToSign = `${this.bucketId}/${path}` const token = await signJWT({ url: urlToSign }, urlSigningKey, expiresIn) - signedURL = `/object/sign/${urlToSign}?token=${token}` + const encodedUrlToSign = encodeBucketAndObjectPath(this.bucketId, path) + signedURL = `/object/sign/${encodedUrlToSign}?token=${token}` } else { error = 'Either the object does not exist or you do not have access to it' } @@ -787,11 +808,12 @@ export class ObjectStorage { */ async signUploadObjectUrl( objectName: string, - url: string, expiresIn: number, owner?: string, options?: { upsert?: boolean } ) { + mustBeValidKey(objectName) + // check if user has INSERT permissions await this.uploader.canUpload({ bucketId: this.bucketId, @@ -801,13 +823,16 @@ export class ObjectStorage { }) const { urlSigningKey } = await getJwtSecret(this.db.tenantId) + const urlToSign = `${this.bucketId}/${objectName}` const token = await signJWT( - { owner, url, upsert: Boolean(options?.upsert) }, + { owner, url: urlToSign, upsert: Boolean(options?.upsert) }, urlSigningKey, expiresIn ) - return { url: `/object/upload/sign/${url}?token=${token}`, token } + const encodedUrlToSign = encodeBucketAndObjectPath(this.bucketId, objectName) + + return { url: `/object/upload/sign/${encodedUrlToSign}?token=${token}`, token } } /** @@ -826,16 +851,12 @@ export class ObjectStorage { throw ERRORS.InvalidJWT(err) } - const { url, exp } = payload + const { url } = payload if (url !== `${this.bucketId}/${objectName}`) { throw ERRORS.InvalidSignature() } - if (exp * 1000 < Date.now()) { - throw ERRORS.ExpiredSignature() - } - return payload } } @@ -853,15 +874,18 @@ const CONTINUATION_TOKEN_PART_MAP: Record = { c: 'sortColumn', a: 'sortColumnAfter', } +const CONTINUATION_TOKEN_VERSION = '1' +const CONTINUATION_TOKEN_VERSION_KEY = 'v' function encodeContinuationToken(tokenInfo: ContinuationToken) { - let result = '' + const result: string[] = [`${CONTINUATION_TOKEN_VERSION_KEY}:${CONTINUATION_TOKEN_VERSION}`] for (const [k, v] of Object.entries(CONTINUATION_TOKEN_PART_MAP)) { - if (tokenInfo[v]) { - result += `${k}:${tokenInfo[v]}\n` + const value = tokenInfo[v] + if (value) { + result.push(`${k}:${encodeURIComponent(value)}`) } } - return Buffer.from(result.slice(0, -1)).toString('base64') + return Buffer.from(result.join('\n')).toString('base64') } function decodeContinuationToken(token: string): ContinuationToken { @@ -870,12 +894,47 @@ function decodeContinuationToken(token: string): ContinuationToken { startAfter: '', sortOrder: 'asc', } + const parsedParts: { key: string; value: string }[] = [] + let version: string | undefined + for (const part of decodedParts) { const partMatch = part.match(/^(\S):(.*)/) - if (!partMatch || partMatch.length !== 3 || !(partMatch[1] in CONTINUATION_TOKEN_PART_MAP)) { + if (!partMatch || partMatch.length !== 3) { + throw new Error('Invalid continuation token') + } + const key = partMatch[1] + const value = partMatch[2] + + if (key === CONTINUATION_TOKEN_VERSION_KEY) { + if (version !== undefined) { + throw new Error('Invalid continuation token') + } + version = value + continue + } + + if (!(key in CONTINUATION_TOKEN_PART_MAP)) { + throw new Error('Invalid continuation token') + } + parsedParts.push({ key, value }) + } + + if (version && version !== CONTINUATION_TOKEN_VERSION) { + throw new Error('Invalid continuation token') + } + + for (const part of parsedParts) { + if (!version) { + // Backward compatibility: legacy cursor values were stored unescaped. + result[CONTINUATION_TOKEN_PART_MAP[part.key]] = part.value + continue + } + + try { + result[CONTINUATION_TOKEN_PART_MAP[part.key]] = decodeURIComponent(part.value) + } catch { throw new Error('Invalid continuation token') } - result[CONTINUATION_TOKEN_PART_MAP[partMatch[1]]] = partMatch[2] } return result } diff --git a/src/storage/protocols/s3/copy-source-parser.ts b/src/storage/protocols/s3/copy-source-parser.ts new file mode 100644 index 00000000..dd847753 --- /dev/null +++ b/src/storage/protocols/s3/copy-source-parser.ts @@ -0,0 +1,64 @@ +import { ERRORS } from '@internal/errors' + +const VERSION_ID_QUERY_DELIMITER = '?versionId=' + +function splitCopySourceVersion(copySource: string): { + encodedPath: string + sourceVersion?: string +} { + const versionQueryIdx = copySource.lastIndexOf(VERSION_ID_QUERY_DELIMITER) + + if (versionQueryIdx === -1) { + return { + encodedPath: copySource, + } + } + + const sourceVersion = copySource.slice(versionQueryIdx + VERSION_ID_QUERY_DELIMITER.length) + if (!sourceVersion) { + throw ERRORS.InvalidParameter('CopySource') + } + + return { + encodedPath: copySource.slice(0, versionQueryIdx), + sourceVersion, + } +} + +export function parseCopySource(copySource: string): { + bucketName: string + objectKey: string + sourceVersion?: string +} { + const normalizedCopySource = copySource.startsWith('/') ? copySource.slice(1) : copySource + // Preserve raw '?' characters in partially encoded keys and only peel off a trailing versionId suffix. + const { encodedPath, sourceVersion } = splitCopySourceVersion(normalizedCopySource) + + let decodedPath = '' + try { + decodedPath = decodeURIComponent(encodedPath) + } catch { + throw ERRORS.InvalidParameter('CopySource') + } + + if (decodedPath.startsWith('/')) { + decodedPath = decodedPath.slice(1) + } + + const separatorIdx = decodedPath.indexOf('/') + if (separatorIdx <= 0) { + throw ERRORS.MissingParameter('CopySource') + } + + const bucketName = decodedPath.slice(0, separatorIdx) + const objectKey = decodedPath.slice(separatorIdx + 1) + if (!objectKey) { + throw ERRORS.MissingParameter('CopySource') + } + + return { + bucketName, + objectKey, + sourceVersion, + } +} diff --git a/src/storage/protocols/s3/s3-handler.ts b/src/storage/protocols/s3/s3-handler.ts index 10f22424..37eea93b 100644 --- a/src/storage/protocols/s3/s3-handler.ts +++ b/src/storage/protocols/s3/s3-handler.ts @@ -19,6 +19,7 @@ import { } from '@aws-sdk/client-s3' import { decrypt, encrypt } from '@internal/auth' import { ERRORS } from '@internal/errors' +import { encodePathPreservingSeparators } from '@internal/http' import { logger, logSchema } from '@internal/monitoring' import { PassThrough, Readable } from 'stream' import stream from 'stream/promises' @@ -28,6 +29,7 @@ import { S3MultipartUpload } from '../../schemas' import { Storage } from '../../storage' import { Uploader, validateMimeType } from '../../uploader' import { ByteLimitTransformStream } from './byte-limit-stream' +import { parseCopySource } from './copy-source-parser' const { storageS3Region, storageS3Bucket } = getConfig() @@ -288,17 +290,31 @@ export class S3ProtocolHandler { const bucket = command.Bucket const limit = maxKeys || 200 + let nextUploadKeyToken: string | undefined + let nextUploadToken: string | undefined + + if (keyContinuationToken) { + try { + nextUploadKeyToken = decodeContinuationToken(keyContinuationToken) + } catch (error) { + throw ERRORS.InvalidParameter('KeyMarker', { error: error as Error }) + } + } + + if (uploadContinuationToken) { + try { + nextUploadToken = decodeContinuationToken(uploadContinuationToken) + } catch (error) { + throw ERRORS.InvalidParameter('UploadIdMarker', { error: error as Error }) + } + } const multipartUploads = await this.storage.db.listMultipartUploads(bucket, { prefix, deltimeter: delimiter, maxKeys: limit + 1, - nextUploadKeyToken: keyContinuationToken - ? decodeContinuationToken(keyContinuationToken) - : undefined, - nextUploadToken: uploadContinuationToken - ? decodeContinuationToken(uploadContinuationToken) - : undefined, + nextUploadKeyToken, + nextUploadToken, }) let results: Partial[] = multipartUploads @@ -319,7 +335,10 @@ export class S3ProtocolHandler { delimitedResults.push({ isFolder: true, id: object.id, - key: command.EncodingType === 'url' ? encodeURIComponent(currPrefix) : currPrefix, + key: + command.EncodingType === 'url' + ? encodePathPreservingSeparators(currPrefix) + : currPrefix, bucket_id: bucket, }) continue @@ -1051,14 +1070,11 @@ export class S3ProtocolHandler { throw ERRORS.MissingParameter('CopySource') } - const sourceBucket = ( - CopySource.startsWith('/') ? CopySource.replace('/', '').split('/') : CopySource.split('/') - ).shift() - - const sourceKey = (CopySource.startsWith('/') ? CopySource.replace('/', '') : CopySource) - .split('/') - .slice(1) - .join('/') + const { + bucketName: sourceBucket, + objectKey: sourceKey, + sourceVersion, + } = parseCopySource(CopySource) if (!sourceBucket) { throw ERRORS.InvalidBucketName('') @@ -1075,6 +1091,7 @@ export class S3ProtocolHandler { const copyResult = await this.storage.from(sourceBucket).copyObject({ sourceKey, + sourceVersion, destinationBucket: Bucket, destinationKey: Key, owner: this.owner, @@ -1186,14 +1203,11 @@ export class S3ProtocolHandler { throw ERRORS.MissingParameter('CopySourceRange') } - const sourceBucketName = ( - CopySource.startsWith('/') ? CopySource.replace('/', '').split('/') : CopySource.split('/') - ).shift() - - const sourceKey = (CopySource.startsWith('/') ? CopySource.replace('/', '') : CopySource) - .split('/') - .slice(1) - .join('/') + const { + bucketName: sourceBucketName, + objectKey: sourceKey, + sourceVersion, + } = parseCopySource(CopySource) if (!sourceBucketName) { throw ERRORS.NoSuchBucket('') @@ -1210,6 +1224,10 @@ export class S3ProtocolHandler { 'id,name,version,metadata' ) + if (sourceVersion && copySource.version !== sourceVersion) { + throw ERRORS.NoSuchKey(sourceKey) + } + let copySize = copySource.metadata?.size || 0 let rangeBytes: { fromByte: number; toByte: number } | undefined = undefined @@ -1268,7 +1286,7 @@ export class S3ProtocolHandler { objectName: copySource.name, tenantId: this.tenantId, }), - copySource.version, + sourceVersion || copySource.version, rangeBytes ) @@ -1402,15 +1420,61 @@ function isUSASCII(str: string): boolean { } function encodeContinuationToken(name: string) { - return Buffer.from(`l:${name}`).toString('base64') + return Buffer.from(`v:1\nl:${encodeURIComponent(name)}`).toString('base64') +} + +function decodeLegacyContinuationToken(decoded: string) { + // Backward compatibility: preserve pre-version behavior for old in-flight tokens. + const continuationToken = decoded.slice(2) + if (!continuationToken) { + throw new Error('Invalid continuation token') + } + return continuationToken } function decodeContinuationToken(token: string) { - const decoded = Buffer.from(token, 'base64').toString().split(':') + const decoded = Buffer.from(token, 'base64').toString() - if (decoded.length === 0) { + if (decoded.startsWith('l:')) { + return decodeLegacyContinuationToken(decoded) + } + + const parts = decoded.split('\n') + let version: string | undefined + let encodedValue: string | undefined + + for (const part of parts) { + const match = part.match(/^(\S):(.*)/) + if (!match || match.length !== 3) { + throw new Error('Invalid continuation token') + } + + if (match[1] === 'v') { + if (version !== undefined) { + throw new Error('Invalid continuation token') + } + version = match[2] + continue + } + + if (match[1] === 'l') { + if (encodedValue !== undefined) { + throw new Error('Invalid continuation token') + } + encodedValue = match[2] + continue + } + + throw new Error('Invalid continuation token') + } + + if (version !== '1' || !encodedValue) { throw new Error('Invalid continuation token') } - return decoded[1] + try { + return decodeURIComponent(encodedValue) + } catch { + throw new Error('Invalid continuation token') + } } diff --git a/src/test/bucket.test.ts b/src/test/bucket.test.ts index 20c9c546..174e30a3 100644 --- a/src/test/bucket.test.ts +++ b/src/test/bucket.test.ts @@ -183,6 +183,17 @@ describe('testing GET all buckets', () => { }) test('user is able to get buckets with limit, offset, search and sorting', async () => { + const allBucketsResponse = await appInstance.inject({ + method: 'GET', + url: `/bucket?sortColumn=name&sortOrder=asc&search=bucket`, + headers: { + authorization: `Bearer ${process.env.AUTHENTICATED_KEY}`, + }, + }) + expect(allBucketsResponse.statusCode).toBe(200) + const allBuckets = JSON.parse(allBucketsResponse.body) + expect(allBuckets.length).toBeGreaterThanOrEqual(4) + const response = await appInstance.inject({ method: 'GET', url: `/bucket?limit=1&offset=3&sortColumn=name&sortOrder=asc&search=bucket`, @@ -193,9 +204,10 @@ describe('testing GET all buckets', () => { expect(response.statusCode).toBe(200) const responseJSON = JSON.parse(response.body) expect(responseJSON.length).toEqual(1) + const expectedBucket = allBuckets[3] expect(responseJSON[0]).toMatchObject({ - id: 'bucket4', - name: 'bucket4', + id: expectedBucket.id, + name: expectedBucket.name, type: expect.any(String), public: false, file_size_limit: null, @@ -449,11 +461,10 @@ describe('testing public bucket functionality', () => { describe('testing count objects in bucket', () => { const { tenantId } = getConfig() - const testObjectCount = 27 + const testObjectNames = ['object-1.txt', 'object-2.txt', 'object-3.txt'] const testOwnerId = randomUUID() let db: StorageKnexDB let testBucketId: string - let testObjectNames: string[] beforeAll(async () => { const serviceKeyUser = await getServiceKeyUser(tenantId) @@ -470,10 +481,6 @@ describe('testing count objects in bucket', () => { }) testBucketId = `count-objects-${randomUUID()}` - testObjectNames = Array.from({ length: testObjectCount }, (_, idx) => { - return `fixtures/count-object-${idx}` - }) - await db.createBucket({ id: testBucketId, name: testBucketId, @@ -490,7 +497,7 @@ describe('testing count objects in bucket', () => { name, owner: testOwnerId, bucket_id: testBucketId, - metadata: { size: 1 }, + metadata: { size: 1234 }, user_metadata: null, version: undefined, }) @@ -505,13 +512,13 @@ describe('testing count objects in bucket', () => { }) it('should return correct object count', async () => { - await expect(db.countObjectsInBucket(testBucketId)).resolves.toBe(testObjectCount) + await expect(db.countObjectsInBucket(testBucketId)).resolves.toBe(testObjectNames.length) }) it('should return limited object count', async () => { - await expect(db.countObjectsInBucket(testBucketId, 22)).resolves.toBe(22) + await expect(db.countObjectsInBucket(testBucketId, 2)).resolves.toBe(2) }) it('should return full object count if limit is greater than total', async () => { - await expect(db.countObjectsInBucket(testBucketId, 999)).resolves.toBe(testObjectCount) + await expect(db.countObjectsInBucket(testBucketId, 999)).resolves.toBe(testObjectNames.length) }) it('should return 0 object count if there are no objects with provided bucket id', async () => { await expect(db.countObjectsInBucket('this-is-not-a-bucket-at-all', 999)).resolves.toBe(0) diff --git a/src/test/common.ts b/src/test/common.ts index 5cf75d5a..092e23cb 100644 --- a/src/test/common.ts +++ b/src/test/common.ts @@ -10,6 +10,42 @@ export const adminApp = app({}) const ENV = process.env const projectRoot = path.join(__dirname, '..', '..') +/** + * Should support all Unicode characters with UTF-8 encoding according to AWS S3 object naming guide, including: + * - Safe characters: 0-9 a-z A-Z !-_.*'() + * - Characters that might require special handling: &$@=;/:+,? and Space and ASCII characters \t, \n, and \r. + * - Characters: \{}^%`[]"<>~#| and non-printable ASCII characters (128–255 decimal characters). + * + * The following characters are not allowed: + * - ASCII characters 0x00–0x1F, except 0x09, 0x0A, and 0x0D. + * - Unicode \u{FFFE} and \u{FFFF}. + * - Lone surrogates characters. + * See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html + * See: https://www.w3.org/TR/REC-xml/#charsets + */ +export function getUnicodeObjectName(): string { + const objectName = 'test' + .concat("!-_*.'()") + // Characters that might require special handling + .concat('&$@=;:+,? \x09\x0A\x0D') + // Characters to avoid + .concat('\\{}^%`[]"<>~#|\xFF') + // MinIO max. length for each '/' separated segment is 255 + .concat('/') + .concat([...Array(127).keys()].map((i) => String.fromCodePoint(i + 128)).join('')) + .concat('/') + // Some special Unicode characters + .concat('\u2028\u202F\u{0001FFFF}') + // Some other Unicode characters + .concat('일이삼\u{0001f642}') + + return objectName +} + +export function getInvalidObjectName(): string { + return 'test\x01\x02\x03.txt' +} + export function useMockQueue() { const queueSpy: jest.SpyInstance | undefined = undefined beforeEach(() => { diff --git a/src/test/database-error-mapping.test.ts b/src/test/database-error-mapping.test.ts new file mode 100644 index 00000000..bf3b57bb --- /dev/null +++ b/src/test/database-error-mapping.test.ts @@ -0,0 +1,41 @@ +import { ErrorCode } from '@internal/errors' +import { DatabaseError } from 'pg' +import { DBError } from '../storage/database/knex' + +function createPgError(message: string, overrides?: Partial): DatabaseError { + return Object.assign(new Error(message), overrides) as DatabaseError +} + +describe('DBError.fromDBError', () => { + test.each([ + ['objects_name_check', 'insert into storage.objects ...'], + ['s3_multipart_uploads_key_check', 'insert into storage.s3_multipart_uploads ...'], + ['s3_multipart_uploads_parts_key_check', 'insert into storage.s3_multipart_uploads_parts ...'], + ])('maps %s violations to a stable Invalid object name message', (constraint, query) => { + const pgError = createPgError(`violates check constraint "${constraint}"`, { + code: '23514', + constraint, + }) + + const err = DBError.fromDBError(pgError, query) + + expect(err.code).toBe(ErrorCode.InvalidKey) + expect(err.message).toBe('Invalid object name') + expect(err.metadata).toMatchObject({ + code: '23514', + query, + }) + }) + + it('preserves the original database message for other check constraints', () => { + const pgError = createPgError('violates check constraint "something_else"', { + code: '23514', + constraint: 'something_else', + }) + + const err = DBError.fromDBError(pgError) + + expect(err.code).toBe(ErrorCode.DatabaseError) + expect(err.message).toBe('violates check constraint "something_else"') + }) +}) diff --git a/src/test/error-codes.test.ts b/src/test/error-codes.test.ts new file mode 100644 index 00000000..b6e3e79a --- /dev/null +++ b/src/test/error-codes.test.ts @@ -0,0 +1,36 @@ +import { ERRORS, ErrorCode, StorageBackendError } from '@internal/errors' + +describe('ERRORS.InvalidKey', () => { + it('does not throw for unpaired high surrogates', () => { + const malformedKey = 'bad-\uD800-key' + + expect(() => ERRORS.InvalidKey(malformedKey)).not.toThrow() + + const error = ERRORS.InvalidKey(malformedKey) + expect(error).toBeInstanceOf(StorageBackendError) + expect(error.code).toBe(ErrorCode.InvalidKey) + expect(error.httpStatusCode).toBe(400) + expect(error.message).toBe('Invalid key: bad-%EF%BF%BD-key') + }) + + it('does not throw for unpaired low surrogates', () => { + const malformedKey = 'bad-\uDC00-key' + + expect(() => ERRORS.InvalidKey(malformedKey)).not.toThrow() + + const error = ERRORS.InvalidKey(malformedKey) + expect(error.code).toBe(ErrorCode.InvalidKey) + expect(error.httpStatusCode).toBe(400) + expect(error.message).toBe('Invalid key: bad-%EF%BF%BD-key') + }) + + it('encodes valid Unicode and reserved characters in InvalidKey messages', () => { + const malformedKey = 'bad-일이삼/🙂?#%.png' + + const error = ERRORS.InvalidKey(malformedKey) + + expect(error.code).toBe(ErrorCode.InvalidKey) + expect(error.httpStatusCode).toBe(400) + expect(error.message).toBe(`Invalid key: ${encodeURIComponent(malformedKey)}`) + }) +}) diff --git a/src/test/file-backend.test.ts b/src/test/file-backend.test.ts index d3f7586a..ca5ed144 100644 --- a/src/test/file-backend.test.ts +++ b/src/test/file-backend.test.ts @@ -314,6 +314,22 @@ describe('FileBackend traversal protection', () => { }) }) + it('rejects multipart keys that only rebase within the storage root', async () => { + const traversalKey = '../multipart-rebased.txt' + + await expect( + backend.createMultiPartUpload('bucket', traversalKey, 'v1', 'text/plain', 'no-cache') + ).rejects.toMatchObject({ + code: 'InvalidKey', + }) + + await expect( + backend.uploadPart('bucket', traversalKey, 'v1', 'upload-id', 1, Readable.from('escape-part')) + ).rejects.toMatchObject({ + code: 'InvalidKey', + }) + }) + it('rejects traversal key in object operations with InvalidKey', async () => { const traversalKey = `${'../'.repeat(20)}tmp/${escapePrefix}/object-escape.txt` @@ -408,6 +424,14 @@ describe('FileBackend traversal protection', () => { code: 'InvalidKey', }) }) + + it('rejects multipart upload ids that would be normalized to sibling in-root paths', async () => { + await expect( + backend.abortMultipartUpload('bucket', 'key', '../upload-id') + ).rejects.toMatchObject({ + code: 'InvalidKey', + }) + }) }) describe('FileBackend lastModified', () => { diff --git a/src/test/limits.test.ts b/src/test/limits.test.ts new file mode 100644 index 00000000..f7704118 --- /dev/null +++ b/src/test/limits.test.ts @@ -0,0 +1,37 @@ +import { isValidKey } from '../storage/limits' + +describe('isValidKey', () => { + test('accepts unicode object names', () => { + expect(isValidKey('folder/일이삼/🙂/a b.txt')).toBe(true) + }) + + test('accepts tab, newline, and carriage return', () => { + expect(isValidKey('a\tb\nc\rd')).toBe(true) + }) + + test('rejects empty keys', () => { + expect(isValidKey('')).toBe(false) + }) + + test('rejects ASCII control characters except tab/newline/carriage return', () => { + expect(isValidKey('invalid\x01name')).toBe(false) + }) + + test('accepts DEL (0x7F) as a valid key character', () => { + expect(isValidKey('valid\x7Fname')).toBe(true) + }) + + test('rejects non-characters U+FFFE and U+FFFF', () => { + expect(isValidKey(`invalid${'\uFFFE'}`)).toBe(false) + expect(isValidKey(`invalid${'\uFFFF'}`)).toBe(false) + }) + + test('rejects lone surrogate code units', () => { + expect(isValidKey(`bad${'\uD83D'}`)).toBe(false) + expect(isValidKey(`bad${'\uDC00'}`)).toBe(false) + }) + + test('accepts valid surrogate pairs', () => { + expect(isValidKey('ok🙂name')).toBe(true) + }) +}) diff --git a/src/test/object-list-v2.test.ts b/src/test/object-list-v2.test.ts index 768eaf32..c89bd419 100644 --- a/src/test/object-list-v2.test.ts +++ b/src/test/object-list-v2.test.ts @@ -578,6 +578,7 @@ describe('objects - list v2 sorting tests', () => { }) const LIST_V2_WILDCARD_BUCKET = `list-v2-wildcard-${randomUUID()}` +const LIST_V2_CURSOR_BUCKET = `list-v2-cursor-${randomUUID()}` describe('objects - list v2 prefix wildcard handling', () => { beforeAll(async () => { @@ -700,3 +701,208 @@ describe('objects - list v2 prefix wildcard handling', () => { expect(data.objects.map((obj) => obj.name)).toEqual([literalMatch]) }) }) + +describe('objects - list v2 cursor encoding', () => { + beforeAll(async () => { + appInstance = app() + await appInstance.inject({ + method: 'POST', + url: `/bucket`, + headers: { + authorization: `Bearer ${serviceKey}`, + }, + payload: { + name: LIST_V2_CURSOR_BUCKET, + }, + }) + await appInstance.close() + }) + + afterAll(async () => { + appInstance = app() + await appInstance.inject({ + method: 'POST', + url: `/bucket/${LIST_V2_CURSOR_BUCKET}/empty`, + headers: { + authorization: `Bearer ${serviceKey}`, + }, + }) + + await appInstance.inject({ + method: 'DELETE', + url: `/bucket/${LIST_V2_CURSOR_BUCKET}`, + headers: { + authorization: `Bearer ${serviceKey}`, + }, + }) + + await appInstance.close() + }) + + test('paginates when keys contain newline and percent characters', async () => { + const runId = randomUUID() + const prefix = `cursor-${runId}-` + const keys = [`${prefix}first-\n-🙂-%.txt`, `${prefix}second-\n-일이삼-:.txt`] + + for (const key of keys) { + const uploadResponse = await appInstance.inject({ + method: 'POST', + url: `/object/${LIST_V2_CURSOR_BUCKET}/${encodeURIComponent(key)}`, + payload: createUpload('utf8.txt', 'cursor test'), + headers: { + authorization: `Bearer ${serviceKey}`, + }, + }) + expect(uploadResponse.statusCode).toBe(200) + } + + const page1Response = await appInstance.inject({ + method: 'POST', + url: `/object/list-v2/${LIST_V2_CURSOR_BUCKET}`, + payload: { + with_delimiter: false, + prefix, + limit: 1, + }, + headers: { + authorization: `Bearer ${serviceKey}`, + }, + }) + expect(page1Response.statusCode).toBe(200) + const page1 = page1Response.json() + expect(page1.objects).toHaveLength(1) + expect(page1.hasNext).toBe(true) + expect(page1.nextCursor).toBeTruthy() + + const page2Response = await appInstance.inject({ + method: 'POST', + url: `/object/list-v2/${LIST_V2_CURSOR_BUCKET}`, + payload: { + with_delimiter: false, + prefix, + limit: 1, + cursor: page1.nextCursor, + }, + headers: { + authorization: `Bearer ${serviceKey}`, + }, + }) + expect(page2Response.statusCode).toBe(200) + const page2 = page2Response.json() + expect(page2.objects).toHaveLength(1) + expect(page2.hasNext).toBe(false) + + const listed = [page1.objects[0]?.name, page2.objects[0]?.name].filter(Boolean).sort() + expect(listed).toEqual([...keys].sort()) + }) + + test('supports legacy unescaped cursor values with literal % hex sequences', async () => { + const runId = randomUUID() + const prefix = `cursor-legacy-${runId}-` + const firstKey = `${prefix}file%20name.txt` + const secondKey = `${prefix}file~name.txt` + + for (const key of [firstKey, secondKey]) { + const uploadResponse = await appInstance.inject({ + method: 'POST', + url: `/object/${LIST_V2_CURSOR_BUCKET}/${encodeURIComponent(key)}`, + payload: createUpload('utf8.txt', 'cursor legacy test'), + headers: { + authorization: `Bearer ${serviceKey}`, + }, + }) + expect(uploadResponse.statusCode).toBe(200) + } + + const page1Response = await appInstance.inject({ + method: 'POST', + url: `/object/list-v2/${LIST_V2_CURSOR_BUCKET}`, + payload: { + with_delimiter: false, + prefix, + limit: 1, + }, + headers: { + authorization: `Bearer ${serviceKey}`, + }, + }) + expect(page1Response.statusCode).toBe(200) + const page1 = page1Response.json() + expect(page1.objects).toHaveLength(1) + expect(page1.objects[0]?.name).toBe(firstKey) + + const legacyCursor = Buffer.from(`l:${firstKey}\no:asc`).toString('base64') + const page2Response = await appInstance.inject({ + method: 'POST', + url: `/object/list-v2/${LIST_V2_CURSOR_BUCKET}`, + payload: { + with_delimiter: false, + prefix, + limit: 1, + cursor: legacyCursor, + }, + headers: { + authorization: `Bearer ${serviceKey}`, + }, + }) + expect(page2Response.statusCode).toBe(200) + const page2 = page2Response.json() + expect(page2.objects).toHaveLength(1) + expect(page2.objects[0]?.name).toBe(secondKey) + }) + + test('supports legacy unescaped cursor values with mixed Unicode and literal % sequences', async () => { + const runId = randomUUID() + const prefix = `cursor-legacy-unicode-${runId}-` + const firstKey = `${prefix}일이삼-%20-🙂.txt` + const secondKey = `${prefix}일이삼-~.txt` + + for (const key of [firstKey, secondKey]) { + const uploadResponse = await appInstance.inject({ + method: 'POST', + url: `/object/${LIST_V2_CURSOR_BUCKET}/${encodeURIComponent(key)}`, + payload: createUpload('utf8.txt', 'cursor legacy unicode test'), + headers: { + authorization: `Bearer ${serviceKey}`, + }, + }) + expect(uploadResponse.statusCode).toBe(200) + } + + const page1Response = await appInstance.inject({ + method: 'POST', + url: `/object/list-v2/${LIST_V2_CURSOR_BUCKET}`, + payload: { + with_delimiter: false, + prefix, + limit: 1, + }, + headers: { + authorization: `Bearer ${serviceKey}`, + }, + }) + expect(page1Response.statusCode).toBe(200) + const page1 = page1Response.json() + expect(page1.objects).toHaveLength(1) + expect(page1.objects[0]?.name).toBe(firstKey) + + const legacyCursor = Buffer.from(`l:${firstKey}\no:asc`).toString('base64') + const page2Response = await appInstance.inject({ + method: 'POST', + url: `/object/list-v2/${LIST_V2_CURSOR_BUCKET}`, + payload: { + with_delimiter: false, + prefix, + limit: 1, + cursor: legacyCursor, + }, + headers: { + authorization: `Bearer ${serviceKey}`, + }, + }) + expect(page2Response.statusCode).toBe(200) + const page2 = page2Response.json() + expect(page2.objects).toHaveLength(1) + expect(page2.objects[0]?.name).toBe(secondKey) + }) +}) diff --git a/src/test/object.test.ts b/src/test/object.test.ts index 77944d8b..782a3848 100644 --- a/src/test/object.test.ts +++ b/src/test/object.test.ts @@ -12,12 +12,14 @@ import app from '../app' import { getConfig, JwksConfig, JwksConfigKeyOCT, mergeConfig } from '../config' import { backends, Obj } from '../storage' import { ObjectAdminDelete } from '../storage/events' -import { useMockObject, useMockQueue } from './common' +import { getInvalidObjectName, getUnicodeObjectName, useMockObject, useMockQueue } from './common' import { withDeleteEnabled } from './utils/storage' const { jwtSecret, serviceKeyAsync, tenantId } = getConfig() const anonKey = process.env.ANON_KEY || '' const S3Backend = backends.S3Backend +const TEST_OWNER_ID = '317eadce-631a-4429-a0bb-f19a7a517b4a' +const routeTestObjectsToCleanup: Array<{ bucketId: string; name: string }> = [] let appInstance: FastifyInstance let tnx: Knex.Transaction | undefined @@ -35,6 +37,60 @@ async function getSuperuserPostgrestClient() { return tnx } +async function seedObjectForRouteTest(name: string, bucketId = 'bucket2') { + const seedTx = await getSuperuserPostgrestClient() + await seedTx.from('objects').insert({ + bucket_id: bucketId, + name, + owner: TEST_OWNER_ID, + version: `seed-version-${randomUUID()}`, + metadata: { mimetype: 'image/png', size: 1234 }, + }) + await seedTx.commit() + tnx = undefined + routeTestObjectsToCleanup.push({ bucketId, name }) +} + +async function cleanupRouteTestObjects() { + const authorization = `Bearer ${await serviceKeyAsync}` + + for (const { bucketId, name } of routeTestObjectsToCleanup.splice(0)) { + const response = await appInstance.inject({ + method: 'DELETE', + url: `/object/${bucketId}/${encodeURIComponent(name)}`, + headers: { + authorization, + }, + }) + + if (response.statusCode !== 200 && response.statusCode !== 400) { + throw new Error(`Unexpected cleanup response ${response.statusCode} for ${bucketId}/${name}`) + } + } +} + +async function cleanupObjectsByPrefix(prefix: string, bucketId = 'bucket2') { + const cleanupTx = await getSuperuserPostgrestClient() + const objects = await cleanupTx + .from('objects') + .select('name') + .where({ bucket_id: bucketId }) + .whereLike('name', `${prefix}%`) + await cleanupTx.commit() + tnx = undefined + + const authorization = `Bearer ${await serviceKeyAsync}` + for (const object of objects) { + await appInstance.inject({ + method: 'DELETE', + url: `/object/${bucketId}/${encodeURIComponent(object.name)}`, + headers: { + authorization, + }, + }) + } +} + useMockObject() useMockQueue() @@ -46,7 +102,13 @@ beforeEach(() => { afterEach(async () => { if (tnx) { await tnx.commit() + tnx = undefined } + + if (routeTestObjectsToCleanup.length > 0) { + await cleanupRouteTestObjects() + } + await appInstance.close() }) @@ -1479,6 +1541,56 @@ describe('testing copy object', () => { expect(response.statusCode).toBe(400) expect(S3Backend.prototype.copyObject).not.toHaveBeenCalled() }) + + test('can copy objects when keys include ASCII URL-reserved characters', async () => { + const sourceKey = `authenticated/copy-src-${randomUUID()}-q?foo=1&bar=%25+plus;semi:colon,.png` + const destinationKey = `authenticated/copy-dst-${randomUUID()}-q?foo=2&bar=%25+plus;semi:colon,.png` + await seedObjectForRouteTest(sourceKey) + + const response = await appInstance.inject({ + method: 'POST', + url: '/object/copy', + headers: { + authorization: `Bearer ${process.env.AUTHENTICATED_KEY}`, + }, + payload: { + bucketId: 'bucket2', + sourceKey, + destinationKey, + }, + }) + + expect(response.statusCode).toBe(200) + expect(S3Backend.prototype.copyObject).toBeCalled() + const jsonResponse = response.json() + expect(jsonResponse.Key).toBe(`bucket2/${destinationKey}`) + expect(jsonResponse.name).toBe(destinationKey) + }) + + test('can copy objects when keys include Unicode and URL-reserved characters', async () => { + const sourceKey = `authenticated/copy-src-${randomUUID()}-일이삼-🙂-q?foo=1&bar=%25+plus;semi:colon,.png` + const destinationKey = `authenticated/copy-dst-${randomUUID()}-éè-中文-🙂-q?foo=2&bar=%25+plus;semi:colon,.png` + await seedObjectForRouteTest(sourceKey) + + const response = await appInstance.inject({ + method: 'POST', + url: '/object/copy', + headers: { + authorization: `Bearer ${process.env.AUTHENTICATED_KEY}`, + }, + payload: { + bucketId: 'bucket2', + sourceKey, + destinationKey, + }, + }) + + expect(response.statusCode).toBe(200) + expect(S3Backend.prototype.copyObject).toBeCalled() + const jsonResponse = response.json() + expect(jsonResponse.Key).toBe(`bucket2/${destinationKey}`) + expect(jsonResponse.name).toBe(destinationKey) + }) }) /** @@ -1645,6 +1757,65 @@ describe('testing deleting multiple objects', () => { expect(results).toHaveLength(1) expect(results[0].name).toBe('authenticated/delete-multiple7.png') }) + + test('can delete multiple objects with Unicode keys', async () => { + const authorization = `Bearer ${await serviceKeyAsync}` + const path = './src/test/assets/sadcat.jpg' + const { size } = fs.statSync(path) + const prefixes = [ + `authenticated/delete-many-${randomUUID()}-일이삼-🙂.png`, + `authenticated/delete-many-${randomUUID()}-éè-中文-?query&x=1#frag%25+plus;semi:colon,.png`, + ] + + for (const prefix of prefixes) { + const uploadResponse = await appInstance.inject({ + method: 'PUT', + url: `/object/bucket2/${encodeURIComponent(prefix)}`, + headers: { + authorization, + 'Content-Length': size, + 'Content-Type': 'image/jpeg', + }, + payload: fs.createReadStream(path), + }) + + expect(uploadResponse.statusCode).toBe(200) + } + + const deleteResponse = await appInstance.inject({ + method: 'DELETE', + url: '/object/bucket2', + headers: { + authorization, + }, + payload: { + prefixes, + }, + }) + + expect(deleteResponse.statusCode).toBe(200) + expect(S3Backend.prototype.deleteObjects).toBeCalled() + const results = JSON.parse(deleteResponse.body) + expect(results).toHaveLength(2) + expect(results.map((item: { name: string }) => item.name).sort()).toEqual([...prefixes].sort()) + + for (const prefix of prefixes) { + const getResponse = await appInstance.inject({ + method: 'GET', + url: `/object/bucket2/${encodeURIComponent(prefix)}`, + headers: { + authorization, + }, + }) + + expect(getResponse.statusCode).toBe(400) + expect(getResponse.json()).toMatchObject({ + statusCode: '404', + error: 'not_found', + message: 'Object not found', + }) + } + }) }) /** @@ -1873,7 +2044,7 @@ describe('testing uploading with generated signed upload URL', () => { }) const BUCKET_ID = 'bucket2' - const OBJECT_NAME = 'public/sadcat-upload1.png' + const OBJECT_NAME = `public/sadcat-upload-${randomUUID()}.png` const urlToSign = `${BUCKET_ID}/${OBJECT_NAME}` const owner = '317eadce-631a-4429-a0bb-f19a7a517b4a' @@ -1917,10 +2088,11 @@ describe('testing uploading with generated signed upload URL', () => { const headers = Object.assign({}, form.getHeaders(), { 'content-type': 'image/jpeg', }) + const objectName = `public/sadcat-upload-${randomUUID()}.png` const response = await appInstance.inject({ method: 'PUT', - url: `/object/upload/sign/bucket2/public/sadcat-upload1.png`, + url: `/object/upload/sign/bucket2/${objectName}`, headers, payload: form, }) @@ -1934,10 +2106,11 @@ describe('testing uploading with generated signed upload URL', () => { const headers = Object.assign({}, form.getHeaders(), { 'content-type': 'image/jpeg', }) + const objectName = `public/sadcat-upload-${randomUUID()}.png` const response = await appInstance.inject({ method: 'PUT', - url: `/object/upload/sign/bucket2/public/sadcat-upload1.png?token=xxx`, + url: `/object/upload/sign/bucket2/${objectName}?token=xxx`, headers, payload: form, }) @@ -1953,7 +2126,7 @@ describe('testing uploading with generated signed upload URL', () => { }) const BUCKET_ID = 'bucket2' - const OBJECT_NAME = 'public/sadcat-upload1.png' + const OBJECT_NAME = `public/sadcat-upload-${randomUUID()}.png` const urlToSign = `${BUCKET_ID}/${OBJECT_NAME}` const owner = '317eadce-631a-4429-a0bb-f19a7a517b4a' @@ -1976,7 +2149,7 @@ describe('testing uploading with generated signed upload URL', () => { } const BUCKET_ID = 'bucket2' - const OBJECT_NAME = 'signed/sadcat-upload-signed-2.png' + const OBJECT_NAME = `signed/sadcat-upload-signed-${randomUUID()}.png` const urlToSign = `${BUCKET_ID}/${OBJECT_NAME}` // Upload a file first @@ -2021,7 +2194,7 @@ describe('testing uploading with generated signed upload URL', () => { } const BUCKET_ID = 'bucket2' - const OBJECT_NAME = 'signed/sadcat-upload-signed-3.png' + const OBJECT_NAME = `signed/sadcat-upload-signed-${randomUUID()}.png` const urlToSign = `${BUCKET_ID}/${OBJECT_NAME}` const owner = '317eadce-631a-4429-a0bb-f19a7a517b4a' @@ -2130,6 +2303,131 @@ describe('testing generating signed URLs', () => { const result = JSON.parse(response.body) expect(result[0].error).toBe('Either the object does not exist or you do not have access to it') }) + + test('can generate and use batch signed URLs with Unicode and URL-reserved characters', async () => { + const objectName = `authenticated/signed-batch-${randomUUID()}-éè-中文-🙂-q?foo=1&bar=%25+plus;semi:colon,#frag.png` + const authorization = `Bearer ${await serviceKeyAsync}` + const path = './src/test/assets/sadcat.jpg' + const { size } = fs.statSync(path) + + const uploadResponse = await appInstance.inject({ + method: 'PUT', + url: `/object/bucket2/${encodeURIComponent(objectName)}`, + headers: { + authorization, + 'Content-Length': size, + 'Content-Type': 'image/jpeg', + }, + payload: fs.createReadStream(path), + }) + expect(uploadResponse.statusCode).toBe(200) + + const signResponse = await appInstance.inject({ + method: 'POST', + url: '/object/sign/bucket2', + headers: { + authorization, + }, + payload: { + expiresIn: 60, + paths: [objectName], + }, + }) + expect(signResponse.statusCode).toBe(200) + + const [signedObject] = + signResponse.json<{ error: string | null; path: string; signedURL: string | null }[]>() + expect(signedObject.error).toBeNull() + expect(signedObject.path).toBe(objectName) + expect(signedObject.signedURL).toBeTruthy() + + const signedURLParsed = new URL(signedObject.signedURL || '', 'http://localhost') + expect(signedURLParsed.searchParams.get('token')).toBeTruthy() + + const getResponse = await appInstance.inject({ + method: 'GET', + url: `${signedURLParsed.pathname}${signedURLParsed.search}`, + }) + expect(getResponse.statusCode).toBe(200) + expect(getResponse.headers['etag']).toBe('abc') + + const deleteResponse = await appInstance.inject({ + method: 'DELETE', + url: `/object/bucket2/${encodeURIComponent(objectName)}`, + headers: { + authorization, + }, + }) + expect(deleteResponse.statusCode).toBe(200) + }) + + test('can generate and use batch signed URLs for nested Unicode and URL-reserved paths', async () => { + const authorization = `Bearer ${await serviceKeyAsync}` + const path = './src/test/assets/sadcat.jpg' + const { size } = fs.statSync(path) + const objectNames = [ + `authenticated/signed-batch-${randomUUID()}-폴더?x=1/세그먼트#tag/leaf+plus;semi,.png`, + `authenticated/signed-batch-${randomUUID()}-éè/中文&name=1/끝🙂.png`, + ] + + for (const objectName of objectNames) { + const uploadResponse = await appInstance.inject({ + method: 'PUT', + url: `/object/bucket2/${encodeURIComponent(objectName)}`, + headers: { + authorization, + 'Content-Length': size, + 'Content-Type': 'image/jpeg', + }, + payload: fs.createReadStream(path), + }) + expect(uploadResponse.statusCode).toBe(200) + } + + const signResponse = await appInstance.inject({ + method: 'POST', + url: '/object/sign/bucket2', + headers: { + authorization, + }, + payload: { + expiresIn: 60, + paths: objectNames, + }, + }) + expect(signResponse.statusCode).toBe(200) + + const signedObjects = + signResponse.json<{ error: string | null; path: string; signedURL: string | null }[]>() + expect(signedObjects).toHaveLength(2) + + for (const signedObject of signedObjects) { + expect(signedObject.error).toBeNull() + expect(objectNames).toContain(signedObject.path) + expect(signedObject.signedURL).toBeTruthy() + + const signedURLParsed = new URL(signedObject.signedURL || '', 'http://localhost') + expect(signedURLParsed.searchParams.get('token')).toBeTruthy() + + const getResponse = await appInstance.inject({ + method: 'GET', + url: `${signedURLParsed.pathname}${signedURLParsed.search}`, + }) + expect(getResponse.statusCode).toBe(200) + expect(getResponse.headers['etag']).toBe('abc') + } + + for (const objectName of objectNames) { + const deleteResponse = await appInstance.inject({ + method: 'DELETE', + url: `/object/bucket2/${encodeURIComponent(objectName)}`, + headers: { + authorization, + }, + }) + expect(deleteResponse.statusCode).toBe(200) + } + }) }) /** @@ -2205,6 +2503,34 @@ describe('testing retrieving signed URL', () => { expect(body.error).toBe('InvalidSignature') }) + test('rejects double-encoded signed object paths', async () => { + const objectName = `authenticated/signed-double-${randomUUID()}-일이삼.txt` + await seedObjectForRouteTest(objectName) + + const urlToSign = `bucket2/${objectName}` + const jwtToken = await signJWT({ url: urlToSign }, jwtSecret, 100) + const encodedPath = urlToSign + .split('/') + .map((segment) => encodeURIComponent(segment)) + .join('/') + + const validResponse = await appInstance.inject({ + method: 'GET', + url: `/object/sign/${encodedPath}?token=${jwtToken}`, + }) + expect(validResponse.statusCode).toBe(200) + + const doubleEncodedPath = encodedPath.replaceAll('%', '%25') + const doubleEncodedResponse = await appInstance.inject({ + method: 'GET', + url: `/object/sign/${doubleEncodedPath}?token=${jwtToken}`, + }) + + expect(doubleEncodedResponse.statusCode).toBe(400) + const body = doubleEncodedResponse.json<{ error: string }>() + expect(body.error).toBe('InvalidSignature') + }) + test('get object without a token', async () => { const response = await appInstance.inject({ method: 'GET', @@ -2421,9 +2747,79 @@ describe('testing move object', () => { expect(S3Backend.prototype.copyObject).not.toHaveBeenCalled() expect(S3Backend.prototype.deleteObject).not.toHaveBeenCalled() }) + + test('can move objects when keys include ASCII URL-reserved characters', async () => { + const sourceKey = `authenticated/move-src-${randomUUID()}-q?foo=1&bar=%25+plus;semi:colon,.png` + const destinationKey = `authenticated/move-dst-${randomUUID()}-q?foo=2&bar=%25+plus;semi:colon,.png` + await seedObjectForRouteTest(sourceKey) + + const response = await appInstance.inject({ + method: 'POST', + url: '/object/move', + payload: { + bucketId: 'bucket2', + sourceKey, + destinationKey, + }, + headers: { + authorization: `Bearer ${process.env.AUTHENTICATED_KEY}`, + }, + }) + + expect(response.statusCode).toBe(200) + expect(S3Backend.prototype.copyObject).toHaveBeenCalled() + expect(S3Backend.prototype.deleteObjects).toHaveBeenCalled() + expect(response.json().message).toBe('Successfully moved') + + const conn = await getSuperuserPostgrestClient() + const movedObject = await conn + .table('objects') + .select('name') + .where('bucket_id', 'bucket2') + .where('name', destinationKey) + .first() + expect(movedObject?.name).toBe(destinationKey) + }) + + test('can move objects when keys include Unicode and URL-reserved characters', async () => { + const sourceKey = `authenticated/move-src-${randomUUID()}-일이삼-🙂-q?foo=1&bar=%25+plus;semi:colon,.png` + const destinationKey = `authenticated/move-dst-${randomUUID()}-éè-中文-🙂-q?foo=2&bar=%25+plus;semi:colon,.png` + await seedObjectForRouteTest(sourceKey) + + const response = await appInstance.inject({ + method: 'POST', + url: '/object/move', + payload: { + bucketId: 'bucket2', + sourceKey, + destinationKey, + }, + headers: { + authorization: `Bearer ${process.env.AUTHENTICATED_KEY}`, + }, + }) + + expect(response.statusCode).toBe(200) + expect(S3Backend.prototype.copyObject).toHaveBeenCalled() + expect(S3Backend.prototype.deleteObjects).toHaveBeenCalled() + expect(response.json().message).toBe('Successfully moved') + + const conn = await getSuperuserPostgrestClient() + const movedObject = await conn + .table('objects') + .select('name') + .where('bucket_id', 'bucket2') + .where('name', destinationKey) + .first() + expect(movedObject?.name).toBe(destinationKey) + }) }) describe('testing list objects', () => { + beforeEach(async () => { + await cleanupObjectsByPrefix('public/signed-double-') + }) + test('searching the bucket root folder', async () => { const response = await appInstance.inject({ method: 'POST', @@ -2881,3 +3277,470 @@ describe('x-robots-tag header', () => { }) }) }) + +describe('Object key names with Unicode characters', () => { + test('can upload, get, list, and delete', async () => { + const prefix = `test-utf8-${randomUUID()}` + const objectName = getUnicodeObjectName() + const authorization = `Bearer ${await serviceKeyAsync}` + + const form = new FormData() + form.append('file', fs.createReadStream(`./src/test/assets/sadcat.jpg`)) + const headers = Object.assign({}, form.getHeaders(), { + authorization, + 'x-upsert': 'true', + }) + + const uploadResponse = await appInstance.inject({ + method: 'POST', + url: `/object/bucket2/${prefix}/${encodeURIComponent(objectName)}`, + headers: { + ...headers, + ...form.getHeaders(), + }, + payload: form, + }) + expect(uploadResponse.statusCode).toBe(200) + + const getResponse = await appInstance.inject({ + method: 'GET', + url: `/object/bucket2/${prefix}/${encodeURIComponent(objectName)}`, + headers: { + authorization, + }, + }) + expect(getResponse.statusCode).toBe(200) + expect(getResponse.headers['etag']).toBe('abc') + expect(getResponse.headers['last-modified']).toBe('Thu, 12 Aug 2021 16:00:00 GMT') + expect(getResponse.headers['cache-control']).toBe('no-cache') + + const listResponse = await appInstance.inject({ + method: 'POST', + url: `/object/list/bucket2`, + headers: { + authorization, + }, + payload: { prefix }, + }) + expect(listResponse.statusCode).toBe(200) + const listResponseJSON = JSON.parse(listResponse.body) + expect(listResponseJSON).toHaveLength(1) + expect(listResponseJSON[0].name).toBe(objectName.split('/')[0]) + + const deleteResponse = await appInstance.inject({ + method: 'DELETE', + url: `/object/bucket2/${prefix}/${encodeURIComponent(objectName)}`, + headers: { + authorization, + }, + }) + expect(deleteResponse.statusCode).toBe(200) + }) + + test('can upload and get via binary upload with a Unicode key', async () => { + const objectName = `binary-${randomUUID()}-일이삼-🙂.jpg` + const authorization = `Bearer ${await serviceKeyAsync}` + const path = './src/test/assets/sadcat.jpg' + const { size } = fs.statSync(path) + + const uploadResponse = await appInstance.inject({ + method: 'PUT', + url: `/object/bucket2/${encodeURIComponent(objectName)}`, + headers: { + authorization, + 'Content-Length': size, + 'Content-Type': 'image/jpeg', + }, + payload: fs.createReadStream(path), + }) + expect(uploadResponse.statusCode).toBe(200) + + const getResponse = await appInstance.inject({ + method: 'GET', + url: `/object/bucket2/${encodeURIComponent(objectName)}`, + headers: { + authorization, + }, + }) + expect(getResponse.statusCode).toBe(200) + expect(getResponse.headers['etag']).toBe('abc') + }) + + test('can upload and read HEAD info with a Unicode and URL-reserved key', async () => { + const objectName = `head-${randomUUID()}-일이삼-🙂-q?foo=1&bar=%25+plus;semi:colon,#frag.jpg` + const authorization = `Bearer ${await serviceKeyAsync}` + const path = './src/test/assets/sadcat.jpg' + const { size } = fs.statSync(path) + + const uploadResponse = await appInstance.inject({ + method: 'PUT', + url: `/object/bucket2/${encodeURIComponent(objectName)}`, + headers: { + authorization, + 'Content-Length': size, + 'Content-Type': 'image/jpeg', + }, + payload: fs.createReadStream(path), + }) + expect(uploadResponse.statusCode).toBe(200) + + const headResponse = await appInstance.inject({ + method: 'HEAD', + url: `/object/authenticated/bucket2/${encodeURIComponent(objectName)}`, + headers: { + authorization, + }, + }) + expect(headResponse.statusCode).toBe(200) + expect(headResponse.headers['etag']).toBe('abc') + expect(headResponse.headers['last-modified']).toBeTruthy() + expect(headResponse.headers['content-length']).toBeTruthy() + expect(headResponse.headers['cache-control']).toBe('no-cache') + }) + + test('treats NFC and NFD object names as distinct keys', async () => { + const prefix = `normalize-${randomUUID()}` + const nfcObjectName = `${prefix}/caf\u00e9.txt` + const nfdObjectName = `${prefix}/cafe\u0301.txt` + const authorization = `Bearer ${await serviceKeyAsync}` + + for (const objectName of [nfcObjectName, nfdObjectName]) { + const form = new FormData() + form.append('file', fs.createReadStream(`./src/test/assets/sadcat.jpg`)) + const uploadResponse = await appInstance.inject({ + method: 'POST', + url: `/object/bucket2/${encodeURIComponent(objectName)}`, + headers: { + authorization, + ...form.getHeaders(), + }, + payload: form, + }) + expect(uploadResponse.statusCode).toBe(200) + } + + const db = await getSuperuserPostgrestClient() + const storedObjects = await db + .from('objects') + .select('name') + .where('bucket_id', 'bucket2') + .whereIn('name', [nfcObjectName, nfdObjectName]) + + const storedNames = storedObjects.map((entry) => entry.name) + expect(storedNames).toEqual(expect.arrayContaining([nfcObjectName, nfdObjectName])) + expect(new Set(storedNames).size).toBe(2) + + for (const objectName of [nfcObjectName, nfdObjectName]) { + const deleteResponse = await appInstance.inject({ + method: 'DELETE', + url: `/object/bucket2/${encodeURIComponent(objectName)}`, + headers: { + authorization, + }, + }) + expect(deleteResponse.statusCode).toBe(200) + } + }) + + test('should not upload if the name contains invalid characters', async () => { + const invalidObjectName = getInvalidObjectName() + const authorization = `Bearer ${await serviceKeyAsync}` + const form = new FormData() + form.append('file', fs.createReadStream(`./src/test/assets/sadcat.jpg`)) + const headers = Object.assign({}, form.getHeaders(), { + authorization, + 'x-upsert': 'true', + }) + const uploadResponse = await appInstance.inject({ + method: 'POST', + url: `/object/bucket2/${encodeURIComponent(invalidObjectName)}`, + headers: { + ...headers, + ...form.getHeaders(), + }, + payload: form, + }) + expect(uploadResponse.statusCode).toBe(400) + expect(S3Backend.prototype.uploadObject).not.toHaveBeenCalled() + expect(uploadResponse.body).toBe( + JSON.stringify({ + statusCode: '400', + error: 'InvalidKey', + message: `Invalid key: ${encodeURIComponent(invalidObjectName)}`, + }) + ) + }) + + test('can generate and use a signed download URL', async () => { + const objectName = `signed-download-${randomUUID()}-일이삼-🙂.png` + const authorization = `Bearer ${await serviceKeyAsync}` + + const form = new FormData() + form.append('file', fs.createReadStream(`./src/test/assets/sadcat.jpg`)) + const uploadHeaders = Object.assign({}, form.getHeaders(), { + authorization, + 'x-upsert': 'true', + }) + + const uploadResponse = await appInstance.inject({ + method: 'POST', + url: `/object/bucket2/${encodeURIComponent(objectName)}`, + headers: { + ...uploadHeaders, + }, + payload: form, + }) + expect(uploadResponse.statusCode).toBe(200) + + const signResponse = await appInstance.inject({ + method: 'POST', + url: `/object/sign/bucket2/${encodeURIComponent(objectName)}`, + headers: { + authorization, + }, + payload: { + expiresIn: 60, + }, + }) + expect(signResponse.statusCode).toBe(200) + + const signedURL = signResponse.json<{ signedURL: string }>().signedURL + expect(signedURL).toContain('?token=') + const token = signedURL.split('?token=').pop() + expect(token).toBeTruthy() + + const getResponse = await appInstance.inject({ + method: 'GET', + url: `/object/sign/bucket2/${encodeURIComponent(objectName)}?token=${token}`, + }) + expect(getResponse.statusCode).toBe(200) + expect(getResponse.headers['etag']).toBe('abc') + expect(getResponse.headers['last-modified']).toBe('Thu, 12 Aug 2021 16:00:00 GMT') + }) + + test('can generate and use a signed upload URL', async () => { + const objectName = `signed-upload-${randomUUID()}-일이삼-🙂.png` + const authorization = `Bearer ${await serviceKeyAsync}` + + const signedUploadResponse = await appInstance.inject({ + method: 'POST', + url: `/object/upload/sign/bucket2/${encodeURIComponent(objectName)}`, + headers: { + authorization, + }, + }) + expect(signedUploadResponse.statusCode).toBe(200) + + const token = signedUploadResponse.json<{ token: string }>().token + expect(token).toBeTruthy() + + const form = new FormData() + form.append('file', fs.createReadStream(`./src/test/assets/sadcat.jpg`)) + const uploadResponse = await appInstance.inject({ + method: 'PUT', + url: `/object/upload/sign/bucket2/${encodeURIComponent(objectName)}?token=${token}`, + headers: { + ...form.getHeaders(), + }, + payload: form, + }) + expect(uploadResponse.statusCode).toBe(200) + expect(S3Backend.prototype.uploadObject).toHaveBeenCalled() + + const db = await getSuperuserPostgrestClient() + const objectResponse = await db + .from('objects') + .select('*') + .where({ + name: objectName, + bucket_id: 'bucket2', + }) + .first() + expect(objectResponse?.name).toBe(objectName) + }) + + test('can generate and use a signed download URL with Unicode and URL-reserved characters', async () => { + const objectName = `signed-download-reserved-${randomUUID()}-일이삼-🙂-q?foo=1&bar=%25+plus;semi:colon,#frag.png` + const authorization = `Bearer ${await serviceKeyAsync}` + + const form = new FormData() + form.append('file', fs.createReadStream(`./src/test/assets/sadcat.jpg`)) + const uploadHeaders = Object.assign({}, form.getHeaders(), { + authorization, + 'x-upsert': 'true', + }) + + const uploadResponse = await appInstance.inject({ + method: 'POST', + url: `/object/bucket2/${encodeURIComponent(objectName)}`, + headers: uploadHeaders, + payload: form, + }) + expect(uploadResponse.statusCode).toBe(200) + + const signResponse = await appInstance.inject({ + method: 'POST', + url: `/object/sign/bucket2/${encodeURIComponent(objectName)}`, + headers: { + authorization, + }, + payload: { + expiresIn: 60, + }, + }) + expect(signResponse.statusCode).toBe(200) + + const signedURL = signResponse.json<{ signedURL: string }>().signedURL + const signedURLParsed = new URL(signedURL, 'http://localhost') + const token = signedURLParsed.searchParams.get('token') + expect(token).toBeTruthy() + + const getResponse = await appInstance.inject({ + method: 'GET', + url: `${signedURLParsed.pathname}${signedURLParsed.search}`, + }) + expect(getResponse.statusCode).toBe(200) + expect(getResponse.headers['etag']).toBe('abc') + }) + + test('can generate and use a signed upload URL with Unicode and URL-reserved characters', async () => { + const objectName = `signed-upload-reserved-${randomUUID()}-éè-中文-🙂-q?foo=1&bar=%25+plus;semi:colon,#frag.png` + const authorization = `Bearer ${await serviceKeyAsync}` + + const signedUploadResponse = await appInstance.inject({ + method: 'POST', + url: `/object/upload/sign/bucket2/${encodeURIComponent(objectName)}`, + headers: { + authorization, + }, + }) + expect(signedUploadResponse.statusCode).toBe(200) + + const signedUpload = signedUploadResponse.json<{ token: string; url: string }>() + const token = signedUpload.token + expect(token).toBeTruthy() + const signedUploadURL = new URL(signedUpload.url, 'http://localhost') + expect(signedUploadURL.searchParams.get('token')).toBe(token) + + const form = new FormData() + form.append('file', fs.createReadStream(`./src/test/assets/sadcat.jpg`)) + const uploadResponse = await appInstance.inject({ + method: 'PUT', + url: `${signedUploadURL.pathname}${signedUploadURL.search}`, + headers: { + ...form.getHeaders(), + }, + payload: form, + }) + expect(uploadResponse.statusCode).toBe(200) + + const db = await getSuperuserPostgrestClient() + const objectResponse = await db + .from('objects') + .select('*') + .where({ + name: objectName, + bucket_id: 'bucket2', + }) + .first() + expect(objectResponse?.name).toBe(objectName) + }) + + test('rejects double-encoded signed upload paths', async () => { + const objectName = `signed-upload-double-${randomUUID()}-éè-中文-🙂.png` + const authorization = `Bearer ${await serviceKeyAsync}` + + const signedUploadResponse = await appInstance.inject({ + method: 'POST', + url: `/object/upload/sign/bucket2/${encodeURIComponent(objectName)}`, + headers: { + authorization, + }, + }) + expect(signedUploadResponse.statusCode).toBe(200) + + const signedUpload = signedUploadResponse.json<{ url: string }>() + const signedUploadURL = new URL(signedUpload.url, 'http://localhost') + const doubleEncodedPath = signedUploadURL.pathname.replaceAll('%', '%25') + + const form = new FormData() + form.append('file', fs.createReadStream(`./src/test/assets/sadcat.jpg`)) + const uploadResponse = await appInstance.inject({ + method: 'PUT', + url: `${doubleEncodedPath}${signedUploadURL.search}`, + headers: { + ...form.getHeaders(), + }, + payload: form, + }) + + expect(uploadResponse.statusCode).toBe(400) + expect(uploadResponse.json<{ error: string }>().error).toBe('InvalidSignature') + }) + + test('can sign and upload using returned signed upload URL for nested Unicode and URL-reserved object names', async () => { + const objectName = `signed-upload-unicode-${randomUUID()}-폴더?x=1&y=%25+plus;semi:colon,/子目录#frag/파일-🙂.png` + const authorization = `Bearer ${await serviceKeyAsync}` + + const signedUploadResponse = await appInstance.inject({ + method: 'POST', + url: `/object/upload/sign/bucket2/${encodeURIComponent(objectName)}`, + headers: { + authorization, + }, + }) + expect(signedUploadResponse.statusCode).toBe(200) + + const signedUpload = signedUploadResponse.json<{ token: string; url: string }>() + expect(signedUpload.token).toBeTruthy() + const signedUploadURL = new URL(signedUpload.url, 'http://localhost') + expect(signedUploadURL.searchParams.get('token')).toBe(signedUpload.token) + + const form = new FormData() + form.append('file', fs.createReadStream(`./src/test/assets/sadcat.jpg`)) + const uploadResponse = await appInstance.inject({ + method: 'PUT', + url: `${signedUploadURL.pathname}${signedUploadURL.search}`, + headers: { + ...form.getHeaders(), + }, + payload: form, + }) + expect(uploadResponse.statusCode).toBe(200) + + const db = await getSuperuserPostgrestClient() + const objectResponse = await db + .from('objects') + .select('*') + .where({ + name: objectName, + bucket_id: 'bucket2', + }) + .first() + expect(objectResponse?.name).toBe(objectName) + + const deleteResponse = await appInstance.inject({ + method: 'DELETE', + url: `/object/bucket2/${encodeURIComponent(objectName)}`, + headers: { + authorization, + }, + }) + expect(deleteResponse.statusCode).toBe(200) + }) + + test('should not generate signed upload URL for invalid key', async () => { + const invalidObjectName = getInvalidObjectName() + const authorization = `Bearer ${await serviceKeyAsync}` + + const signedUploadResponse = await appInstance.inject({ + method: 'POST', + url: `/object/upload/sign/bucket2/${encodeURIComponent(invalidObjectName)}`, + headers: { + authorization, + }, + }) + expect(signedUploadResponse.statusCode).toBe(400) + expect(signedUploadResponse.body).toContain('Invalid key') + }) +}) diff --git a/src/test/render-routes.test.ts b/src/test/render-routes.test.ts index 408226b9..3a89dbe5 100644 --- a/src/test/render-routes.test.ts +++ b/src/test/render-routes.test.ts @@ -1,3 +1,4 @@ +import { randomUUID } from 'node:crypto' import { generateHS512JWK, SignedToken, signJWT, verifyJWT } from '@internal/auth' import axios from 'axios' import dotenv from 'dotenv' @@ -194,4 +195,53 @@ describe('image rendering routes', () => { const body = response.json<{ error: string }>() expect(body.error).toBe('InvalidSignature') }) + + it('will reject double-encoded signed render paths', async () => { + const objectName = `authenticated/render-double-${randomUUID()}-일이삼.png` + const encodedObjectName = objectName + .split('/') + .map((segment) => encodeURIComponent(segment)) + .join('/') + + const uploadResponse = await appInstance.inject({ + method: 'POST', + url: `/object/bucket2/${encodedObjectName}`, + payload: Buffer.from('render double-encoded test'), + headers: { + authorization: `Bearer ${process.env.SERVICE_KEY}`, + 'content-type': 'image/png', + 'x-upsert': 'true', + }, + }) + expect(uploadResponse.statusCode).toBe(200) + + const signURLResponse = await appInstance.inject({ + method: 'POST', + url: `/object/sign/bucket2/${encodedObjectName}`, + payload: { + expiresIn: 60000, + transform: { + width: 100, + height: 100, + resize: 'contain', + }, + }, + headers: { + authorization: `Bearer ${process.env.SERVICE_KEY}`, + }, + }) + expect(signURLResponse.statusCode).toBe(200) + + const signedURL = signURLResponse.json<{ signedURL: string }>().signedURL + const signedURLParsed = new URL(signedURL, 'http://localhost') + const doubleEncodedPath = signedURLParsed.pathname.replaceAll('%', '%25') + const doubleEncodedResponse = await appInstance.inject({ + method: 'GET', + url: `${doubleEncodedPath}${signedURLParsed.search}`, + }) + + expect(doubleEncodedResponse.statusCode).toBe(400) + const body = doubleEncodedResponse.json<{ error: string }>() + expect(body.error).toBe('InvalidSignature') + }) }) diff --git a/src/test/s3-adapter.test.ts b/src/test/s3-adapter.test.ts index 0d5288e5..fdb1f276 100644 --- a/src/test/s3-adapter.test.ts +++ b/src/test/s3-adapter.test.ts @@ -1,8 +1,9 @@ 'use strict' -import { S3Client } from '@aws-sdk/client-s3' +import { CopyObjectCommand, S3Client, UploadPartCopyCommand } from '@aws-sdk/client-s3' import { Readable } from 'stream' import { S3Backend } from '../storage/backend/s3/adapter' +import { encodeBucketAndObjectPathForTest } from './utils/path-encoding' jest.mock('@aws-sdk/client-s3', () => { const originalModule = jest.requireActual('@aws-sdk/client-s3') @@ -74,4 +75,83 @@ describe('S3Backend', () => { expect(result.metadata.mimetype).toBe('image/png') }) }) + + describe('copyObject', () => { + test('should preserve "/" and encode path tokens in CopySource', async () => { + const lastModified = new Date('2024-01-01T00:00:00.000Z') + mockSend.mockResolvedValue({ + CopyObjectResult: { + ETag: '"copy-etag"', + LastModified: lastModified, + }, + $metadata: { + httpStatusCode: 200, + }, + }) + + const backend = new S3Backend({ + region: 'us-east-1', + endpoint: 'http://localhost:9000', + }) + + const sourceKey = 'source path/일이삼-🙂?#%.jpg' + const destinationKey = 'dest/path/copied-🙂.jpg' + + await backend.copyObject('test-bucket', sourceKey, undefined, destinationKey, undefined) + + expect(mockSend).toHaveBeenCalledTimes(1) + const command = mockSend.mock.calls[0][0] as CopyObjectCommand + expect(command).toBeInstanceOf(CopyObjectCommand) + expect(command.input.CopySource).toBe( + encodeBucketAndObjectPathForTest('test-bucket', sourceKey) + ) + expect(command.input.CopySource).toContain('test-bucket/source%20path/') + expect(command.input.CopySource).not.toContain('test-bucket%2F') + }) + }) + + describe('uploadPartCopy', () => { + test('should preserve "/" and encode path tokens in CopySource for unicode source keys', async () => { + const lastModified = new Date('2024-01-01T00:00:00.000Z') + mockSend.mockResolvedValue({ + CopyPartResult: { + ETag: '"copy-etag"', + LastModified: lastModified, + }, + }) + + const backend = new S3Backend({ + region: 'us-east-1', + endpoint: 'http://localhost:9000', + }) + + const sourceKey = 'source path/folder/일이삼-🙂?#%.jpg' + const destinationKey = 'dest/path/copied-🙂.jpg' + + const result = await backend.uploadPartCopy( + 'test-bucket', + destinationKey, + '', + 'upload-id', + 1, + sourceKey, + undefined, + { fromByte: 0, toByte: 1024 } + ) + + expect(mockSend).toHaveBeenCalledTimes(1) + const command = mockSend.mock.calls[0][0] as UploadPartCopyCommand + expect(command).toBeInstanceOf(UploadPartCopyCommand) + expect(command.input.CopySource).toBe( + encodeBucketAndObjectPathForTest('test-bucket', sourceKey) + ) + expect(command.input.CopySource).toContain('test-bucket/source%20path/folder/') + expect(command.input.CopySource).not.toContain('test-bucket%2F') + expect(command.input.CopySourceRange).toBe('bytes=0-1024') + expect(result).toEqual({ + eTag: '"copy-etag"', + lastModified, + }) + }) + }) }) diff --git a/src/test/s3-backup.test.ts b/src/test/s3-backup.test.ts new file mode 100644 index 00000000..24b15f16 --- /dev/null +++ b/src/test/s3-backup.test.ts @@ -0,0 +1,113 @@ +'use strict' + +import { + CompleteMultipartUploadCommand, + CopyObjectCommand, + CreateMultipartUploadCommand, + S3Client, + UploadPartCopyCommand, +} from '@aws-sdk/client-s3' +import { ObjectBackup } from '../storage/backend/s3/backup' +import { encodeBucketAndObjectPathForTest } from './utils/path-encoding' + +jest.mock('@aws-sdk/client-s3', () => { + const originalModule = jest.requireActual('@aws-sdk/client-s3') + return { + ...originalModule, + S3Client: jest.fn().mockImplementation(() => ({ + send: jest.fn(), + })), + } +}) + +describe('ObjectBackup', () => { + let mockSend: jest.Mock + let client: S3Client + + beforeEach(() => { + jest.clearAllMocks() + mockSend = jest.fn() + ;(S3Client as jest.Mock).mockImplementation(() => ({ + send: mockSend, + })) + client = new S3Client({}) as unknown as S3Client + }) + + test('singleCopy preserves path separators for Unicode source keys', async () => { + mockSend.mockResolvedValue({}) + + const sourceKey = 'folder one/일이삼/子目录/🙂?#%.png' + const destinationKey = 'backup/folder/복사본.png' + + const backup = new ObjectBackup(client, { + sourceBucket: 'source-bucket', + sourceKey, + destinationBucket: 'backup-bucket', + destinationKey, + size: 1024, + }) + + await backup.backup() + + expect(mockSend).toHaveBeenCalledTimes(1) + const command = mockSend.mock.calls[0][0] as CopyObjectCommand + expect(command).toBeInstanceOf(CopyObjectCommand) + expect(command.input.CopySource).toBe( + encodeBucketAndObjectPathForTest('source-bucket', sourceKey) + ) + expect(command.input.CopySource).toContain('source-bucket/folder%20one/') + expect(command.input.CopySource).not.toContain('%2Fsource-bucket%2F') + expect(command.input.CopySource).not.toContain('source-bucket%2F') + }) + + test('multipartCopy preserves path separators for Unicode source keys', async () => { + mockSend.mockImplementation((command: unknown) => { + if (command instanceof CreateMultipartUploadCommand) { + return Promise.resolve({ UploadId: 'upload-id' }) + } + + if (command instanceof UploadPartCopyCommand) { + return Promise.resolve({ + CopyPartResult: { + ETag: `"etag-${command.input.PartNumber}"`, + }, + }) + } + + if (command instanceof CompleteMultipartUploadCommand) { + return Promise.resolve({}) + } + + return Promise.resolve({}) + }) + + const sourceKey = 'folder one/일이삼/子目录/🙂?#%.png' + const destinationKey = 'backup/folder/복사본.png' + const partSize = 5 * 1024 * 1024 * 1024 + const backup = new ObjectBackup(client, { + sourceBucket: 'source-bucket', + sourceKey, + destinationBucket: 'backup-bucket', + destinationKey, + size: partSize + 1024, + }) + + await backup.backup() + + const uploadPartCommands = mockSend.mock.calls + .map(([command]) => command) + .filter( + (command): command is UploadPartCopyCommand => command instanceof UploadPartCopyCommand + ) + + expect(uploadPartCommands).toHaveLength(2) + for (const command of uploadPartCommands) { + expect(command.input.CopySource).toBe( + encodeBucketAndObjectPathForTest('source-bucket', sourceKey) + ) + expect(command.input.CopySource).toContain('source-bucket/folder%20one/') + expect(command.input.CopySource).not.toContain('%2Fsource-bucket%2F') + expect(command.input.CopySource).not.toContain('source-bucket%2F') + } + }) +}) diff --git a/src/test/s3-copy-source-parser.test.ts b/src/test/s3-copy-source-parser.test.ts new file mode 100644 index 00000000..3adbea07 --- /dev/null +++ b/src/test/s3-copy-source-parser.test.ts @@ -0,0 +1,60 @@ +'use strict' + +import { ErrorCode, StorageBackendError } from '@internal/errors' +import { parseCopySource } from '../storage/protocols/s3/copy-source-parser' + +describe('parseCopySource', () => { + test('preserves raw question marks and hashes in partially encoded keys', () => { + const objectKey = 'folder/일이삼?x=1#frag.png' + const result = parseCopySource(`bucket/${encodeURI(objectKey)}`) + + expect(result).toEqual({ + bucketName: 'bucket', + objectKey, + sourceVersion: undefined, + }) + }) + + test('preserves question marks in versionId query values', () => { + const result = parseCopySource( + 'bucket/folder%20one/%EC%9D%BC%EC%9D%B4%EC%82%BC.png?versionId=v1?part=2' + ) + + expect(result).toEqual({ + bucketName: 'bucket', + objectKey: 'folder one/일이삼.png', + sourceVersion: 'v1?part=2', + }) + }) + + test('accepts fully URL-encoded CopySource values with versionId', () => { + const result = parseCopySource( + `${encodeURIComponent('bucket/folder one/일이삼/🙂?#%.png')}?versionId=ver-123` + ) + + expect(result).toEqual({ + bucketName: 'bucket', + objectKey: 'folder one/일이삼/🙂?#%.png', + sourceVersion: 'ver-123', + }) + }) + + test('accepts fully URL-encoded leading-slash CopySource values', () => { + const result = parseCopySource(encodeURIComponent('/bucket/folder one/일이삼/🙂?#%.png')) + + expect(result).toEqual({ + bucketName: 'bucket', + objectKey: 'folder one/일이삼/🙂?#%.png', + sourceVersion: undefined, + }) + }) + + test('rejects an empty versionId query value', () => { + expect(() => parseCopySource('bucket/key?versionId=')).toThrow( + expect.objectContaining>({ + code: ErrorCode.MissingParameter, + message: 'Invalid Parameter CopySource', + }) + ) + }) +}) diff --git a/src/test/s3-protocol.test.ts b/src/test/s3-protocol.test.ts index dcb31196..b1f97654 100644 --- a/src/test/s3-protocol.test.ts +++ b/src/test/s3-protocol.test.ts @@ -28,14 +28,16 @@ import { Upload } from '@aws-sdk/lib-storage' import { createPresignedPost } from '@aws-sdk/s3-presigned-post' import { getSignedUrl } from '@aws-sdk/s3-request-presigner' import { wait } from '@internal/concurrency' +import { getPostgresConnection, getServiceKeyUser } from '@internal/database' import axios from 'axios' import { randomUUID } from 'crypto' import { FastifyInstance } from 'fastify' import { ReadableStreamBuffer } from 'stream-buffers' import app from '../app' import { getConfig, mergeConfig } from '../config' +import { getInvalidObjectName, getUnicodeObjectName } from './common' -const { s3ProtocolAccessKeySecret, s3ProtocolAccessKeyId, storageS3Region } = getConfig() +const { s3ProtocolAccessKeySecret, s3ProtocolAccessKeyId, storageS3Region, tenantId } = getConfig() async function createBucket(client: S3Client, name?: string, publicRead = true) { let bucketName: string @@ -76,6 +78,37 @@ async function uploadFile( return await uploader.done() } +async function getObjectVersion(bucketId: string, objectName: string): Promise { + const superUser = await getServiceKeyUser(tenantId) + const connection = await getPostgresConnection({ + superUser, + user: superUser, + tenantId, + host: 'localhost', + }) + const tx = await connection.transaction() + + try { + const object = (await tx + .from('objects') + .select('version') + .where({ + bucket_id: bucketId, + name: objectName, + }) + .first()) as { version?: string } | undefined + + if (!object?.version) { + throw new Error(`Object version not found for ${bucketId}/${objectName}`) + } + + return object.version + } finally { + await tx.rollback() + await connection.dispose() + } +} + jest.setTimeout(10 * 1000) describe('S3 Protocol', () => { @@ -263,6 +296,54 @@ describe('S3 Protocol', () => { expect(resp2.CommonPrefixes?.length).toBe(2) expect(resp2.Contents?.length).toBe(1) }) + + it('paginates v1 listings with Unicode keys across pages', async () => { + const bucket = await createBucket(client) + const keys = [ + `v1-unicode-${randomUUID()}-éè-中文-🙂.jpg`, + `v1-unicode-${randomUUID()}-일이삼-🙂.jpg`, + `v1-unicode-${randomUUID()}-폴더-子目录-🙂.jpg`, + ].sort() + + await Promise.all(keys.map((key) => uploadFile(client, bucket, key, 1))) + + const page1 = await client.send( + new ListObjectsCommand({ + Bucket: bucket, + MaxKeys: 1, + }) + ) + expect(page1.Contents?.length).toBe(1) + expect(page1.Marker).toBeTruthy() + + const page2 = await client.send( + new ListObjectsCommand({ + Bucket: bucket, + MaxKeys: 1, + Marker: page1.Marker, + }) + ) + expect(page2.Contents?.length).toBe(1) + expect(page2.Marker).toBeTruthy() + + const page3 = await client.send( + new ListObjectsCommand({ + Bucket: bucket, + MaxKeys: 1, + Marker: page2.Marker, + }) + ) + expect(page3.Contents?.length).toBe(1) + + const pagedKeys = [ + page1.Contents?.[0]?.Key, + page2.Contents?.[0]?.Key, + page3.Contents?.[0]?.Key, + ].filter(Boolean) + expect(pagedKeys).toHaveLength(3) + expect(new Set(pagedKeys).size).toBe(3) + expect(pagedKeys).toEqual(keys) + }) }) describe('ListObjectsV2Command', () => { @@ -352,6 +433,25 @@ describe('S3 Protocol', () => { expect(resp.KeyCount).toBe(3) }) + it('preserves delimiter slashes in CommonPrefixes when EncodingType=url', async () => { + const bucket = await createBucket(client) + await Promise.all([ + uploadFile(client, bucket, '日本語/子目录/test-1.jpg', 1), + uploadFile(client, bucket, 'plain-root.jpg', 1), + ]) + + const resp = await client.send( + new ListObjectsV2Command({ + Bucket: bucket, + Delimiter: '/', + EncodingType: 'url', + }) + ) + + expect(resp.CommonPrefixes?.[0].Prefix).toBe(`${encodeURIComponent('日本語')}/`) + expect(resp.CommonPrefixes?.[0].Prefix).not.toContain('%2F') + }) + it('paginate keys and common prefixes', async () => { const bucket = await createBucket(client) const listBucketsPage1 = new ListObjectsV2Command({ @@ -526,6 +626,41 @@ describe('S3 Protocol', () => { expect(resp.status).toBe(200) }) + it('can upload using multipart/form-data with a Unicode key', async () => { + const bucketName = await createBucket(client) + const key = `post-form-${randomUUID()}-中文-일이삼-🙂.jpg` + const signedURL = await createPresignedPost(client, { + Bucket: bucketName, + Key: key, + Expires: 5000, + Fields: { + 'Content-Type': 'image/jpg', + }, + }) + + const formData = new FormData() + Object.keys(signedURL.fields).forEach((fieldName) => { + formData.set(fieldName, signedURL.fields[fieldName]) + }) + + const data = Buffer.alloc(1024) + formData.set('file', new Blob([data]), 'upload.jpg') + + const resp = await axios.post(signedURL.url, formData, { + validateStatus: () => true, + }) + expect(resp.status).toBe(200) + + const listResp = await client.send( + new ListObjectsV2Command({ + Bucket: bucketName, + }) + ) + expect((listResp.Contents || []).map((item) => item.Key)).toEqual( + expect.arrayContaining([key]) + ) + }) + it('prevent uploading files larger than the maxFileSize limit', async () => { mergeConfig({ uploadFileSizeLimit: 1024, @@ -1140,6 +1275,61 @@ describe('S3 Protocol', () => { expect(headObj.CacheControl).toBe('max-age=2009') }) + it('will copy an object when CopySource includes versionId', async () => { + const bucketName = await createBucket(client) + const sourceKey = `test-copy-versioned-${randomUUID()}.jpg` + await uploadFile(client, bucketName, sourceKey, 1) + const sourceVersion = await getObjectVersion(bucketName, sourceKey) + + const copyObjectCommand = new CopyObjectCommand({ + Bucket: bucketName, + Key: `test-copied-versioned-${randomUUID()}.jpg`, + CopySource: `${bucketName}/${sourceKey}?versionId=${sourceVersion}`, + }) + + const resp = await client.send(copyObjectCommand) + expect(resp.CopyObjectResult?.ETag).toBeTruthy() + }) + + it('will copy an object when CopySource is fully URL-encoded and includes versionId', async () => { + const bucketName = await createBucket(client) + const sourceKey = getUnicodeObjectName() + const destinationKey = `test-copied-versioned-encoded-${randomUUID()}.jpg` + + await uploadFile(client, bucketName, sourceKey, 1) + const sourceVersion = await getObjectVersion(bucketName, sourceKey) + + const copyObjectCommand = new CopyObjectCommand({ + Bucket: bucketName, + Key: destinationKey, + CopySource: `${encodeURIComponent(`${bucketName}/${sourceKey}`)}?versionId=${sourceVersion}`, + }) + + const resp = await client.send(copyObjectCommand) + expect(resp.CopyObjectResult?.ETag).toBeTruthy() + }) + + it('will not copy an object when CopySource versionId does not match', async () => { + const bucketName = await createBucket(client) + const sourceKey = `test-copy-versioned-${randomUUID()}.jpg` + await uploadFile(client, bucketName, sourceKey, 1) + + const copyObjectCommand = new CopyObjectCommand({ + Bucket: bucketName, + Key: `test-copied-versioned-${randomUUID()}.jpg`, + CopySource: `${bucketName}/${sourceKey}?versionId=${randomUUID()}`, + }) + + try { + await client.send(copyObjectCommand) + throw new Error('Should not reach here') + } catch (e) { + expect((e as Error).message).not.toEqual('Should not reach here') + expect((e as S3ServiceException).$metadata.httpStatusCode).toEqual(404) + expect((e as S3ServiceException).message).toEqual('Object not found') + } + }) + it('will not be able to copy an object that doesnt exist', async () => { const bucketName1 = await createBucket(client) await uploadFile(client, bucketName1, 'test-copy-1.jpg', 1) @@ -1159,6 +1349,44 @@ describe('S3 Protocol', () => { expect((e as S3ServiceException).message).toEqual('Object not found') } }) + + it('will reject malformed url-encoded CopySource', async () => { + const bucketName = await createBucket(client) + + const copyObjectCommand = new CopyObjectCommand({ + Bucket: bucketName, + Key: 'test-copied-malformed.jpg', + CopySource: `${bucketName}/test-copy-%ZZ.jpg`, + }) + + try { + await client.send(copyObjectCommand) + throw new Error('Should not reach here') + } catch (e) { + expect((e as Error).message).not.toEqual('Should not reach here') + expect((e as S3ServiceException).$metadata.httpStatusCode).toEqual(400) + expect((e as S3ServiceException).message).toEqual('Invalid Parameter CopySource') + } + }) + + it('will reject CopySource without an object key', async () => { + const bucketName = await createBucket(client) + + const copyObjectCommand = new CopyObjectCommand({ + Bucket: bucketName, + Key: 'test-copied-missing-source-key.jpg', + CopySource: bucketName, + }) + + try { + await client.send(copyObjectCommand) + throw new Error('Should not reach here') + } catch (e) { + expect((e as Error).message).not.toEqual('Should not reach here') + expect((e as S3ServiceException).$metadata.httpStatusCode).toEqual(400) + expect((e as S3ServiceException).message).toEqual('Missing Required Parameter CopySource') + } + }) }) describe('ListMultipartUploads', () => { @@ -1222,6 +1450,32 @@ describe('S3 Protocol', () => { expect(resp.CommonPrefixes?.[0].Prefix).toBe('nested/') }) + it('preserves delimiter slashes in multipart CommonPrefixes when EncodingType=url', async () => { + const bucketName = await createBucket(client) + const createMultiPartUpload = (key: string) => + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: key, + ContentType: 'image/jpg', + }) + + await Promise.all([ + client.send(createMultiPartUpload('日本語/子目录/test-4.jpg')), + client.send(createMultiPartUpload('root-file.jpg')), + ]) + + const resp = await client.send( + new ListMultipartUploadsCommand({ + Bucket: bucketName, + Delimiter: '/', + EncodingType: 'url', + }) + ) + + expect(resp.CommonPrefixes?.[0].Prefix).toBe(`${encodeURIComponent('日本語')}/`) + expect(resp.CommonPrefixes?.[0].Prefix).not.toContain('%2F') + }) + it('treats % as a literal character in multipart prefix filtering with delimiter', async () => { const bucketName = await createBucket(client) const createMultiPartUpload = (key: string) => @@ -1505,87 +1759,286 @@ describe('S3 Protocol', () => { const parts = await client.send(listPartsCmd) expect(parts.Parts?.length).toBe(1) }) - }) - describe('S3 Presigned URL', () => { - it('can call a simple method with presigned url', async () => { + it('will copy a part when CopySource includes versionId', async () => { const bucket = await createBucket(client) - const bucketVersioningCommand = new GetBucketVersioningCommand({ + const sourceKey = `${randomUUID()}.jpg` + const newKey = `new-${randomUUID()}.jpg` + + await uploadFile(client, bucket, sourceKey, 12) + const sourceVersion = await getObjectVersion(bucket, sourceKey) + + const createMultiPartUpload = new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: newKey, + ContentType: 'image/jpg', + CacheControl: 'max-age=2000', + }) + const resp = await client.send(createMultiPartUpload) + expect(resp.UploadId).toBeTruthy() + + const copyPart = new UploadPartCopyCommand({ Bucket: bucket, + Key: newKey, + UploadId: resp.UploadId, + PartNumber: 1, + CopySource: `${bucket}/${sourceKey}?versionId=${sourceVersion}`, + CopySourceRange: `bytes=0-${1024 * 4}`, }) - const signedUrl = await getSignedUrl(client, bucketVersioningCommand, { expiresIn: 100 }) - const resp = await fetch(signedUrl) - expect(resp.ok).toBeTruthy() - }) + const copyResp = await client.send(copyPart) + expect(copyResp.CopyPartResult?.ETag).toBeTruthy() - it('cannot request a presigned url if expired', async () => { - const bucket = await createBucket(client) - const bucketVersioningCommand = new GetBucketVersioningCommand({ + const listPartsCmd = new ListPartsCommand({ Bucket: bucket, + Key: newKey, + UploadId: resp.UploadId, }) - const signedUrl = await getSignedUrl(client, bucketVersioningCommand, { expiresIn: 1 }) - await wait(1500) - const resp = await fetch(signedUrl) - expect(resp.ok).toBeFalsy() - expect(resp.status).toBe(400) + const parts = await client.send(listPartsCmd) + expect(parts.Parts?.length).toBe(1) }) - it('doesnt crash when invalid headers returned', async () => { + it('will copy a part when CopySource is fully URL-encoded and includes versionId', async () => { const bucket = await createBucket(client) - const key = 'test-1.jpg' - await uploadFile(client, bucket, key, 2, { - 'invalid-header-r': Buffer.alloc(1024 * 9, 'a').toString(), - }) + const sourceKey = getUnicodeObjectName() + const newKey = `new-versioned-encoded-${randomUUID()}.jpg` - const headObj = new HeadObjectCommand({ + await uploadFile(client, bucket, sourceKey, 12) + const sourceVersion = await getObjectVersion(bucket, sourceKey) + + const createMultiPartUpload = new CreateMultipartUploadCommand({ Bucket: bucket, - Key: key, + Key: newKey, + ContentType: 'image/jpg', + CacheControl: 'max-age=2000', }) + const resp = await client.send(createMultiPartUpload) + expect(resp.UploadId).toBeTruthy() - const r = await client.send(headObj) - - expect(r.$metadata.httpStatusCode).toBe(200) - expect(r.MissingMeta).toBe(1) - }) - - it('can upload with presigned URL', async () => { - const bucket = await createBucket(client) - const key = 'test-1.jpg' - const body = Buffer.alloc(1024 * 2) + const copyPart = new UploadPartCopyCommand({ + Bucket: bucket, + Key: newKey, + UploadId: resp.UploadId, + PartNumber: 1, + CopySource: `${encodeURIComponent(`${bucket}/${sourceKey}`)}?versionId=${sourceVersion}`, + CopySourceRange: `bytes=0-${1024 * 4}`, + }) - const uploadUrl = await getSignedUrl( - client, - new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - }), - { expiresIn: 100 } - ) + const copyResp = await client.send(copyPart) + expect(copyResp.CopyPartResult?.ETag).toBeTruthy() - const resp = await fetch(uploadUrl, { - method: 'PUT', - body, - headers: { - 'Content-Length': body.length.toString(), - }, + const listPartsCmd = new ListPartsCommand({ + Bucket: bucket, + Key: newKey, + UploadId: resp.UploadId, }) - expect(resp.ok).toBeTruthy() + const parts = await client.send(listPartsCmd) + expect(parts.Parts?.length).toBe(1) }) - it('can fetch an asset via presigned URL', async () => { + it('will not copy a part when CopySource versionId does not match', async () => { const bucket = await createBucket(client) - const key = 'test-1.jpg' + const sourceKey = `${randomUUID()}.jpg` + const newKey = `new-${randomUUID()}.jpg` - await uploadFile(client, bucket, key, 2) + await uploadFile(client, bucket, sourceKey, 12) - const getUrl = await getSignedUrl( - client, - new GetObjectCommand({ - Bucket: bucket, + const createMultiPartUpload = new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: newKey, + ContentType: 'image/jpg', + CacheControl: 'max-age=2000', + }) + const resp = await client.send(createMultiPartUpload) + expect(resp.UploadId).toBeTruthy() + + const copyPart = new UploadPartCopyCommand({ + Bucket: bucket, + Key: newKey, + UploadId: resp.UploadId, + PartNumber: 1, + CopySource: `${bucket}/${sourceKey}?versionId=${randomUUID()}`, + CopySourceRange: `bytes=0-${1024 * 4}`, + }) + + try { + await client.send(copyPart) + throw new Error('Should not reach here') + } catch (e) { + expect((e as Error).message).not.toEqual('Should not reach here') + expect((e as S3ServiceException).$metadata.httpStatusCode).toEqual(404) + expect((e as S3ServiceException).message).toEqual('Object not found') + } + }) + + it('will reject malformed url-encoded CopySource', async () => { + const bucket = await createBucket(client) + const newKey = `new-${randomUUID()}.jpg` + + const createMultiPartUpload = new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: newKey, + ContentType: 'image/jpg', + CacheControl: 'max-age=2000', + }) + const resp = await client.send(createMultiPartUpload) + expect(resp.UploadId).toBeTruthy() + const uploadId = resp.UploadId + + const copyPart = new UploadPartCopyCommand({ + Bucket: bucket, + Key: newKey, + UploadId: uploadId, + PartNumber: 1, + CopySource: `${bucket}/test-copy-%ZZ.jpg`, + CopySourceRange: `bytes=0-${1024 * 4}`, + }) + + try { + await client.send(copyPart) + throw new Error('Should not reach here') + } catch (e) { + expect((e as Error).message).not.toEqual('Should not reach here') + expect((e as S3ServiceException).$metadata.httpStatusCode).toEqual(400) + expect((e as S3ServiceException).message).toEqual('Invalid Parameter CopySource') + } finally { + if (uploadId) { + await client.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: newKey, + UploadId: uploadId, + }) + ) + } + } + }) + + it('will reject CopySource without an object key', async () => { + const bucket = await createBucket(client) + const newKey = `new-${randomUUID()}.jpg` + + const createMultiPartUpload = new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: newKey, + ContentType: 'image/jpg', + CacheControl: 'max-age=2000', + }) + const resp = await client.send(createMultiPartUpload) + expect(resp.UploadId).toBeTruthy() + const uploadId = resp.UploadId + + const copyPart = new UploadPartCopyCommand({ + Bucket: bucket, + Key: newKey, + UploadId: uploadId, + PartNumber: 1, + CopySource: bucket, + CopySourceRange: `bytes=0-${1024 * 4}`, + }) + + try { + await client.send(copyPart) + throw new Error('Should not reach here') + } catch (e) { + expect((e as Error).message).not.toEqual('Should not reach here') + expect((e as S3ServiceException).$metadata.httpStatusCode).toEqual(400) + expect((e as S3ServiceException).message).toEqual('Missing Required Parameter CopySource') + } finally { + if (uploadId) { + await client.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: newKey, + UploadId: uploadId, + }) + ) + } + } + }) + }) + + describe('S3 Presigned URL', () => { + it('can call a simple method with presigned url', async () => { + const bucket = await createBucket(client) + const bucketVersioningCommand = new GetBucketVersioningCommand({ + Bucket: bucket, + }) + const signedUrl = await getSignedUrl(client, bucketVersioningCommand, { expiresIn: 100 }) + const resp = await fetch(signedUrl) + + expect(resp.ok).toBeTruthy() + }) + + it('cannot request a presigned url if expired', async () => { + const bucket = await createBucket(client) + const bucketVersioningCommand = new GetBucketVersioningCommand({ + Bucket: bucket, + }) + const signedUrl = await getSignedUrl(client, bucketVersioningCommand, { expiresIn: 1 }) + await wait(1500) + const resp = await fetch(signedUrl) + + expect(resp.ok).toBeFalsy() + expect(resp.status).toBe(400) + }) + + it('doesnt crash when invalid headers returned', async () => { + const bucket = await createBucket(client) + const key = 'test-1.jpg' + await uploadFile(client, bucket, key, 2, { + 'invalid-header-r': Buffer.alloc(1024 * 9, 'a').toString(), + }) + + const headObj = new HeadObjectCommand({ + Bucket: bucket, + Key: key, + }) + + const r = await client.send(headObj) + + expect(r.$metadata.httpStatusCode).toBe(200) + expect(r.MissingMeta).toBe(1) + }) + + it('can upload with presigned URL', async () => { + const bucket = await createBucket(client) + const key = 'test-1.jpg' + const body = Buffer.alloc(1024 * 2) + + const uploadUrl = await getSignedUrl( + client, + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + }), + { expiresIn: 100 } + ) + + const resp = await fetch(uploadUrl, { + method: 'PUT', + body, + headers: { + 'Content-Length': body.length.toString(), + }, + }) + + expect(resp.ok).toBeTruthy() + }) + + it('can fetch an asset via presigned URL', async () => { + const bucket = await createBucket(client) + const key = 'test-1.jpg' + + await uploadFile(client, bucket, key, 2) + + const getUrl = await getSignedUrl( + client, + new GetObjectCommand({ + Bucket: bucket, Key: key, }), { expiresIn: 100 } @@ -1798,5 +2251,654 @@ describe('S3 Protocol', () => { expect(response.ContentLanguage).toBeUndefined() }) }) + + describe('Object key names with Unicode characters', () => { + it('can be used with MultipartUpload commands', async () => { + const bucketName = await createBucket(client) + const objectName = getUnicodeObjectName() + const createMultiPartUpload = new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: objectName, + ContentType: 'image/jpg', + CacheControl: 'max-age=2000', + }) + const createMultipartResp = await client.send(createMultiPartUpload) + expect(createMultipartResp.UploadId).toBeTruthy() + const uploadId = createMultipartResp.UploadId + + const listMultipartUploads = new ListMultipartUploadsCommand({ + Bucket: bucketName, + }) + const listMultipartResp = await client.send(listMultipartUploads) + expect(listMultipartResp.Uploads?.length).toBe(1) + expect(listMultipartResp.Uploads?.[0].Key).toBe(objectName) + + const data = Buffer.alloc(1024 * 1024 * 2) + const uploadPart = new UploadPartCommand({ + Bucket: bucketName, + Key: objectName, + ContentLength: data.length, + UploadId: uploadId, + Body: data, + PartNumber: 1, + }) + const uploadPartResp = await client.send(uploadPart) + expect(uploadPartResp.ETag).toBeTruthy() + + const listParts = new ListPartsCommand({ + Bucket: bucketName, + Key: objectName, + UploadId: uploadId, + }) + const listPartsResp = await client.send(listParts) + expect(listPartsResp.Parts?.length).toBe(1) + + const completeMultiPartUpload = new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: objectName, + UploadId: uploadId, + MultipartUpload: { + Parts: [ + { + PartNumber: 1, + ETag: uploadPartResp.ETag, + }, + ], + }, + }) + const completeMultipartResp = await client.send(completeMultiPartUpload) + expect(completeMultipartResp.$metadata.httpStatusCode).toBe(200) + expect(completeMultipartResp.Key).toEqual(objectName) + }) + + it('can be used with Put, List, and Delete Object commands', async () => { + const bucketName = await createBucket(client) + const objectName = getUnicodeObjectName() + const putObject = new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: Buffer.alloc(1024 * 1024 * 1), + }) + const putObjectResp = await client.send(putObject) + expect(putObjectResp.$metadata.httpStatusCode).toEqual(200) + + const listObjects = new ListObjectsCommand({ + Bucket: bucketName, + }) + const listObjectsResp = await client.send(listObjects) + expect(listObjectsResp.Contents?.length).toBe(1) + expect(listObjectsResp.Contents?.[0].Key).toBe(objectName) + + const listObjectsV2 = new ListObjectsV2Command({ + Bucket: bucketName, + }) + const listObjectsV2Resp = await client.send(listObjectsV2) + expect(listObjectsV2Resp.Contents?.length).toBe(1) + expect(listObjectsV2Resp.Contents?.[0].Key).toBe(objectName) + + const getObject = new GetObjectCommand({ + Bucket: bucketName, + Key: objectName, + }) + const getObjectResp = await client.send(getObject) + const getObjectRespData = await getObjectResp.Body?.transformToByteArray() + expect(getObjectRespData).toBeTruthy() + expect(getObjectResp.ETag).toBeTruthy() + + const deleteObjects = new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: [ + { + Key: objectName, + }, + ], + }, + }) + const deleteObjectsResp = await client.send(deleteObjects) + expect(deleteObjectsResp.Errors).toBeFalsy() + expect(deleteObjectsResp.Deleted).toEqual([ + { + Key: objectName, + }, + ]) + + const getObjectDeleted = new GetObjectCommand({ + Bucket: bucketName, + Key: objectName, + }) + try { + await client.send(getObjectDeleted) + throw new Error('Should not reach here') + } catch (e) { + expect((e as S3ServiceException).$metadata.httpStatusCode).toEqual(404) + } + }) + + it('can head an object with Unicode and URL-reserved characters in key', async () => { + const bucketName = await createBucket(client) + const objectName = `head-${randomUUID()}-일이삼-🙂-q?foo=1&bar=%25+plus;semi:colon,#frag.jpg` + + await client.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: Buffer.alloc(1024), + }) + ) + + const headResp = await client.send( + new HeadObjectCommand({ + Bucket: bucketName, + Key: objectName, + }) + ) + expect(headResp.$metadata.httpStatusCode).toEqual(200) + expect(headResp.ETag).toBeTruthy() + }) + + it('can delete an object with Unicode and URL-reserved characters using DeleteObjectCommand', async () => { + const bucketName = await createBucket(client) + const objectName = `delete-one-${randomUUID()}-éè-中文-🙂-q?foo=1&bar=%25+plus;semi:colon,#frag.jpg` + + await client.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: Buffer.alloc(1024), + }) + ) + + await client.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + }) + ) + + try { + await client.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectName, + }) + ) + throw new Error('Should not reach here') + } catch (e) { + expect((e as S3ServiceException).$metadata.httpStatusCode).toEqual(404) + } + }) + + it('can paginate ListObjectsV2 with Unicode keys', async () => { + const bucketName = await createBucket(client) + const keys = [ + `utf8-page-${randomUUID()}-🙂.jpg`, + `utf8-page-${randomUUID()}-일이삼.jpg`, + `utf8-page-${randomUUID()}-éè.jpg`, + ] + + await Promise.all( + keys.map((key) => + client.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: key, + Body: Buffer.alloc(64), + }) + ) + ) + ) + + const page1 = await client.send( + new ListObjectsV2Command({ + Bucket: bucketName, + MaxKeys: 1, + }) + ) + expect(page1.Contents?.length).toBe(1) + expect(page1.NextContinuationToken).toBeTruthy() + + const page2 = await client.send( + new ListObjectsV2Command({ + Bucket: bucketName, + MaxKeys: 1, + ContinuationToken: page1.NextContinuationToken, + }) + ) + expect(page2.Contents?.length).toBe(1) + expect(page2.NextContinuationToken).toBeTruthy() + + const page3 = await client.send( + new ListObjectsV2Command({ + Bucket: bucketName, + MaxKeys: 1, + ContinuationToken: page2.NextContinuationToken, + }) + ) + expect(page3.Contents?.length).toBe(1) + + const pagedKeys = [ + page1.Contents?.[0]?.Key, + page2.Contents?.[0]?.Key, + page3.Contents?.[0]?.Key, + ].filter(Boolean) + expect(pagedKeys).toHaveLength(3) + expect(new Set(pagedKeys).size).toBe(3) + expect([...pagedKeys].sort()).toEqual([...keys].sort()) + }) + + it('returns a structured 400 for an invalid ListObjectsV2 continuation token', async () => { + const bucketName = await createBucket(client) + const invalidToken = Buffer.from('v:1\nl:%E0%A4%A').toString('base64') + + await expect( + client.send( + new ListObjectsV2Command({ + Bucket: bucketName, + ContinuationToken: invalidToken, + }) + ) + ).rejects.toMatchObject({ + $metadata: { + httpStatusCode: 400, + }, + message: 'Invalid Parameter ContinuationToken', + }) + }) + + it('can paginate ListMultipartUploads with Unicode and reserved characters in keys', async () => { + const bucketName = await createBucket(client) + const keys = [`mp-${randomUUID()}-🙂:one.jpg`, `mp-${randomUUID()}-일이삼:two.jpg`] + + await Promise.all( + keys.map((key) => + client.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: key, + ContentType: 'image/jpg', + }) + ) + ) + ) + + const page1 = await client.send( + new ListMultipartUploadsCommand({ + Bucket: bucketName, + MaxUploads: 1, + }) + ) + expect(page1.Uploads?.length).toBe(1) + expect(page1.NextKeyMarker).toBeTruthy() + expect(Buffer.from(page1.NextKeyMarker!, 'base64').toString()).toMatch(/^v:1\nl:/) + + const page2 = await client.send( + new ListMultipartUploadsCommand({ + Bucket: bucketName, + MaxUploads: 1, + KeyMarker: page1.NextKeyMarker, + }) + ) + expect(page2.Uploads?.length).toBe(1) + + const pagedKeys = [page1.Uploads?.[0].Key, page2.Uploads?.[0].Key].filter(Boolean) + expect(pagedKeys).toHaveLength(2) + expect(new Set(pagedKeys).size).toBe(2) + expect([...pagedKeys].sort()).toEqual([...keys].sort()) + }) + + it('accepts legacy unversioned KeyMarker tokens for in-flight pagination', async () => { + const bucketName = await createBucket(client) + const runId = randomUUID() + const firstKey = `mp-legacy-${runId}:001%25.jpg` + const secondKey = `mp-legacy-${runId}:002%25.jpg` + + await Promise.all( + [firstKey, secondKey].map((key) => + client.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: key, + ContentType: 'image/jpg', + }) + ) + ) + ) + + const page1 = await client.send( + new ListMultipartUploadsCommand({ + Bucket: bucketName, + MaxUploads: 1, + }) + ) + + expect(page1.Uploads?.[0]?.Key).toBe(firstKey) + expect(page1.NextKeyMarker).toBeTruthy() + expect(Buffer.from(page1.NextKeyMarker!, 'base64').toString()).toMatch(/^v:1\nl:/) + + const legacyToken = Buffer.from(`l:${firstKey}`).toString('base64') + const pageUsingLegacyToken = await client.send( + new ListMultipartUploadsCommand({ + Bucket: bucketName, + MaxUploads: 1, + KeyMarker: legacyToken, + }) + ) + + // Legacy unversioned markers keep legacy parsing semantics for rollout safety. + expect(pageUsingLegacyToken.Uploads?.[0]?.Key).toBe(secondKey) + expect(pageUsingLegacyToken.Uploads?.map((upload) => upload.Key)).not.toContain(firstKey) + expect(pageUsingLegacyToken.NextKeyMarker).toBeFalsy() + }) + + it('returns a structured 400 for an invalid multipart KeyMarker', async () => { + const bucketName = await createBucket(client) + const invalidToken = Buffer.from('v:1\nl:%E0%A4%A').toString('base64') + + await expect( + client.send( + new ListMultipartUploadsCommand({ + Bucket: bucketName, + KeyMarker: invalidToken, + }) + ) + ).rejects.toMatchObject({ + $metadata: { + httpStatusCode: 400, + }, + message: 'Invalid Parameter KeyMarker', + }) + }) + + it('returns a structured 400 for an invalid multipart UploadIdMarker', async () => { + const bucketName = await createBucket(client) + const invalidToken = Buffer.from('v:1\nl:%E0%A4%A').toString('base64') + + await expect( + client.send( + new ListMultipartUploadsCommand({ + Bucket: bucketName, + UploadIdMarker: invalidToken, + }) + ) + ).rejects.toMatchObject({ + $metadata: { + httpStatusCode: 400, + }, + message: 'Invalid Parameter UploadIdMarker', + }) + }) + + it('can copy objects using Unicode keys in CopySource', async () => { + const bucketName = await createBucket(client) + const sourceKey = `copy-src-${randomUUID()}-일이삼-🙂.jpg` + const destinationKey = `copy-dst-${randomUUID()}-일이삼-🙂.jpg` + const destinationKeyWithLeadingSlashSource = `copy-dst-leading-${randomUUID()}-🙂.jpg` + + await client.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: sourceKey, + Body: Buffer.alloc(1024 * 128), + }) + ) + + const copyObjectResp = await client.send( + new CopyObjectCommand({ + Bucket: bucketName, + Key: destinationKey, + CopySource: encodeURI(`${bucketName}/${sourceKey}`), + }) + ) + expect(copyObjectResp.$metadata.httpStatusCode).toBe(200) + + const copyObjectRespLeadingSlash = await client.send( + new CopyObjectCommand({ + Bucket: bucketName, + Key: destinationKeyWithLeadingSlashSource, + CopySource: `/${encodeURI(`${bucketName}/${sourceKey}`)}`, + }) + ) + expect(copyObjectRespLeadingSlash.$metadata.httpStatusCode).toBe(200) + + const listObjectsResp = await client.send( + new ListObjectsV2Command({ + Bucket: bucketName, + }) + ) + const listedKeys = (listObjectsResp.Contents || []).map((item) => item.Key) + expect(listedKeys).toEqual( + expect.arrayContaining([sourceKey, destinationKey, destinationKeyWithLeadingSlashSource]) + ) + }) + + it('can copy objects using fully URL-encoded CopySource', async () => { + const bucketName = await createBucket(client) + const sourceKey = getUnicodeObjectName() + const destinationKey = `copy-dst-encoded-${randomUUID()}-🙂.jpg` + + await client.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: sourceKey, + Body: Buffer.alloc(1024 * 128), + }) + ) + + const copyObjectResp = await client.send( + new CopyObjectCommand({ + Bucket: bucketName, + Key: destinationKey, + CopySource: encodeURIComponent(`${bucketName}/${sourceKey}`), + }) + ) + expect(copyObjectResp.$metadata.httpStatusCode).toBe(200) + + const listObjectsResp = await client.send( + new ListObjectsV2Command({ + Bucket: bucketName, + }) + ) + const listedKeys = (listObjectsResp.Contents || []).map((item) => item.Key) + expect(listedKeys).toEqual(expect.arrayContaining([sourceKey, destinationKey])) + }) + + it('can copy objects when partially encoded CopySource keeps raw ? and # in the key', async () => { + const bucketName = await createBucket(client) + const sourceKey = `copy-src-reserved-${randomUUID()}-일이삼?x=1#frag.png` + const destinationKey = `copy-dst-reserved-${randomUUID()}-🙂.jpg` + + await client.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: sourceKey, + Body: Buffer.alloc(1024 * 128), + }) + ) + + const copyObjectResp = await client.send( + new CopyObjectCommand({ + Bucket: bucketName, + Key: destinationKey, + CopySource: encodeURI(`${bucketName}/${sourceKey}`), + }) + ) + expect(copyObjectResp.$metadata.httpStatusCode).toBe(200) + + const listObjectsResp = await client.send( + new ListObjectsV2Command({ + Bucket: bucketName, + }) + ) + const listedKeys = (listObjectsResp.Contents || []).map((item) => item.Key) + expect(listedKeys).toEqual(expect.arrayContaining([sourceKey, destinationKey])) + }) + + it('can copy objects using fully URL-encoded leading-slash CopySource', async () => { + const bucketName = await createBucket(client) + const sourceKey = getUnicodeObjectName() + const destinationKey = `copy-dst-leading-encoded-${randomUUID()}-🙂.jpg` + + await client.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: sourceKey, + Body: Buffer.alloc(1024 * 128), + }) + ) + + const copyObjectResp = await client.send( + new CopyObjectCommand({ + Bucket: bucketName, + Key: destinationKey, + CopySource: encodeURIComponent(`/${bucketName}/${sourceKey}`), + }) + ) + expect(copyObjectResp.$metadata.httpStatusCode).toBe(200) + + const listObjectsResp = await client.send( + new ListObjectsV2Command({ + Bucket: bucketName, + }) + ) + const listedKeys = (listObjectsResp.Contents || []).map((item) => item.Key) + expect(listedKeys).toEqual(expect.arrayContaining([sourceKey, destinationKey])) + }) + + it('can upload part copy using Unicode keys in CopySource', async () => { + const bucketName = await createBucket(client) + const sourceKey = `copy-part-src-${randomUUID()}-일이삼-🙂.jpg` + const destinationKey = `copy-part-dst-${randomUUID()}-일이삼-🙂.jpg` + + await uploadFile(client, bucketName, sourceKey, 8) + + const createMultiPartUpload = new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: destinationKey, + ContentType: 'image/jpg', + CacheControl: 'max-age=2000', + }) + const createMultipartResp = await client.send(createMultiPartUpload) + expect(createMultipartResp.UploadId).toBeTruthy() + + const uploadPartCopyResp = await client.send( + new UploadPartCopyCommand({ + Bucket: bucketName, + Key: destinationKey, + UploadId: createMultipartResp.UploadId, + PartNumber: 1, + CopySource: `/${encodeURI(`${bucketName}/${sourceKey}`)}`, + CopySourceRange: 'bytes=0-4096', + }) + ) + expect(uploadPartCopyResp.CopyPartResult?.ETag).toBeTruthy() + + const listPartsResp = await client.send( + new ListPartsCommand({ + Bucket: bucketName, + Key: destinationKey, + UploadId: createMultipartResp.UploadId, + }) + ) + expect(listPartsResp.Parts?.length).toBe(1) + }) + + it('can upload part copy using fully URL-encoded CopySource', async () => { + const bucketName = await createBucket(client) + const sourceKey = getUnicodeObjectName() + const destinationKey = `copy-part-dst-encoded-${randomUUID()}-🙂.jpg` + + await uploadFile(client, bucketName, sourceKey, 8) + + const createMultiPartUpload = new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: destinationKey, + ContentType: 'image/jpg', + CacheControl: 'max-age=2000', + }) + const createMultipartResp = await client.send(createMultiPartUpload) + expect(createMultipartResp.UploadId).toBeTruthy() + + const uploadPartCopyResp = await client.send( + new UploadPartCopyCommand({ + Bucket: bucketName, + Key: destinationKey, + UploadId: createMultipartResp.UploadId, + PartNumber: 1, + CopySource: encodeURIComponent(`${bucketName}/${sourceKey}`), + CopySourceRange: 'bytes=0-4096', + }) + ) + expect(uploadPartCopyResp.CopyPartResult?.ETag).toBeTruthy() + + const listPartsResp = await client.send( + new ListPartsCommand({ + Bucket: bucketName, + Key: destinationKey, + UploadId: createMultipartResp.UploadId, + }) + ) + expect(listPartsResp.Parts?.length).toBe(1) + }) + + it('can upload part copy when partially encoded CopySource keeps raw ? and # in the key', async () => { + const bucketName = await createBucket(client) + const sourceKey = `copy-part-src-reserved-${randomUUID()}-일이삼?x=1#frag.png` + const destinationKey = `copy-part-dst-reserved-${randomUUID()}-🙂.jpg` + + await uploadFile(client, bucketName, sourceKey, 8) + + const createMultiPartUpload = new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: destinationKey, + ContentType: 'image/jpg', + CacheControl: 'max-age=2000', + }) + const createMultipartResp = await client.send(createMultiPartUpload) + expect(createMultipartResp.UploadId).toBeTruthy() + + const uploadPartCopyResp = await client.send( + new UploadPartCopyCommand({ + Bucket: bucketName, + Key: destinationKey, + UploadId: createMultipartResp.UploadId, + PartNumber: 1, + CopySource: encodeURI(`${bucketName}/${sourceKey}`), + CopySourceRange: 'bytes=0-4096', + }) + ) + expect(uploadPartCopyResp.CopyPartResult?.ETag).toBeTruthy() + + const listPartsResp = await client.send( + new ListPartsCommand({ + Bucket: bucketName, + Key: destinationKey, + UploadId: createMultipartResp.UploadId, + }) + ) + expect(listPartsResp.Parts?.length).toBe(1) + }) + + it('should not upload if the name contains invalid characters', async () => { + const bucketName = await createBucket(client) + const invalidObjectName = getInvalidObjectName() + try { + const putObject = new PutObjectCommand({ + Bucket: bucketName, + Key: invalidObjectName, + Body: Buffer.alloc(1024 * 1024 * 1), + }) + await client.send(putObject) + throw new Error('Should not reach here') + } catch (e) { + expect((e as Error).message).not.toBe('Should not reach here') + expect((e as S3ServiceException).$metadata.httpStatusCode).toEqual(400) + expect((e as S3ServiceException).name).toEqual('InvalidKey') + expect((e as S3ServiceException).message).toEqual( + `Invalid key: ${encodeURIComponent(invalidObjectName)}` + ) + } + }) + }) }) }) diff --git a/src/test/signed-url-route.test.ts b/src/test/signed-url-route.test.ts new file mode 100644 index 00000000..aa6b9e80 --- /dev/null +++ b/src/test/signed-url-route.test.ts @@ -0,0 +1,51 @@ +import { doesSignedTokenMatchRequestPath } from '@internal/http' +import { encodePathPreservingSeparatorsForTest } from './utils/path-encoding' + +describe('signed URL route path verification', () => { + test('matches canonical encoded object path for object signed route', () => { + const signedObjectPath = 'bucket2/public/일이삼-🙂-q?foo=1&bar=%25+plus;semi:colon,#frag.png' + const rawPath = `/object/sign/${encodePathPreservingSeparatorsForTest(signedObjectPath)}?token=jwt` + + expect(doesSignedTokenMatchRequestPath(rawPath, '/object/sign', signedObjectPath)).toBe(true) + }) + + test('matches canonical encoded object path for render signed route', () => { + const signedObjectPath = 'bucket2/authenticated/casestudy.png' + const rawPath = `/render/image/sign/${encodePathPreservingSeparatorsForTest(signedObjectPath)}?token=jwt` + + expect(doesSignedTokenMatchRequestPath(rawPath, '/render/image/sign', signedObjectPath)).toBe( + true + ) + }) + + test('matches canonical encoded object path for upload signed route', () => { + const signedObjectPath = 'bucket2/public/일이삼-🙂-q?foo=1&bar=%25+plus;semi:colon,#frag.png' + const rawPath = `/object/upload/sign/${encodePathPreservingSeparatorsForTest(signedObjectPath)}?token=jwt` + + expect(doesSignedTokenMatchRequestPath(rawPath, '/object/upload/sign', signedObjectPath)).toBe( + true + ) + }) + + test('rejects double-encoded request paths', () => { + const signedObjectPath = 'bucket2/public/일이삼.txt' + const encodedPath = encodePathPreservingSeparatorsForTest(signedObjectPath) + const doubleEncodedPath = encodedPath.replaceAll('%', '%25') + const rawPath = `/object/sign/${doubleEncodedPath}?token=jwt` + + expect(doesSignedTokenMatchRequestPath(rawPath, '/object/sign', signedObjectPath)).toBe(false) + }) + + test('rejects decoded raw unicode path', () => { + const signedObjectPath = 'bucket2/public/일이삼.txt' + const rawPath = `/object/sign/${signedObjectPath}?token=jwt` + + expect(doesSignedTokenMatchRequestPath(rawPath, '/object/sign', signedObjectPath)).toBe(false) + }) + + test('returns false for missing raw url', () => { + expect( + doesSignedTokenMatchRequestPath(undefined, '/object/sign', 'bucket2/public/sadcat-upload.png') + ).toBe(false) + }) +}) diff --git a/src/test/test-hygiene.test.ts b/src/test/test-hygiene.test.ts new file mode 100644 index 00000000..16fab66c --- /dev/null +++ b/src/test/test-hygiene.test.ts @@ -0,0 +1,56 @@ +import fs from 'fs' +import path from 'path' + +function collectTestFiles(dir: string): string[] { + const entries = fs.readdirSync(dir, { withFileTypes: true }) + const files: string[] = [] + + for (const entry of entries) { + const fullPath = path.join(dir, entry.name) + + if (entry.isDirectory()) { + files.push(...collectTestFiles(fullPath)) + continue + } + + if (entry.isFile() && fullPath.endsWith('.test.ts')) { + files.push(fullPath) + } + } + + return files +} + +describe('test hygiene', () => { + it('does not leak ad-hoc app instances in test files', () => { + const testFiles = collectTestFiles(__dirname) + const violations: string[] = [] + const forbiddenPattern = `app().${'inject('}` + const appFactoryPattern = /\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*app\(/g + + for (const file of testFiles) { + const content = fs.readFileSync(file, 'utf8') + const lines = content.split('\n') + + lines.forEach((line, index) => { + if (line.includes(forbiddenPattern)) { + violations.push(`${path.relative(process.cwd(), file)}:${index + 1}`) + } + }) + + let match = appFactoryPattern.exec(content) + while (match) { + const appVar = match[1] + if (!content.includes(`${appVar}.close(`)) { + const declarationLine = content.slice(0, match.index).split('\n').length + violations.push(`${path.relative(process.cwd(), file)}:${declarationLine}`) + } + match = appFactoryPattern.exec(content) + } + + appFactoryPattern.lastIndex = 0 + } + + expect(violations).toEqual([]) + }) +}) diff --git a/src/test/tus.test.ts b/src/test/tus.test.ts index 6c49e99b..014f6215 100644 --- a/src/test/tus.test.ts +++ b/src/test/tus.test.ts @@ -16,7 +16,7 @@ import { DetailedError } from 'tus-js-client' import app from '../app' import { getConfig } from '../config' import { backends, Storage, StorageKnexDB } from '../storage' -import { checkBucketExists } from './common' +import { checkBucketExists, getInvalidObjectName, getUnicodeObjectName } from './common' const { serviceKeyAsync, tenantId, storageS3Bucket, storageBackendType } = getConfig() const oneChunkFile = fs.createReadStream(path.resolve(__dirname, 'assets', 'sadcat.jpg')) @@ -256,9 +256,7 @@ describe('Tus multipart', () => { const objectName = randomUUID() + '-cat.jpeg' - const signedUpload = await storage - .from(bucketName) - .signUploadObjectUrl(objectName, `${bucketName}/${objectName}`, 3600) + const signedUpload = await storage.from(bucketName).signUploadObjectUrl(objectName, 3600) const result = await new Promise((resolve, reject) => { const upload = new tus.Upload(oneChunkFile, { @@ -320,6 +318,48 @@ describe('Tus multipart', () => { }) }) + it('will allow uploading using signed upload url with a Unicode object key', async () => { + const bucket = await storage.createBucket({ + id: bucketName, + name: bucketName, + public: true, + }) + + const objectName = `${randomUUID()}-${getUnicodeObjectName()}` + const signedUpload = await storage.from(bucketName).signUploadObjectUrl(objectName, 3600) + + const result = await new Promise((resolve, reject) => { + const upload = new tus.Upload(oneChunkFile, { + endpoint: `${localServerAddress}/upload/resumable/sign`, + onShouldRetry: () => false, + uploadDataDuringCreation: false, + headers: { + 'x-signature': signedUpload.token, + }, + metadata: { + bucketName, + objectName, + contentType: 'image/jpeg', + cacheControl: '3600', + }, + onError(error) { + reject(error) + }, + onSuccess: () => { + resolve(true) + }, + }) + + upload.start() + }) + + expect(result).toEqual(true) + + const dbAsset = await storage.from(bucket.id).findObject(objectName, '*') + expect(dbAsset?.name).toBe(objectName) + expect(dbAsset?.bucket_id).toBe(bucket.id) + }) + it('will allow uploading using signed upload url without authorization token, honouring the owner id', async () => { const bucket = await storage.createBucket({ id: bucketName, @@ -331,7 +371,7 @@ describe('Tus multipart', () => { const signedUpload = await storage .from(bucketName) - .signUploadObjectUrl(objectName, `${bucketName}/${objectName}`, 3600, 'some-owner-id') + .signUploadObjectUrl(objectName, 3600, 'some-owner-id') const result = await new Promise((resolve, reject) => { const upload = new tus.Upload(oneChunkFile, { @@ -395,9 +435,7 @@ describe('Tus multipart', () => { const objectName = randomUUID() + '-cat.jpeg' - const signedUpload = await storage - .from(bucketName) - .signUploadObjectUrl(objectName, `${bucketName}/${objectName}`, 1) + const signedUpload = await storage.from(bucketName).signUploadObjectUrl(objectName, 1) await wait(2000) @@ -531,4 +569,334 @@ describe('Tus multipart', () => { } }) }) + + describe('TUS control endpoints', () => { + function encodeTusMetadataValue(value: string) { + return Buffer.from(value, 'utf8').toString('base64') + } + + function buildTusMetadata(objectName: string) { + return [ + `bucketName ${encodeTusMetadataValue(bucketName)}`, + `objectName ${encodeTusMetadataValue(objectName)}`, + `contentType ${encodeTusMetadataValue('image/jpeg')}`, + `cacheControl ${encodeTusMetadataValue('3600')}`, + ].join(',') + } + + async function createTusUploadSession(objectName: string, authorization: string) { + const createResponse = await fetch(`${localServerAddress}/upload/resumable`, { + method: 'POST', + headers: { + authorization, + 'x-upsert': 'true', + 'tus-resumable': '1.0.0', + 'upload-length': '32', + 'upload-metadata': buildTusMetadata(objectName), + }, + }) + + expect(createResponse.status).toBe(201) + const location = createResponse.headers.get('location') + expect(location).toBeTruthy() + + return new URL(location || '', localServerAddress).toString() + } + + async function patchTusUploadSession(uploadUrl: string, authorization: string, body: Buffer) { + const patchResponse = await fetch(uploadUrl, { + method: 'PATCH', + headers: { + authorization, + 'tus-resumable': '1.0.0', + 'upload-offset': '0', + 'content-type': 'application/offset+octet-stream', + 'content-length': String(body.length), + }, + body, + }) + + expect(patchResponse.status).toBe(204) + expect(patchResponse.headers.get('upload-offset')).toBe(String(body.length)) + } + + test('supports HEAD and DELETE flow for ASCII object keys', async () => { + await storage.createBucket({ + id: bucketName, + name: bucketName, + public: true, + }) + + const authorization = `Bearer ${await serviceKeyAsync}` + const objectName = `${randomUUID()}-ascii-control-q?foo=1&bar=%25+plus;semi:colon,.jpg` + const uploadUrl = await createTusUploadSession(objectName, authorization) + + const headBeforeDelete = await fetch(uploadUrl, { + method: 'HEAD', + headers: { + authorization, + 'tus-resumable': '1.0.0', + }, + }) + expect(headBeforeDelete.status).toBe(200) + expect(headBeforeDelete.headers.get('upload-offset')).toBe('0') + expect(headBeforeDelete.headers.get('upload-length')).toBe('32') + + const deleteResponse = await fetch(uploadUrl, { + method: 'DELETE', + headers: { + authorization, + 'tus-resumable': '1.0.0', + }, + }) + expect([200, 204]).toContain(deleteResponse.status) + + const headAfterDelete = await fetch(uploadUrl, { + method: 'HEAD', + headers: { + authorization, + 'tus-resumable': '1.0.0', + }, + }) + expect([404, 410]).toContain(headAfterDelete.status) + }) + + test('supports HEAD and DELETE flow for Unicode object keys', async () => { + await storage.createBucket({ + id: bucketName, + name: bucketName, + public: true, + }) + + const authorization = `Bearer ${await serviceKeyAsync}` + const objectName = `${randomUUID()}-${getUnicodeObjectName()}` + const uploadUrl = await createTusUploadSession(objectName, authorization) + + const headBeforeDelete = await fetch(uploadUrl, { + method: 'HEAD', + headers: { + authorization, + 'tus-resumable': '1.0.0', + }, + }) + expect(headBeforeDelete.status).toBe(200) + expect(headBeforeDelete.headers.get('upload-offset')).toBe('0') + expect(headBeforeDelete.headers.get('upload-length')).toBe('32') + + const deleteResponse = await fetch(uploadUrl, { + method: 'DELETE', + headers: { + authorization, + 'tus-resumable': '1.0.0', + }, + }) + expect([200, 204]).toContain(deleteResponse.status) + + const headAfterDelete = await fetch(uploadUrl, { + method: 'HEAD', + headers: { + authorization, + 'tus-resumable': '1.0.0', + }, + }) + expect([404, 410]).toContain(headAfterDelete.status) + }) + + test('supports upload completion and object GET for ASCII URL-reserved keys', async () => { + const bucket = await storage.createBucket({ + id: bucketName, + name: bucketName, + public: true, + }) + + const authorization = `Bearer ${await serviceKeyAsync}` + const objectName = `${randomUUID()}-ascii-get-q?foo=1&bar=%25+plus;semi:colon,#frag.jpg` + const uploadUrl = await createTusUploadSession(objectName, authorization) + const payload = Buffer.from('abcdefghijklmnopqrstuvwxyz012345') + + await patchTusUploadSession(uploadUrl, authorization, payload) + + const appInstance = app() + try { + const getResponse = await appInstance.inject({ + method: 'GET', + url: `/object/${bucket.id}/${encodeURIComponent(objectName)}`, + headers: { + authorization, + }, + }) + expect(getResponse.statusCode).toBe(200) + expect(getResponse.headers['content-length']).toBe(String(payload.length)) + } finally { + await appInstance.close() + } + }) + + test('supports upload completion and object GET for Unicode URL-reserved keys', async () => { + const bucket = await storage.createBucket({ + id: bucketName, + name: bucketName, + public: true, + }) + + const authorization = `Bearer ${await serviceKeyAsync}` + const objectName = `${randomUUID()}-${getUnicodeObjectName()}-q?foo=1&bar=%25+plus;semi:colon,#frag.jpg` + const uploadUrl = await createTusUploadSession(objectName, authorization) + const payload = Buffer.from('abcdefghijklmnopqrstuvwxyz012345') + + await patchTusUploadSession(uploadUrl, authorization, payload) + + const appInstance = app() + try { + const getResponse = await appInstance.inject({ + method: 'GET', + url: `/object/${bucket.id}/${encodeURIComponent(objectName)}`, + headers: { + authorization, + }, + }) + expect(getResponse.statusCode).toBe(200) + expect(getResponse.headers['content-length']).toBe(String(payload.length)) + } finally { + await appInstance.close() + } + }) + }) + + describe('Object key names with Unicode characters', () => { + it('can be uploaded with the TUS protocol', async () => { + const objectName = randomUUID() + '-' + getUnicodeObjectName() + const authorization = `Bearer ${await serviceKeyAsync}` + + const bucket = await storage.createBucket({ + id: bucketName, + name: bucketName, + public: true, + }) + + const result = await new Promise((resolve, reject) => { + const upload = new tus.Upload(oneChunkFile, { + endpoint: `${localServerAddress}/upload/resumable`, + onShouldRetry: () => false, + uploadDataDuringCreation: false, + headers: { + authorization, + 'x-upsert': 'true', + }, + metadata: { + bucketName, + objectName, + contentType: 'image/jpeg', + cacheControl: '3600', + metadata: JSON.stringify({ + test1: 'test1', + test2: 'test2', + }), + }, + onError(error) { + reject(error) + }, + onSuccess: () => { + resolve(true) + }, + }) + + upload.start() + }) + + expect(result).toEqual(true) + + const dbAsset = await storage.from(bucket.id).findObject(objectName, '*') + expect(dbAsset).toEqual({ + bucket_id: bucket.id, + created_at: expect.any(Date), + id: expect.any(String), + last_accessed_at: expect.any(Date), + metadata: { + cacheControl: 'max-age=3600', + contentLength: 29526, + eTag: '"53e1323c929d57b09b95fbe6d531865c-1"', + httpStatusCode: 200, + lastModified: expect.any(String), + mimetype: 'image/jpeg', + size: 29526, + }, + user_metadata: { + test1: 'test1', + test2: 'test2', + }, + name: objectName, + owner: null, + owner_id: null, + path_tokens: objectName.split('/'), + updated_at: expect.any(Date), + version: expect.any(String), + }) + + const appInstance = app() + try { + const getResponse = await appInstance.inject({ + method: 'GET', + url: `/object/${bucketName}/${encodeURIComponent(objectName)}`, + headers: { + authorization, + }, + }) + expect(getResponse.statusCode).toBe(200) + expect(getResponse.headers['etag']).toBe('"53e1323c929d57b09b95fbe6d531865c-1"') + expect(getResponse.headers['cache-control']).toBe('max-age=3600') + expect(getResponse.headers['content-length']).toBe('29526') + expect(getResponse.headers['content-type']).toBe('image/jpeg') + } finally { + await appInstance.close() + } + }) + + it('should not upload if the name contains invalid characters', async () => { + await storage.createBucket({ + id: bucketName, + name: bucketName, + public: true, + }) + + const invalidObjectName = randomUUID() + '-' + getInvalidObjectName() + const authorization = `Bearer ${await serviceKeyAsync}` + try { + await new Promise((resolve, reject) => { + const upload = new tus.Upload(oneChunkFile, { + endpoint: `${localServerAddress}/upload/resumable`, + onShouldRetry: () => false, + uploadDataDuringCreation: false, + headers: { + authorization, + 'x-upsert': 'true', + }, + metadata: { + bucketName, + objectName: invalidObjectName, + contentType: 'image/jpeg', + cacheControl: '3600', + }, + onError(error) { + reject(error) + }, + onSuccess: () => { + resolve(true) + }, + }) + + upload.start() + }) + + throw new Error('it should error with invalid key') + } catch (e) { + expect((e as Error).message).not.toEqual('it should error with invalid key') + const err = e as DetailedError + expect(err.originalResponse.getStatus()).toEqual(400) + expect(err.originalResponse.getBody()).toEqual( + `Invalid key: ${encodeURIComponent(invalidObjectName)}` + ) + } + }) + }) }) diff --git a/src/test/unicode-object-name-db-constraints.test.ts b/src/test/unicode-object-name-db-constraints.test.ts new file mode 100644 index 00000000..7f55700b --- /dev/null +++ b/src/test/unicode-object-name-db-constraints.test.ts @@ -0,0 +1,88 @@ +'use strict' + +import { randomUUID } from 'node:crypto' +import { DatabaseError } from 'pg' +import { useStorage } from './utils/storage' + +describe('Unicode object name database constraints', () => { + const tHelper = useStorage() + const testBucketName = `unicode-db-constraints-${Date.now()}` + + beforeAll(async () => { + await tHelper.database.createBucket({ + id: testBucketName, + name: testBucketName, + }) + }) + + const invalidKey = `invalid-\u000b-${randomUUID()}` + + it('rejects invalid object names at the storage.objects constraint', async () => { + const db = tHelper.database.connection.pool.acquire() + const tnx = await db.transaction() + + try { + await expect( + tnx.raw( + 'INSERT INTO storage.objects (bucket_id, name, owner, version) VALUES (?, ?, ?, ?)', + [testBucketName, invalidKey, null, randomUUID()] + ) + ).rejects.toMatchObject>({ + code: '23514', + constraint: 'objects_name_check', + }) + } finally { + await tnx.rollback() + } + }) + + it('rejects invalid multipart upload keys at the storage.s3_multipart_uploads constraint', async () => { + const db = tHelper.database.connection.pool.acquire() + const tnx = await db.transaction() + + try { + await expect( + tnx.raw( + `INSERT INTO storage.s3_multipart_uploads + (id, in_progress_size, upload_signature, bucket_id, key, version, owner_id) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + [randomUUID(), 0, 'sig', testBucketName, invalidKey, randomUUID(), null] + ) + ).rejects.toMatchObject>({ + code: '23514', + constraint: 's3_multipart_uploads_key_check', + }) + } finally { + await tnx.rollback() + } + }) + + it('rejects invalid multipart part keys at the storage.s3_multipart_uploads_parts constraint', async () => { + const db = tHelper.database.connection.pool.acquire() + const tnx = await db.transaction() + const uploadId = randomUUID() + + try { + await tnx.raw( + `INSERT INTO storage.s3_multipart_uploads + (id, in_progress_size, upload_signature, bucket_id, key, version, owner_id) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + [uploadId, 0, 'sig', testBucketName, `valid-${randomUUID()}.txt`, randomUUID(), null] + ) + + await expect( + tnx.raw( + `INSERT INTO storage.s3_multipart_uploads_parts + (upload_id, size, part_number, bucket_id, key, etag, owner_id, version) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + [uploadId, 1, 1, testBucketName, invalidKey, 'etag', null, randomUUID()] + ) + ).rejects.toMatchObject>({ + code: '23514', + constraint: 's3_multipart_uploads_parts_key_check', + }) + } finally { + await tnx.rollback() + } + }) +}) diff --git a/src/test/unicode-object-name-migration.test.ts b/src/test/unicode-object-name-migration.test.ts new file mode 100644 index 00000000..8795e72e --- /dev/null +++ b/src/test/unicode-object-name-migration.test.ts @@ -0,0 +1,31 @@ +'use strict' + +import fs from 'node:fs' +import path from 'node:path' + +describe('unicode object name migration', () => { + test('keeps both SQL_ASCII and non-SQL_ASCII constraint branches', () => { + const migrationPath = path.resolve( + __dirname, + '../../migrations/tenant/57-unicode-object-names.sql' + ) + const sql = fs.readFileSync(migrationPath, 'utf8') + + expect(sql).toContain(`server_encoding = 'SQL_ASCII'`) + expect(sql).toContain(String.raw`name !~ E'\\xEF\\xBF\\xBE|\\xEF\\xBF\\xBF'`) + expect(sql).toContain(String.raw`name !~ E'\\xED[\\xA0-\\xBF][\\x80-\\xBF]'`) + expect(sql).toContain(String.raw`POSITION(U&'\FFFE' IN name) = 0`) + expect(sql).toContain(String.raw`POSITION(U&'\FFFF' IN name) = 0`) + expect(sql).toContain('ADD CONSTRAINT objects_name_check') + expect(sql).toContain('ADD CONSTRAINT s3_multipart_uploads_key_check') + expect(sql).toContain('ADD CONSTRAINT s3_multipart_uploads_parts_key_check') + expect(sql).toContain(String.raw`key !~ E'[\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F]'`) + expect(sql).toContain(String.raw`key !~ E'\\xED[\\xA0-\\xBF][\\x80-\\xBF]'`) + expect(sql).toContain(String.raw`POSITION(U&'\FFFE' IN key) = 0`) + expect(sql).toContain(String.raw`POSITION(U&'\FFFF' IN key) = 0`) + expect(sql).toContain('NOT VALID') + expect(sql).toContain('VALIDATE CONSTRAINT objects_name_check') + expect(sql).toContain('VALIDATE CONSTRAINT s3_multipart_uploads_key_check') + expect(sql).toContain('VALIDATE CONSTRAINT s3_multipart_uploads_parts_key_check') + }) +}) diff --git a/src/test/utils/path-encoding.ts b/src/test/utils/path-encoding.ts new file mode 100644 index 00000000..fc5625b7 --- /dev/null +++ b/src/test/utils/path-encoding.ts @@ -0,0 +1,10 @@ +export function encodePathPreservingSeparatorsForTest(path: string): string { + return path + .split('/') + .map((pathToken) => encodeURIComponent(pathToken)) + .join('/') +} + +export function encodeBucketAndObjectPathForTest(bucket: string, key: string): string { + return `${encodeURIComponent(bucket)}/${encodePathPreservingSeparatorsForTest(key)}` +} diff --git a/src/test/webhook-filter.test.ts b/src/test/webhook-filter.test.ts new file mode 100644 index 00000000..3e84d749 --- /dev/null +++ b/src/test/webhook-filter.test.ts @@ -0,0 +1,55 @@ +import { shouldDisableWebhookEvent } from '@storage/events/lifecycle/webhook-filter' + +describe('webhook filter', () => { + test('matches object-level disableEvents entries with Unicode and URL-reserved object names', () => { + const objectName = '폴더/子目录/파일-🙂-q?foo=1&bar=%25+plus;semi:colon,#frag.png' + const eventType = 'ObjectCreated:Post' + + const disabled = shouldDisableWebhookEvent(disabledEvents(eventType, objectName), eventType, { + bucketId: 'bucket6', + name: objectName, + }) + + expect(disabled).toBe(true) + }) + + test('does not match URL-encoded object-level disableEvents entries', () => { + const objectName = '폴더/子目录/파일-🙂-q?foo=1&bar=%25+plus;semi:colon,#frag.png' + const eventType = 'ObjectCreated:Post' + + const disabled = shouldDisableWebhookEvent( + [`Webhook:${eventType}:bucket6/${encodeURIComponent(objectName)}`], + eventType, + { + bucketId: 'bucket6', + name: objectName, + } + ) + + expect(disabled).toBe(false) + }) + + test('does not match path-segment URL-encoded disableEvents entries from external config', () => { + const objectName = '폴더/子目录/파일-🙂-q?foo=1&bar=%25+plus;semi:colon,#frag.png' + const eventType = 'ObjectCreated:Post' + const encodedByPathSegment = objectName + .split('/') + .map((segment) => encodeURIComponent(segment)) + .join('/') + + const disabled = shouldDisableWebhookEvent( + [`Webhook:${eventType}:bucket6/${encodedByPathSegment}`], + eventType, + { + bucketId: 'bucket6', + name: objectName, + } + ) + + expect(disabled).toBe(false) + }) +}) + +function disabledEvents(eventType: string, objectName: string) { + return [`Webhook:${eventType}:bucket6/${objectName}`] +} diff --git a/src/test/webhooks.test.ts b/src/test/webhooks.test.ts index 9fcdace1..f08165c2 100644 --- a/src/test/webhooks.test.ts +++ b/src/test/webhooks.test.ts @@ -8,7 +8,10 @@ mergeConfig({ }) import { getPostgresConnection, getServiceKeyUser } from '@internal/database' +import { StorageKnexDB } from '@storage/database' +import { TenantLocation } from '@storage/locator' import { Obj } from '@storage/schemas' +import { Storage } from '@storage/storage' import { randomUUID } from 'crypto' import { FastifyInstance } from 'fastify' import FormData from 'form-data' @@ -385,17 +388,195 @@ describe('Webhooks', () => { }) ) }) + + it('will emit Unicode and URL-reserved object names in creation webhook payloads', async () => { + const form = new FormData() + const authorization = `Bearer ${await serviceKeyAsync}` + form.append('file', fs.createReadStream(`./src/test/assets/sadcat.jpg`)) + const headers = Object.assign({}, form.getHeaders(), { + authorization, + }) + + const objectName = `public/${randomUUID()}-폴더/子目录/파일-🙂-q?foo=1&bar=%25+plus;semi:colon,#frag.png` + + const response = await appInstance.inject({ + method: 'POST', + url: `/object/bucket6/${encodeURIComponent(objectName)}`, + headers, + payload: form, + }) + + expect(response.statusCode).toBe(200) + expect(sendSpy).toBeCalledTimes(1) + expect(sendSpy).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + event: expect.objectContaining({ + type: 'ObjectCreated:Post', + payload: expect.objectContaining({ + bucketId: 'bucket6', + name: objectName, + }), + }), + }), + }) + ) + }) + + it('will emit a webhook with ObjectCreated:Put when uploading with upsert to an existing key', async () => { + const objectName = `upsert-${randomUUID()}-existing-key.png` + await createObject(pg, 'bucket6', objectName) + + const authorization = `Bearer ${await serviceKeyAsync}` + const response = await appInstance.inject({ + method: 'PUT', + url: `/object/bucket6/${encodeURIComponent(objectName)}`, + headers: { + authorization, + 'Content-Type': 'image/png', + }, + payload: fs.createReadStream(`./src/test/assets/sadcat.jpg`), + }) + + expect(response.statusCode).toBe(200) + + const webhookCalls = sendSpy.mock.calls + .map(([payload]) => payload) + .filter((payload) => payload?.name === 'webhooks') + + expect(webhookCalls).toHaveLength(1) + expect(webhookCalls[0]).toEqual( + expect.objectContaining({ + data: expect.objectContaining({ + event: expect.objectContaining({ + type: 'ObjectCreated:Put', + payload: expect.objectContaining({ + bucketId: 'bucket6', + name: objectName, + uploadType: 'standard', + }), + }), + }), + }) + ) + }) + + it('will emit a webhook with ObjectUpdated:Metadata when object metadata is updated', async () => { + const objectName = `metadata-${randomUUID()}-update-target.png` + const obj = await createObject(pg, 'bucket6', objectName) + + const db = new StorageKnexDB(pg, { + tenantId, + host: 'localhost', + }) + const storage = new Storage({} as any, db, new TenantLocation('bucket')) + + const metadata = { + cacheControl: 'public, max-age=120', + contentLength: 3746, + eTag: 'etag-metadata-update', + lastModified: new Date('2026-03-05T12:00:00.000Z'), + httpStatusCode: 200, + mimetype: 'image/png', + size: 3746, + xRobotsTag: 'noindex', + } + + await storage.from('bucket6').updateObjectMetadata(objectName, metadata) + + expect(sendSpy).toBeCalledTimes(1) + expect(sendSpy).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'webhooks', + data: expect.objectContaining({ + event: expect.objectContaining({ + type: 'ObjectUpdated:Metadata', + payload: expect.objectContaining({ + bucketId: 'bucket6', + name: objectName, + metadata: expect.objectContaining({ + cacheControl: metadata.cacheControl, + mimetype: metadata.mimetype, + size: metadata.size, + xRobotsTag: metadata.xRobotsTag, + }), + }), + }), + }), + }) + ) + }) + + it('will preserve Unicode and URL-reserved object names in move webhook payloads', async () => { + const sourceKey = `source-${randomUUID()}-일이삼/子目录/파일-🙂-q?foo=1&bar=%25+plus;semi:colon,#frag.png` + const destinationKey = `dest-${randomUUID()}-폴더/子目录/파일-🙂-q?x=1&y=%25+plus;semi:colon,#frag.png` + const obj = await createObject(pg, 'bucket6', sourceKey) + + const authorization = `Bearer ${await serviceKeyAsync}` + const response = await appInstance.inject({ + method: 'POST', + url: `/object/move`, + headers: { + authorization, + }, + payload: { + bucketId: 'bucket6', + sourceKey: obj.name, + destinationKey, + }, + }) + + expect(response.statusCode).toBe(200) + expect(sendSpy).toBeCalledTimes(3) + + expect(sendSpy).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + data: expect.objectContaining({ + event: expect.objectContaining({ + type: 'ObjectRemoved:Move', + payload: expect.objectContaining({ + bucketId: 'bucket6', + name: sourceKey, + }), + }), + }), + }) + ) + + expect(sendSpy).toHaveBeenNthCalledWith( + 3, + expect.objectContaining({ + data: expect.objectContaining({ + event: expect.objectContaining({ + type: 'ObjectCreated:Move', + payload: expect.objectContaining({ + bucketId: 'bucket6', + name: destinationKey, + oldObject: expect.objectContaining({ + bucketId: 'bucket6', + name: sourceKey, + }), + }), + }), + }), + }) + ) + }) }) -async function createObject(pg: TenantConnection, bucketId: string) { - const objectName = Date.now() +async function createObject( + pg: TenantConnection, + bucketId: string, + objectName = Date.now().toString() +) { const tnx = await pg.transaction() const [data] = await tnx .from('objects') .insert([ { - name: objectName.toString(), + name: objectName, bucket_id: bucketId, version: randomUUID(), metadata: { diff --git a/src/test/xml-parser-plugin.test.ts b/src/test/xml-parser-plugin.test.ts index 6bb014da..ee06f8e6 100644 --- a/src/test/xml-parser-plugin.test.ts +++ b/src/test/xml-parser-plugin.test.ts @@ -50,6 +50,62 @@ describe('xmlParser plugin', () => { } }) + it('accepts valid numeric entities and still applies value processors', async () => { + const app = await buildXmlApp(['CompleteMultipartUpload.Part']) + + try { + const response = await app.inject({ + method: 'POST', + url: '/xml', + headers: { + 'content-type': 'application/xml', + accept: 'application/json', + }, + payload: + '1🙂', + }) + + expect(response.statusCode).toBe(200) + expect(response.json()).toEqual({ + body: { + CompleteMultipartUpload: { + Part: [{ PartNumber: 1, ETag: '🙂' }], + }, + }, + }) + } finally { + await app.close() + } + }) + + it('accepts decimal astral numeric entities', async () => { + const app = await buildXmlApp(['CompleteMultipartUpload.Part']) + + try { + const response = await app.inject({ + method: 'POST', + url: '/xml', + headers: { + 'content-type': 'application/xml', + accept: 'application/json', + }, + payload: + '🙂', + }) + + expect(response.statusCode).toBe(200) + expect(response.json()).toEqual({ + body: { + CompleteMultipartUpload: { + Part: [{ ETag: '🙂' }], + }, + }, + }) + } finally { + await app.close() + } + }) + it('returns 400 for malformed XML payloads', async () => { const app = await buildXmlApp() @@ -71,6 +127,57 @@ describe('xmlParser plugin', () => { } }) + it.each([ + '�', + '�', + '�', + '�', + '�', + '￿', + '￿', + ])('returns 400 for XML-forbidden numeric entity %s', async (entity) => { + const app = await buildXmlApp() + + try { + const response = await app.inject({ + method: 'POST', + url: '/xml', + headers: { + 'content-type': 'application/xml', + accept: 'application/json', + }, + payload: `${entity}`, + }) + + expect(response.statusCode).toBe(400) + expect(response.json().message).toContain('Invalid XML payload') + } finally { + await app.close() + } + }) + + it('returns 400 for out-of-range numeric entities', async () => { + const app = await buildXmlApp() + + try { + const response = await app.inject({ + method: 'POST', + url: '/xml', + headers: { + 'content-type': 'application/xml', + accept: 'application/json', + }, + payload: + '', + }) + + expect(response.statusCode).toBe(400) + expect(response.json().message).toContain('Invalid XML payload') + } finally { + await app.close() + } + }) + it('serializes response payloads as XML when requested', async () => { const app = await buildXmlApp()