feat: support unicode in object names

migrations/tenant/57-unicode-object-names.sql

@@ -0,0 +1,143 @@
+DO $$
+DECLARE
+  -- SQL_ASCII databases do not have reliable Unicode code point semantics, so
+  -- the migration needs a byte-pattern branch instead of U&'' character checks.
+  -- This is the case for OrioleDB and/or --locale=C databases.
+  server_encoding text := current_setting('server_encoding');
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1
+    FROM pg_constraint c
+    JOIN pg_class t ON t.oid = c.conrelid
+    JOIN pg_namespace n ON n.oid = t.relnamespace
+    WHERE c.conname = 'objects_name_check'
+      AND n.nspname = 'storage'
+      AND t.relname = 'objects'
+  ) THEN
+    IF server_encoding = 'SQL_ASCII' THEN
+      EXECUTE $ddl$
+        ALTER TABLE "storage"."objects"
+          ADD CONSTRAINT objects_name_check
+          CHECK (
+            name !~ E'[\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F]'
+            AND name !~ E'\\xED[\\xA0-\\xBF][\\x80-\\xBF]'
+            AND name !~ E'\\xEF\\xBF\\xBE|\\xEF\\xBF\\xBF'
+          ) NOT VALID
+      $ddl$;
+    ELSE
+      EXECUTE $ddl$
+        ALTER TABLE "storage"."objects"
+          ADD CONSTRAINT objects_name_check
+          CHECK (
+            name !~ E'[\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F]'
+            AND POSITION(U&'\FFFE' IN name) = 0
+            AND POSITION(U&'\FFFF' IN name) = 0
+          ) NOT VALID
+      $ddl$;
+    END IF;
+  END IF;
+
+  IF EXISTS (
+    SELECT 1
+    FROM pg_constraint c
+    JOIN pg_class t ON t.oid = c.conrelid
+    JOIN pg_namespace n ON n.oid = t.relnamespace
+    WHERE c.conname = 'objects_name_check'
+      AND n.nspname = 'storage'
+      AND t.relname = 'objects'
+      AND c.convalidated = false
+  ) THEN
+    EXECUTE 'ALTER TABLE "storage"."objects" VALIDATE CONSTRAINT objects_name_check';
+  END IF;
+
+  IF NOT EXISTS (
+    SELECT 1
+    FROM pg_constraint c
+    JOIN pg_class t ON t.oid = c.conrelid
+    JOIN pg_namespace n ON n.oid = t.relnamespace
+    WHERE c.conname = 's3_multipart_uploads_key_check'
+      AND n.nspname = 'storage'
+      AND t.relname = 's3_multipart_uploads'
+  ) THEN
+    IF server_encoding = 'SQL_ASCII' THEN
+      EXECUTE $ddl$
+        ALTER TABLE "storage"."s3_multipart_uploads"
+          ADD CONSTRAINT s3_multipart_uploads_key_check
+          CHECK (
+            key !~ E'[\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F]'
+            AND key !~ E'\\xED[\\xA0-\\xBF][\\x80-\\xBF]'
+            AND key !~ E'\\xEF\\xBF\\xBE|\\xEF\\xBF\\xBF'
+          ) NOT VALID
+      $ddl$;
+    ELSE
+      EXECUTE $ddl$
+        ALTER TABLE "storage"."s3_multipart_uploads"
+          ADD CONSTRAINT s3_multipart_uploads_key_check
+          CHECK (
+            key !~ E'[\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F]'
+            AND POSITION(U&'\FFFE' IN key) = 0
+            AND POSITION(U&'\FFFF' IN key) = 0
+          ) NOT VALID
+      $ddl$;
+    END IF;
+  END IF;
+
+  IF EXISTS (
+    SELECT 1
+    FROM pg_constraint c
+    JOIN pg_class t ON t.oid = c.conrelid
+    JOIN pg_namespace n ON n.oid = t.relnamespace
+    WHERE c.conname = 's3_multipart_uploads_key_check'
+      AND n.nspname = 'storage'
+      AND t.relname = 's3_multipart_uploads'
+      AND c.convalidated = false
+  ) THEN
+    EXECUTE 'ALTER TABLE "storage"."s3_multipart_uploads" VALIDATE CONSTRAINT s3_multipart_uploads_key_check';
+  END IF;
+
+  IF NOT EXISTS (
+    SELECT 1
+    FROM pg_constraint c
+    JOIN pg_class t ON t.oid = c.conrelid
+    JOIN pg_namespace n ON n.oid = t.relnamespace
+    WHERE c.conname = 's3_multipart_uploads_parts_key_check'
+      AND n.nspname = 'storage'
+      AND t.relname = 's3_multipart_uploads_parts'
+  ) THEN
+    IF server_encoding = 'SQL_ASCII' THEN
+      EXECUTE $ddl$
+        ALTER TABLE "storage"."s3_multipart_uploads_parts"
+          ADD CONSTRAINT s3_multipart_uploads_parts_key_check
+          CHECK (
+            key !~ E'[\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F]'
+            AND key !~ E'\\xED[\\xA0-\\xBF][\\x80-\\xBF]'
+            AND key !~ E'\\xEF\\xBF\\xBE|\\xEF\\xBF\\xBF'
+          ) NOT VALID
+      $ddl$;
+    ELSE
+      EXECUTE $ddl$
+        ALTER TABLE "storage"."s3_multipart_uploads_parts"
+          ADD CONSTRAINT s3_multipart_uploads_parts_key_check
+          CHECK (
+            key !~ E'[\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F]'
+            AND POSITION(U&'\FFFE' IN key) = 0
+            AND POSITION(U&'\FFFF' IN key) = 0
+          ) NOT VALID
+      $ddl$;
+    END IF;
+  END IF;
+
+  IF EXISTS (
+    SELECT 1
+    FROM pg_constraint c
+    JOIN pg_class t ON t.oid = c.conrelid
+    JOIN pg_namespace n ON n.oid = t.relnamespace
+    WHERE c.conname = 's3_multipart_uploads_parts_key_check'
+      AND n.nspname = 'storage'
+      AND t.relname = 's3_multipart_uploads_parts'
+      AND c.convalidated = false
+  ) THEN
+    EXECUTE 'ALTER TABLE "storage"."s3_multipart_uploads_parts" VALIDATE CONSTRAINT s3_multipart_uploads_parts_key_check';
+  END IF;
+END
+$$;

src/http/plugins/xml.ts

@@ -6,6 +6,38 @@ import xml from 'xml2js'
 type XmlParserOptions = { disableContentParser?: boolean; parseAsArray?: string[] }
 type RequestError = Error & { statusCode?: number }
 
+function isValidXmlCodePoint(codePoint: number): boolean {
+  if (!Number.isInteger(codePoint) || codePoint < 0 || codePoint > 0x10ffff) {
+    return false
+  }
+
+  return (
+    codePoint === 0x9 ||
+    codePoint === 0xa ||
+    codePoint === 0xd ||
+    (codePoint >= 0x20 && codePoint <= 0xd7ff) ||
+    (codePoint >= 0xe000 && codePoint <= 0xfffd) ||
+    (codePoint >= 0x10000 && codePoint <= 0x10ffff)
+  )
+}
+
+function getInvalidXmlNumericEntity(value: string): string | undefined {
+  const numericEntityPattern = /&#([xX][0-9a-fA-F]{1,6}|[0-9]{1,7});/g
+
+  let match = numericEntityPattern.exec(value)
+  while (match) {
+    const rawValue = match[1]
+    const isHex = rawValue[0].toLowerCase() === 'x'
+    const codePoint = Number.parseInt(isHex ? rawValue.slice(1) : rawValue, isHex ? 16 : 10)
+
+    if (!isValidXmlCodePoint(codePoint)) {
+      return match[0]
+    }
+
+    match = numericEntityPattern.exec(value)
+  }
+}
+
 function forcePathAsArray(node: unknown, pathSegments: string[]): void {
   if (pathSegments.length === 0 || node === null || node === undefined) {
     return
@@ -52,8 +84,19 @@ export const xmlParser = fastifyPlugin(
             return
           }
 
+          const xmlBody = typeof body === 'string' ? body : body.toString('utf8')
+          const invalidNumericEntity = getInvalidXmlNumericEntity(xmlBody)
+          if (invalidNumericEntity) {
+            const parseError: RequestError = new Error(
+              `Invalid XML payload: invalid numeric entity ${invalidNumericEntity}`
+            )
+            parseError.statusCode = 400
+            done(parseError)
+            return
+          }
+
           xml.parseString(
-            body,
+            xmlBody,
             {
               explicitArray: false,
               trim: true,

src/http/routes/object/getSignedObject.ts

@@ -4,6 +4,7 @@ import { getConfig } from '../../../config'
 import { SignedToken, verifyJWT } from '../../../internal/auth'
 import { getJwtSecret } from '../../../internal/database'
 import { ERRORS } from '../../../internal/errors'
+import { doesSignedTokenMatchRequestPath } from '../../../internal/http'
 import { ROUTE_OPERATIONS } from '../operations'
 
 const { storageS3Bucket } = getConfig()
@@ -71,9 +72,8 @@ export default async function routes(fastify: FastifyInstance) {
       }
 
       const { url, exp } = payload
-      const path = `${request.params.bucketName}/${request.params['*']}`
 
-      if (url !== path) {
+      if (!doesSignedTokenMatchRequestPath(request.raw.url, '/object/sign', url)) {
         throw ERRORS.InvalidSignature()
       }
 

src/http/routes/object/getSignedURL.ts

@@ -67,7 +67,6 @@ export default async function routes(fastify: FastifyInstance) {
       const objectName = request.params['*']
       const { expiresIn } = request.body
 
-      const urlPath = request.url.split('?').shift()
       const imageTransformationEnabled = await isImageTransformationEnabled(request.tenantId)
 
       const transformationOptions = imageTransformationEnabled
@@ -82,7 +81,7 @@ export default async function routes(fastify: FastifyInstance) {
 
       const signedURL = await request.storage
         .from(bucketName)
-        .signObjectUrl(objectName, urlPath as string, expiresIn, transformationOptions)
+        .signObjectUrl(objectName, expiresIn, transformationOptions)
 
       return response.status(200).send({ signedURL })
     }

src/http/routes/object/getSignedUploadURL.ts

@@ -67,11 +67,9 @@ export default async function routes(fastify: FastifyInstance) {
       const objectName = request.params['*']
       const owner = request.owner
 
-      const urlPath = `${bucketName}/${objectName}`
-
       const signedUpload = await request.storage
         .from(bucketName)
-        .signUploadObjectUrl(objectName, urlPath as string, uploadSignedUrlExpirationTime, owner, {
+        .signUploadObjectUrl(objectName, uploadSignedUrlExpirationTime, owner, {
           upsert: request.headers['x-upsert'] === 'true',
         })
 

src/http/routes/object/uploadSignedObject.ts

@@ -1,4 +1,8 @@
 import fastifyMultipart from '@fastify/multipart'
+import { SignedUploadToken, verifyJWT } from '@internal/auth'
+import { getJwtSecret } from '@internal/database'
+import { ERRORS } from '@internal/errors'
+import { doesSignedTokenMatchRequestPath } from '@internal/http'
 import { FastifyInstance } from 'fastify'
 import { FromSchema } from 'json-schema-to-ts'
 import { ROUTE_OPERATIONS } from '../operations'
@@ -83,9 +87,21 @@ export default async function routes(fastify: FastifyInstance) {
       const { bucketName } = request.params
       const objectName = request.params['*']
 
-      const { owner, upsert } = await request.storage
-        .from(bucketName)
-        .verifyObjectSignature(token, objectName)
+      let payload: SignedUploadToken
+      const { secret: jwtSecret, jwks } = await getJwtSecret(request.tenantId)
+
+      try {
+        payload = (await verifyJWT(token, jwtSecret, jwks)) as SignedUploadToken
+      } catch (e) {
+        const err = e as Error
+        throw ERRORS.InvalidJWT(err)
+      }
+
+      const { owner, upsert, url } = payload
+
+      if (!doesSignedTokenMatchRequestPath(request.raw.url, '/object/upload/sign', url)) {
+        throw ERRORS.InvalidSignature()
+      }
 
       const { objectMetadata, path } = await request.storage
         .asSuperUser()

src/http/routes/render/renderSignedImage.ts

@@ -1,6 +1,7 @@
 import { SignedToken, verifyJWT } from '@internal/auth'
 import { getJwtSecret, getTenantConfig } from '@internal/database'
 import { ERRORS } from '@internal/errors'
+import { doesSignedTokenMatchRequestPath } from '@internal/http'
 import { ImageRenderer } from '@storage/renderer'
 import { FastifyInstance } from 'fastify'
 import { FromSchema } from 'json-schema-to-ts'
@@ -68,10 +69,7 @@ export default async function routes(fastify: FastifyInstance) {
       }
 
       const { url, transformations, exp } = payload
-
-      const path = `${request.params.bucketName}/${request.params['*']}`
-
-      if (url !== path) {
+      if (!doesSignedTokenMatchRequestPath(request.raw.url, '/render/image/sign', url)) {
         throw ERRORS.InvalidSignature()
       }
 

src/internal/database/migrations/types.ts

@@ -56,4 +56,5 @@ export const DBMigration = {
   'drop-index-object-level': 54,
   'prevent-direct-deletes': 55,
   'fix-optimized-search-function': 56,
-}
+  'unicode-object-names': 57,
+} as const

src/internal/errors/codes.ts

@@ -1,3 +1,4 @@
+import { safeEncodeURIComponent } from '../http'
 import { StorageBackendError } from './storage-error'
 
 export enum ErrorCode {
@@ -319,12 +320,20 @@ export const ERRORS = {
       originalError: e,
     }),
 
+  InvalidObjectName: (e?: Error) =>
+    new StorageBackendError({
+      code: ErrorCode.InvalidKey,
+      httpStatusCode: 400,
+      message: 'Invalid object name',
+      originalError: e,
+    }),
+
   InvalidKey: (key: string, e?: Error) =>
     new StorageBackendError({
       code: ErrorCode.InvalidKey,
       resource: key,
       httpStatusCode: 400,
-      message: `Invalid key: ${key}`,
+      message: `Invalid key: ${safeEncodeURIComponent(key)}`,
       originalError: e,
     }),
 

src/internal/http/index.ts

@@ -1 +1,2 @@
 export * from './agent'
+export * from './url'