diff --git a/.github/workflows/RavenClient.yml b/.github/workflows/RavenClient.yml index a3e4581a..c0c48d2d 100644 --- a/.github/workflows/RavenClient.yml +++ b/.github/workflows/RavenClient.yml @@ -2,9 +2,9 @@ name: tests/python on: push: - branches: [v7.1] + branches: [v7.2] pull_request: - branches: [v7.1] + branches: [v7.2] schedule: - cron: '0 10 * * *' workflow_dispatch: @@ -30,7 +30,7 @@ jobs: strategy: matrix: python-version: [ '3.10' ,'3.11', '3.12', '3.13', '3.14'] - serverVersion: [ '7.1', '7.2' ] + serverVersion: [ '7.2' ] fail-fast: false steps: diff --git a/README.md b/README.md index dcca358a..edd34065 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ pip install ravendb ```` ## Introduction and changelog -Python client API (v7.1) for [RavenDB](https://ravendb.net/), a NoSQL document database. +Python client API (v7.2) for [RavenDB](https://ravendb.net/), a NoSQL document database. **Type-hinted entire project and API results** - using the API is now much more comfortable with IntelliSense diff --git a/README_pypi.md b/README_pypi.md index 9dbc2da8..cefc905b 100644 --- a/README_pypi.md +++ b/README_pypi.md @@ -8,7 +8,7 @@ pip install ravendb ``` ## Introduction -Python client API (v7.1) for [RavenDB](https://ravendb.net/) , a NoSQL document database. +Python client API (v7.2) for [RavenDB](https://ravendb.net/) , a NoSQL document database. **Type-hinted entire project and API results** - using the API is now much more comfortable with IntelliSense diff --git a/ravendb/documents/bulk_insert_operation.py b/ravendb/documents/bulk_insert_operation.py index 93a2ffb8..0efdd4ef 100644 --- a/ravendb/documents/bulk_insert_operation.py +++ b/ravendb/documents/bulk_insert_operation.py @@ -30,6 +30,7 @@ from ravendb.json.metadata_as_dictionary import MetadataAsDictionary from ravendb.documents.commands.batches import CommandType from ravendb.documents.commands.bulkinsert import GetNextOperationIdCommand, KillOperationCommand +from ravendb.documents.operations.attachments import StoreAttachmentParameters from ravendb.exceptions.documents.bulkinsert import BulkInsertAbortedException from ravendb.documents.identity.hilo import GenerateEntityIdOnTheClient from ravendb.tools.utils import Utils @@ -615,13 +616,16 @@ def __init__(self, operation: BulkInsertOperation, key: str): self.key = key def store(self, name: str, attachment_bytes: bytes, content_type: Optional[str] = None) -> None: - self.operation._attachments_operation.store(self.key, name, attachment_bytes, content_type) + self.store_with_parameters(StoreAttachmentParameters(name, attachment_bytes, content_type=content_type)) + + def store_with_parameters(self, parameters: StoreAttachmentParameters) -> None: + self.operation._attachments_operation.store(self.key, parameters) class AttachmentsBulkInsertOperation: def __init__(self, operation: BulkInsertOperation): self.operation = operation - def store(self, key: str, name: str, attachment_bytes: bytes, content_type: Optional[str] = None): + def store(self, key: str, parameters: StoreAttachmentParameters): release_lock_callback = self.operation._concurrency_check() try: self.operation._end_previous_command_if_needed() @@ -634,22 +638,30 @@ def store(self, key: str, name: str, attachment_bytes: bytes, content_type: Opti if not self.operation._first: self.operation._write_comma() + self.operation._first = False + self.operation._in_progress_command = CommandType.NONE + self.operation._write_string_no_escape('{"Id":"') self.operation._write_string(key) self.operation._write_string_no_escape('","Type":"AttachmentPUT","Name":"') - self.operation._write_string(name) + self.operation._write_string(parameters.name) - if content_type: - self.operation._write_string_no_escape('","ContentType:"') - self.operation._write_string(content_type) + if parameters.content_type: + self.operation._write_string_no_escape('","ContentType":"') + self.operation._write_string(parameters.content_type) self.operation._write_string_no_escape('","ContentLength":') - self.operation._write_string_no_escape(str(len(attachment_bytes))) + self.operation._write_string_no_escape(str(len(parameters.stream))) + + if parameters.remote_parameters is not None: + self.operation._write_string_no_escape(',"RemoteParameters":') + self.operation._write_string_no_escape(json.dumps(parameters.remote_parameters.to_json())) + self.operation._write_string_no_escape("}") self.operation._flush_if_needed() - self.operation._current_data_buffer += bytearray(attachment_bytes) + self.operation._current_data_buffer += bytearray(parameters.stream) self.operation._flush_if_needed() diff --git a/ravendb/documents/commands/batches.py b/ravendb/documents/commands/batches.py index 53538f1c..613c0fa4 100644 --- a/ravendb/documents/commands/batches.py +++ b/ravendb/documents/commands/batches.py @@ -23,6 +23,7 @@ if TYPE_CHECKING: from ravendb.documents.conventions import DocumentConventions + from ravendb.documents.operations.attachments import RemoteAttachmentParameters from ravendb.documents.operations.patch import PatchRequest from ravendb.documents.session.document_session_operations.in_memory_document_session_operations import ( InMemoryDocumentSessionOperations, @@ -117,6 +118,8 @@ def __init__( if self.__attachment_streams is None: self.__attachment_streams = [] stream = command.stream + if stream is None: + continue # remote-only attachment — no stream to track if stream in self.__attachment_streams: raise RuntimeError( "It is forbidden to re-use the same stream for more than one attachment. " @@ -139,12 +142,13 @@ def create_request(self, node: ServerNode) -> requests.Request: for command in self.__commands: if command.command_type == CommandType.ATTACHMENT_PUT: command: PutAttachmentCommandData - files[command.name] = ( - command.name, - command.stream, - command.content_type, - {"Command-Type": "AttachmentStream"}, - ) + if command.stream is not None: + files[command.name] = ( + command.name, + command.stream, + command.content_type, + {"Command-Type": "AttachmentStream"}, + ) if not request.data: request.data = {"Commands": []} request.data["Commands"].append(command.serialize(self.__conventions)) @@ -526,7 +530,17 @@ def serialize(self, conventions: DocumentConventions) -> Dict: class PutAttachmentCommandData(CommandData): - def __init__(self, document_id: str, name: str, stream: bytes, content_type: str, change_vector: str): + def __init__( + self, + document_id: str, + name: str, + stream: bytes, + content_type: str, + change_vector: str, + remote_parameters: Optional["RemoteAttachmentParameters"] = None, + hash: str = None, + size_in_bytes: int = None, + ): if not document_id: raise ValueError(document_id) if not name: @@ -535,6 +549,9 @@ def __init__(self, document_id: str, name: str, stream: bytes, content_type: str super(PutAttachmentCommandData, self).__init__(document_id, name, change_vector, CommandType.ATTACHMENT_PUT) self.__stream = stream self.__content_type = content_type + self.__remote_parameters = remote_parameters + self.__hash = hash + self.__size_in_bytes = size_in_bytes @property def stream(self): @@ -544,14 +561,23 @@ def stream(self): def content_type(self): return self.__content_type + @property + def remote_parameters(self) -> Optional["RemoteAttachmentParameters"]: + return self.__remote_parameters + def serialize(self, conventions: DocumentConventions) -> dict: - return { + result = { "Id": self._key, "Name": self._name, "ContentType": self.__content_type, "ChangeVector": self._change_vector, "Type": str(self._command_type), + "RemoteParameters": self.__remote_parameters.to_json() if self.__remote_parameters is not None else None, + "Hash": self.__hash, } + if self.__size_in_bytes is not None: + result["SizeInBytes"] = self.__size_in_bytes + return result class CopyAttachmentCommandData(CommandData): diff --git a/ravendb/documents/indexes/definitions.py b/ravendb/documents/indexes/definitions.py index b4935eac..27d04376 100644 --- a/ravendb/documents/indexes/definitions.py +++ b/ravendb/documents/indexes/definitions.py @@ -2,7 +2,7 @@ import datetime import enum import re -from enum import Enum +from enum import Enum, IntFlag from abc import ABC from typing import Union, Optional, List, Dict, Set, Iterable from ravendb.documents.indexes.spatial.configuration import SpatialOptions, AutoSpatialOptions @@ -140,6 +140,38 @@ def __str__(self): return self.value +class IndexDefinitionCompareDifferences(IntFlag): + NONE = 0 + MAPS = 1 << 0 + REDUCE = 1 << 1 + FIELDS = 1 << 2 + CONFIGURATION = 1 << 3 + LOCK_MODE = 1 << 4 + PRIORITY = 1 << 5 + STATE = 1 << 6 + ADDITIONAL_SOURCES = 1 << 7 + ADDITIONAL_ASSEMBLIES = 1 << 8 + DEPLOYMENT_MODE = 1 << 12 + COMPOUND_FIELDS = 1 << 13 + ARCHIVED_DATA_PROCESSING_BEHAVIOR = 1 << 14 + SCHEMA_VALIDATION_CONFIGURATION = 1 << 15 + ALL = ( + MAPS + | REDUCE + | FIELDS + | CONFIGURATION + | LOCK_MODE + | PRIORITY + | STATE + | ADDITIONAL_SOURCES + | ADDITIONAL_ASSEMBLIES + | DEPLOYMENT_MODE + | COMPOUND_FIELDS + | ARCHIVED_DATA_PROCESSING_BEHAVIOR + | SCHEMA_VALIDATION_CONFIGURATION + ) + + class GroupByArrayBehavior(Enum): NOT_APPLICABLE = "NotApplicable" BY_CONTENT = "ByContent" @@ -205,6 +237,7 @@ def __init__( pattern_references_collection_name: Optional[str] = None, deployment_mode: Optional[IndexDeploymentMode] = None, search_engine_type: Optional[SearchEngineType] = None, + schema_definitions: Optional[Dict[str, str]] = None, ): super(IndexDefinition, self).__init__(name, priority, state) self.lock_mode = lock_mode @@ -222,6 +255,7 @@ def __init__( self.pattern_references_collection_name = pattern_references_collection_name self.deployment_mode = deployment_mode self.search_engine_type = search_engine_type + self.schema_definitions = schema_definitions @classmethod def from_json(cls, json_dict: dict) -> IndexDefinition: @@ -255,6 +289,7 @@ def from_json(cls, json_dict: dict) -> IndexDefinition: deploy = json_dict.get("DeploymentMode", None) if deploy is not None: result.deployment_mode = IndexDeploymentMode(deploy) + result.schema_definitions = json_dict.get("SchemaDefinitions") return result def to_json(self) -> dict: @@ -278,6 +313,7 @@ def to_json(self) -> dict: "PatternForOutputReduceToCollectionReferences": self.pattern_for_output_reduce_to_collection_references, "PatternReferencesCollectionName": self.pattern_references_collection_name, "DeploymentMode": self.deployment_mode, + "SchemaDefinitions": self.schema_definitions, } @property diff --git a/ravendb/documents/operations/attachments/__init__.py b/ravendb/documents/operations/attachments/__init__.py index 0327e6ad..1dfdcd40 100644 --- a/ravendb/documents/operations/attachments/__init__.py +++ b/ravendb/documents/operations/attachments/__init__.py @@ -1,17 +1,23 @@ from __future__ import annotations +import enum import http import json -from typing import Optional, TYPE_CHECKING, List +import urllib.parse +from datetime import datetime +from typing import Optional, TYPE_CHECKING, List, Dict import requests from ravendb.primitives import constants from ravendb.data.operation import AttachmentType -from ravendb.documents.operations.definitions import IOperation, VoidOperation +from ravendb.documents.operations.backups.settings import S3StorageClass +from ravendb.documents.operations.definitions import IOperation, VoidOperation, MaintenanceOperation from ravendb.http.http_cache import HttpCache from ravendb.http.misc import ResponseDisposeHandling from ravendb.http.raven_command import RavenCommand, RavenCommandResponseType, VoidRavenCommand +from ravendb.http.topology import RaftCommand +from ravendb.util.util import RaftIdGenerator from ravendb.http.server_node import ServerNode from ravendb.tools.utils import Utils @@ -26,10 +32,15 @@ def __init__(self, name: str, hash: str, content_type: str, size: int): self.hash = hash self.content_type = content_type self.size = size + self.remote_parameters: Optional[RemoteAttachmentParameters] = None @classmethod def from_json(cls, json_dict: dict) -> AttachmentName: - return cls(json_dict["Name"], json_dict["Hash"], json_dict["ContentType"], json_dict["Size"]) + obj = cls(json_dict["Name"], json_dict["Hash"], json_dict["ContentType"], json_dict["Size"]) + remote_raw = json_dict.get("RemoteParameters") + if remote_raw is not None: + obj.remote_parameters = RemoteAttachmentParameters.from_json(remote_raw) + return obj class AttachmentDetails(AttachmentName): @@ -86,6 +97,19 @@ def __init__(self, document_id: str, name: str): self.name = name +class StoreAttachmentParameters: + def __init__(self, name: str, stream, content_type: Optional[str] = None, change_vector: Optional[str] = None): + if not name or name.isspace(): + raise ValueError("Attachment name cannot be null or whitespace.") + if stream is None: + raise ValueError("Attachment stream cannot be null.") + self.name = name + self.stream = stream + self.change_vector = change_vector + self.content_type = content_type + self.remote_parameters: Optional[RemoteAttachmentParameters] = None + + class PutAttachmentOperation(IOperation[AttachmentDetails]): def __init__( self, @@ -93,6 +117,7 @@ def __init__( name: str, stream: bytes, content_type: Optional[str] = None, + remote_parameters: Optional[RemoteAttachmentParameters] = None, change_vector: Optional[str] = None, ): super().__init__() @@ -100,8 +125,22 @@ def __init__( self.__name = name self.__stream = stream self.__content_type = content_type + self.__remote_parameters = remote_parameters self.__change_vector = change_vector + @classmethod + def from_store_attachment_parameters( + cls, document_id: str, parameters: StoreAttachmentParameters + ) -> PutAttachmentOperation: + return cls( + document_id, + parameters.name, + parameters.stream, + parameters.content_type, + parameters.remote_parameters, + parameters.change_vector, + ) + def get_command(self, store, conventions, cache=None): return self.__PutAttachmentCommand( self.__document_id, @@ -109,10 +148,19 @@ def get_command(self, store, conventions, cache=None): self.__stream, self.__content_type, self.__change_vector, + self.__remote_parameters, ) class __PutAttachmentCommand(RavenCommand[AttachmentDetails]): - def __init__(self, document_id: str, name: str, stream: bytes, content_type: str, change_vector: str): + def __init__( + self, + document_id: str, + name: str, + stream: bytes, + content_type: str, + change_vector: str, + remote_parameters: Optional[RemoteAttachmentParameters] = None, + ): super().__init__(AttachmentDetails) if not document_id: @@ -126,6 +174,7 @@ def __init__(self, document_id: str, name: str, stream: bytes, content_type: str self.__stream = stream self.__content_type = content_type self.__change_vector = change_vector + self.__remote_parameters = remote_parameters def create_request(self, node: ServerNode) -> requests.Request: url = ( @@ -136,6 +185,10 @@ def create_request(self, node: ServerNode) -> requests.Request: if not self.__content_type.isspace(): url += f"&contentType={Utils.escape(self.__content_type, True, False)}" + if self.__remote_parameters is not None: + url += f"&remoteAt={Utils.escape(Utils.datetime_to_string(self.__remote_parameters.at), True, False)}" + url += f"&remoteIdentifier={Utils.escape(self.__remote_parameters.identifier, True, False)}" + request = requests.Request("PUT", url) if isinstance(self.__stream, (bytes, bytearray)): request.files = {self.__name: (self.__name, self.__stream, self.__content_type)} @@ -204,11 +257,32 @@ def is_read_request(self) -> bool: def process_response(self, cache: HttpCache, response: requests.Response, url) -> http.ResponseDisposeHandling: content_type = response.headers.get("Content-Type") change_vector = response.headers.get(constants.Headers.ETAG) - hash = response.headers.get("Attachment-Hash") - size = response.headers.get("Attachment-Size", 0) + hash = response.headers.get(constants.Headers.ATTACHMENT_HASH) + size = response.headers.get(constants.Headers.ATTACHMENT_SIZE, 0) + + remote_identifier_raw = response.headers.get(constants.Headers.ATTACHMENT_REMOTE_PARAMETERS_IDENTIFIER) + remote_identifier = urllib.parse.unquote(remote_identifier_raw) if remote_identifier_raw else None + remote_parameters = None + if remote_identifier: + at_raw = response.headers.get(constants.Headers.ATTACHMENT_REMOTE_PARAMETERS_AT) + if at_raw is None: + raise RuntimeError( + f"Attachment remote parameters header '{constants.Headers.ATTACHMENT_REMOTE_PARAMETERS_AT}' " + f"is missing for attachment '{self.__name}' on document '{self.__document_id}'." + ) + flags_raw = response.headers.get(constants.Headers.ATTACHMENT_REMOTE_PARAMETERS_FLAGS) + if flags_raw is None: + raise RuntimeError( + f"Attachment remote parameters header '{constants.Headers.ATTACHMENT_REMOTE_PARAMETERS_FLAGS}' " + f"is missing for attachment '{self.__name}' on document '{self.__document_id}'." + ) + remote_parameters = RemoteAttachmentParameters(remote_identifier, Utils.string_to_datetime(at_raw)) + remote_parameters.flags = RemoteAttachmentFlags.from_str(flags_raw) + attachment_details = AttachmentDetails( self.__name, hash, content_type, size, change_vector, self.__document_id ) + attachment_details.remote_parameters = remote_parameters self.result = CloseableAttachmentResult(response, attachment_details) return ResponseDisposeHandling.MANUALLY @@ -280,3 +354,333 @@ def create_request(self, server_node): ) self._add_change_vector_if_not_none(self.__change_vector, request) return request + + +class DeleteAttachmentsOperation(VoidOperation): + def __init__(self, attachments: List[AttachmentRequest]): + self.__attachments = attachments + + def get_command(self, store: "DocumentStore", conventions: "DocumentConventions", cache=None) -> VoidRavenCommand: + return self.__DeleteAttachmentsCommand(self.__attachments) + + class __DeleteAttachmentsCommand(VoidRavenCommand): + def __init__(self, attachments: List[AttachmentRequest]): + super().__init__() + if attachments is None: + raise ValueError("Attachments cannot be None") + self.__attachments = attachments + + def create_request(self, node: ServerNode) -> requests.Request: + return requests.Request( + "DELETE", + f"{node.url}/databases/{node.database}/attachments/bulk", + data={"Attachments": [{"DocumentId": a.document_id, "Name": a.name} for a in self.__attachments]}, + ) + + +class RemoteAttachmentFlags(enum.IntFlag): + NONE = 0 + REMOTE = 0x1 + + def to_str(self) -> str: + """Returns the PascalCase string representation matching C# Flags.ToString().""" + return self.name.capitalize() if self != RemoteAttachmentFlags.NONE else "None" + + @classmethod + def from_str(cls, value: str) -> RemoteAttachmentFlags: + """Parses a PascalCase string from the server (e.g. 'None', 'Remote').""" + return cls[value.upper()] + + +class RemoteAttachmentsS3Settings: + def __init__( + self, + aws_access_key: str = None, + aws_secret_key: str = None, + aws_session_token: str = None, + aws_region_name: str = None, + remote_folder_name: str = None, + bucket_name: str = None, + custom_server_url: str = None, + force_path_style: bool = None, + storage_class: Optional[S3StorageClass] = None, + ): + self.aws_access_key = aws_access_key + self.aws_secret_key = aws_secret_key + self.aws_session_token = aws_session_token + self.aws_region_name = aws_region_name + self.remote_folder_name = remote_folder_name + self.bucket_name = bucket_name + self.custom_server_url = custom_server_url + self.force_path_style = force_path_style + self.storage_class = storage_class + + @classmethod + def from_json(cls, json_dict: dict) -> RemoteAttachmentsS3Settings: + storage_class_raw = json_dict.get("StorageClass") + return cls( + json_dict.get("AwsAccessKey"), + json_dict.get("AwsSecretKey"), + json_dict.get("AwsSessionToken"), + json_dict.get("AwsRegionName"), + json_dict.get("RemoteFolderName"), + json_dict.get("BucketName"), + json_dict.get("CustomServerUrl"), + json_dict.get("ForcePathStyle"), + S3StorageClass(storage_class_raw) if storage_class_raw is not None else None, + ) + + def to_json(self) -> dict: + result = { + "AwsAccessKey": self.aws_access_key, + "AwsSecretKey": self.aws_secret_key, + "AwsSessionToken": self.aws_session_token, + "AwsRegionName": self.aws_region_name, + "RemoteFolderName": self.remote_folder_name, + "BucketName": self.bucket_name, + "CustomServerUrl": self.custom_server_url, + "ForcePathStyle": self.force_path_style, + } + if self.storage_class is not None: + result["StorageClass"] = self.storage_class.value + return result + + +class RemoteAttachmentsAzureSettings: + def __init__( + self, + storage_container: str = None, + remote_folder_name: str = None, + account_name: str = None, + account_key: str = None, + sas_token: str = None, + ): + self.storage_container = storage_container + self.remote_folder_name = remote_folder_name + self.account_name = account_name + self.account_key = account_key + self.sas_token = sas_token + + @classmethod + def from_json(cls, json_dict: dict) -> RemoteAttachmentsAzureSettings: + return cls( + json_dict.get("StorageContainer"), + json_dict.get("RemoteFolderName"), + json_dict.get("AccountName"), + json_dict.get("AccountKey"), + json_dict.get("SasToken"), + ) + + def to_json(self) -> dict: + return { + "StorageContainer": self.storage_container, + "RemoteFolderName": self.remote_folder_name, + "AccountName": self.account_name, + "AccountKey": self.account_key, + "SasToken": self.sas_token, + } + + +class RemoteAttachmentsDestinationConfiguration: + def __init__( + self, + disabled: bool = False, + s3_settings: Optional[RemoteAttachmentsS3Settings] = None, + azure_settings: Optional[RemoteAttachmentsAzureSettings] = None, + ): + self.disabled = disabled + self.s3_settings = s3_settings + self.azure_settings = azure_settings + + @classmethod + def from_json(cls, json_dict: dict) -> RemoteAttachmentsDestinationConfiguration: + s3_raw = json_dict.get("S3Settings") + azure_raw = json_dict.get("AzureSettings") + return cls( + json_dict.get("Disabled", False), + RemoteAttachmentsS3Settings.from_json(s3_raw) if s3_raw is not None else None, + RemoteAttachmentsAzureSettings.from_json(azure_raw) if azure_raw is not None else None, + ) + + def _is_s3_configured(self) -> bool: + return self.s3_settings is not None and bool(self.s3_settings.bucket_name) + + def _is_azure_configured(self) -> bool: + return ( + self.azure_settings is not None + and bool(self.azure_settings.account_name) + and bool(self.azure_settings.storage_container) + ) + + def assert_configuration(self, key: str, database_name: str = None) -> None: + db_str = f" for database '{database_name}'" if database_name else "" + if not self._is_s3_configured() and not self._is_azure_configured(): + raise ValueError(f"Exactly one uploader for RemoteAttachmentsConfiguration{db_str} must be configured.") + if self._is_s3_configured() and self._is_azure_configured(): + raise ValueError(f"Only one uploader for RemoteAttachmentsConfiguration{db_str} can be configured.") + + def to_json(self) -> dict: + return { + "Disabled": self.disabled, + "S3Settings": self.s3_settings.to_json() if self.s3_settings is not None else None, + "AzureSettings": self.azure_settings.to_json() if self.azure_settings is not None else None, + } + + +class RemoteAttachmentsConfiguration: + def __init__( + self, + destinations: Optional[Dict[str, RemoteAttachmentsDestinationConfiguration]] = None, + check_frequency_in_sec: Optional[int] = None, + max_items_to_process: Optional[int] = None, + concurrent_uploads: Optional[int] = None, + disabled: bool = False, + ): + self.destinations = destinations if destinations is not None else {} + self.check_frequency_in_sec = check_frequency_in_sec + self.max_items_to_process = max_items_to_process + self.concurrent_uploads = concurrent_uploads + self.disabled = disabled + + @classmethod + def from_json(cls, json_dict: dict) -> RemoteAttachmentsConfiguration: + destinations_raw = json_dict.get("Destinations") or {} + destinations = {k: RemoteAttachmentsDestinationConfiguration.from_json(v) for k, v in destinations_raw.items()} + return cls( + destinations, + json_dict.get("CheckFrequencyInSec"), + json_dict.get("MaxItemsToProcess"), + json_dict.get("ConcurrentUploads"), + json_dict.get("Disabled", False), + ) + + def assert_configuration(self, database_name: str = None) -> None: + db_str = f" for database '{database_name}'" if database_name else "" + + if self.check_frequency_in_sec is not None and self.check_frequency_in_sec <= 0: + raise ValueError(f"Remote attachments check frequency{db_str} must be greater than 0.") + if self.max_items_to_process is not None and self.max_items_to_process <= 0: + raise ValueError(f"Max items to process{db_str} must be greater than 0.") + if self.concurrent_uploads is not None and self.concurrent_uploads <= 0: + raise ValueError(f"Concurrent attachments uploads{db_str} must be greater than 0.") + + if not self.destinations: + return + + seen_keys = set() + for key, dest in self.destinations.items(): + lower_key = key.lower() + if lower_key in seen_keys: + raise ValueError( + f"Destination key '{key}' is duplicate. Duplicate keys are not allowed in remote attachments configuration{db_str}." + ) + seen_keys.add(lower_key) + if dest is None: + raise ValueError(f"Destination configuration for key {key} is null{db_str}.") + dest.assert_configuration(key, database_name) + + def to_json(self) -> dict: + return { + "Destinations": {k: v.to_json() for k, v in self.destinations.items()} if self.destinations else {}, + "CheckFrequencyInSec": self.check_frequency_in_sec, + "MaxItemsToProcess": self.max_items_to_process, + "ConcurrentUploads": self.concurrent_uploads, + "Disabled": self.disabled, + } + + +class RemoteAttachmentParameters: + def __init__(self, identifier: str, at: datetime): + if not identifier or identifier.isspace(): + raise ValueError("Attachment identifier cannot be None or whitespace.") + if at is None or at == datetime.min: + raise ValueError("Attachment upload date cannot be default value.") + self.identifier = identifier + self.at = at + self.flags = RemoteAttachmentFlags.NONE + + @classmethod + def from_json(cls, json_dict: dict) -> RemoteAttachmentParameters: + obj = cls.__new__(cls) + obj.identifier = json_dict["Identifier"] + obj.at = Utils.string_to_datetime(json_dict["At"]) + obj.flags = RemoteAttachmentFlags.from_str(json_dict.get("Flags", "None")) + return obj + + def to_json(self) -> dict: + return { + "At": Utils.datetime_to_string(self.at), + "Identifier": self.identifier, + "Flags": self.flags.to_str(), + } + + +class ConfigureRemoteAttachmentsOperationResult: + def __init__(self, raft_command_index: Optional[int] = None): + self.raft_command_index = raft_command_index + + @classmethod + def from_json(cls, json_dict: dict) -> ConfigureRemoteAttachmentsOperationResult: + return cls(json_dict.get("RaftCommandIndex")) + + +class ConfigureRemoteAttachmentsOperation(MaintenanceOperation[ConfigureRemoteAttachmentsOperationResult]): + def __init__(self, configuration: RemoteAttachmentsConfiguration): + if configuration is None: + raise ValueError("Configuration cannot be None.") + configuration.assert_configuration() + self.__configuration = configuration + + def get_command( + self, conventions: "DocumentConventions" + ) -> RavenCommand[ConfigureRemoteAttachmentsOperationResult]: + return self.__ConfigureAttachmentsRemoteCommand(self.__configuration) + + class __ConfigureAttachmentsRemoteCommand(RavenCommand[ConfigureRemoteAttachmentsOperationResult], RaftCommand): + def __init__(self, configuration: RemoteAttachmentsConfiguration): + super().__init__(ConfigureRemoteAttachmentsOperationResult) + if configuration is None: + raise ValueError("Configuration cannot be None.") + self.__configuration = configuration + + def is_read_request(self) -> bool: + return False + + def create_request(self, node: ServerNode) -> requests.Request: + request = requests.Request( + "PUT", + f"{node.url}/databases/{node.database}/admin/attachments/remote/config", + data=self.__configuration.to_json(), + ) + return request + + def set_response(self, response: Optional[str], from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + self.result = ConfigureRemoteAttachmentsOperationResult.from_json(json.loads(response)) + + def get_raft_unique_request_id(self) -> str: + return RaftIdGenerator.new_id() + + +class GetRemoteAttachmentsConfigurationOperation(MaintenanceOperation[RemoteAttachmentsConfiguration]): + def get_command(self, conventions: "DocumentConventions") -> RavenCommand[RemoteAttachmentsConfiguration]: + return self.__GetRemoteAttachmentsConfigurationCommand() + + class __GetRemoteAttachmentsConfigurationCommand(RavenCommand[RemoteAttachmentsConfiguration]): + def __init__(self): + super().__init__(RemoteAttachmentsConfiguration) + + def is_read_request(self) -> bool: + return True + + def create_request(self, node: ServerNode) -> requests.Request: + return requests.Request( + "GET", + f"{node.url}/databases/{node.database}/admin/attachments/remote/config", + ) + + def set_response(self, response: Optional[str], from_cache: bool) -> None: + if response is None: + return + self.result = RemoteAttachmentsConfiguration.from_json(json.loads(response)) diff --git a/ravendb/documents/operations/backups/settings.py b/ravendb/documents/operations/backups/settings.py index 12c16629..565e3df8 100644 --- a/ravendb/documents/operations/backups/settings.py +++ b/ravendb/documents/operations/backups/settings.py @@ -12,6 +12,18 @@ class BackupType(Enum): SNAPSHOT = "Snapshot" +class S3StorageClass(Enum): + DEEP_ARCHIVE = "DeepArchive" + GLACIER = "Glacier" + GLACIER_INSTANT_RETRIEVAL = "GlacierInstantRetrieval" + INTELLIGENT_TIERING = "IntelligentTiering" + ONE_ZONE_INFREQUENT_ACCESS = "OneZoneInfrequentAccess" + REDUCED_REDUNDANCY = "ReducedRedundancy" + STANDARD = "Standard" + STANDARD_INFREQUENT_ACCESS = "StandardInfrequentAccess" + EXPRESS_ONE_ZONE = "ExpressOneZone" + + class CompressionLevel(Enum): OPTIMAL = "Optimal" FASTEST = "Fastest" diff --git a/ravendb/documents/operations/schema_validation/__init__.py b/ravendb/documents/operations/schema_validation/__init__.py new file mode 100644 index 00000000..2e70176c --- /dev/null +++ b/ravendb/documents/operations/schema_validation/__init__.py @@ -0,0 +1,239 @@ +from __future__ import annotations + +import json +from datetime import datetime +from typing import Optional, Dict, Any, List + +import requests + +from ravendb.http.raven_command import RavenCommand +from ravendb.http.server_node import ServerNode +from ravendb.http.topology import RaftCommand +from ravendb.documents.conventions import DocumentConventions +from ravendb.documents.operations.definitions import MaintenanceOperation, OperationIdResult +from ravendb.util.util import RaftIdGenerator + + +class SchemaDefinition: + def __init__( + self, + schema: str = None, + disabled: bool = False, + last_modified_time: Optional[datetime] = None, + ): + self.schema = schema + self.disabled = disabled + self.last_modified_time = last_modified_time or datetime.utcnow() + + def to_json(self) -> Dict[str, Any]: + return { + "Schema": self.schema, + "Disabled": self.disabled, + "LastModifiedTime": self.last_modified_time.isoformat(), + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "SchemaDefinition": + last_modified_raw = json_dict.get("LastModifiedTime") + last_modified = datetime.fromisoformat(last_modified_raw) if last_modified_raw else None + return cls( + schema=json_dict.get("Schema"), + disabled=json_dict.get("Disabled", False), + last_modified_time=last_modified, + ) + + +class SchemaValidationConfiguration: + def __init__( + self, + disabled: bool = False, + validators_per_collection: Optional[Dict[str, SchemaDefinition]] = None, + ): + self.disabled = disabled + self.validators_per_collection: Dict[str, SchemaDefinition] = validators_per_collection or {} + + def to_json(self) -> Dict[str, Any]: + validators = {k: v.to_json() for k, v in self.validators_per_collection.items()} + return { + "Disabled": self.disabled, + "ValidatorsPerCollection": validators, + } + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "SchemaValidationConfiguration": + raw_validators = json_dict.get("ValidatorsPerCollection") or {} + validators = {k: SchemaDefinition.from_json(v) for k, v in raw_validators.items()} + return cls( + disabled=json_dict.get("Disabled", False), + validators_per_collection=validators, + ) + + +class ConfigureSchemaValidationOperationResult: + def __init__(self, raft_command_index: Optional[int] = None): + self.raft_command_index = raft_command_index + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "ConfigureSchemaValidationOperationResult": + return cls(raft_command_index=json_dict.get("RaftCommandIndex")) + + +class ConfigureSchemaValidationOperation(MaintenanceOperation[ConfigureSchemaValidationOperationResult]): + def __init__(self, configuration: SchemaValidationConfiguration): + if configuration is None: + raise ValueError("configuration cannot be None") + self._configuration = configuration + + def get_command(self, conventions: DocumentConventions) -> RavenCommand: + return self._ConfigureSchemaValidationCommand(self._configuration) + + class _ConfigureSchemaValidationCommand(RavenCommand[ConfigureSchemaValidationOperationResult], RaftCommand): + def __init__(self, configuration: SchemaValidationConfiguration): + super().__init__(ConfigureSchemaValidationOperationResult) + self._configuration = configuration + self._raft_unique_request_id = RaftIdGenerator.new_id() + + def is_read_request(self) -> bool: + return False + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/admin/schema-validation/config" + request = requests.Request("POST", url) + request.data = json.dumps(self._configuration.to_json()) + request.headers = {"Content-Type": "application/json"} + return request + + def set_response(self, response: Optional[str], from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + self.result = ConfigureSchemaValidationOperationResult.from_json(json.loads(response)) + + def get_raft_unique_request_id(self) -> str: + return self._raft_unique_request_id + + +class GetSchemaValidationConfiguration(MaintenanceOperation[Optional[SchemaValidationConfiguration]]): + def get_command(self, conventions: DocumentConventions) -> RavenCommand: + return self._GetSchemaValidationCommand() + + class _GetSchemaValidationCommand(RavenCommand[Optional[SchemaValidationConfiguration]]): + def __init__(self): + super().__init__(SchemaValidationConfiguration) + + def is_read_request(self) -> bool: + return True + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/schema-validation/config" + return requests.Request("GET", url) + + def set_response(self, response: Optional[str], from_cache: bool) -> None: + if response is None: + self.result = None + return + self.result = SchemaValidationConfiguration.from_json(json.loads(response)) + + +class ValidateSchemaProgress: + def __init__(self, error_count: int = 0, validated_count: int = 0): + self.error_count = error_count + self.validated_count = validated_count + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "ValidateSchemaProgress": + return cls( + error_count=json_dict.get("ErrorCount", 0), + validated_count=json_dict.get("ValidatedCount", 0), + ) + + +class ValidateSchemaResult(ValidateSchemaProgress): + def __init__( + self, + error_count: int = 0, + validated_count: int = 0, + errors: Optional[Dict[str, str]] = None, + last_etag: int = 0, + ): + super().__init__(error_count=error_count, validated_count=validated_count) + self.errors: Dict[str, str] = errors or {} + self.last_etag = last_etag + + @classmethod + def from_json(cls, json_dict: Dict[str, Any]) -> "ValidateSchemaResult": + return cls( + error_count=json_dict.get("ErrorCount", 0), + validated_count=json_dict.get("ValidatedCount", 0), + errors=json_dict.get("Errors") or {}, + last_etag=json_dict.get("LastEtag", 0), + ) + + +class StartSchemaValidationOperation(MaintenanceOperation[OperationIdResult]): + class Parameters: + def __init__( + self, + schema_definition: str, + collection: str, + max_error_messages: Optional[int] = None, + max_documents_to_validate: Optional[int] = None, + start_etag: Optional[int] = None, + ): + self.schema_definition = schema_definition + self.collection = collection + self.max_error_messages = max_error_messages + self.max_documents_to_validate = max_documents_to_validate + self.start_etag = start_etag + + def to_json(self) -> Dict[str, Any]: + d: Dict[str, Any] = { + "SchemaDefinition": self.schema_definition, + "Collection": self.collection, + } + if self.max_error_messages is not None: + d["MaxErrorMessages"] = self.max_error_messages + if self.max_documents_to_validate is not None: + d["MaxDocumentsToValidate"] = self.max_documents_to_validate + if self.start_etag is not None: + d["StartEtag"] = self.start_etag + return d + + def __init__(self, parameters: "StartSchemaValidationOperation.Parameters"): + if parameters is None: + raise ValueError("parameters cannot be None") + if not parameters.schema_definition or not parameters.schema_definition.strip(): + raise ValueError("Schema must be provided.") + if not parameters.collection or not parameters.collection.strip(): + raise ValueError("Collection must be provided.") + if parameters.max_error_messages is not None and parameters.max_error_messages < 0: + raise ValueError("max_error_messages must be >= 0.") + if parameters.max_documents_to_validate is not None and parameters.max_documents_to_validate <= 0: + raise ValueError("max_documents_to_validate must be > 0.") + self._parameters = parameters + + def get_command(self, conventions: DocumentConventions) -> RavenCommand: + return self._StartSchemaValidationCommand(self._parameters) + + class _StartSchemaValidationCommand(RavenCommand[OperationIdResult], RaftCommand): + def __init__(self, parameters: "StartSchemaValidationOperation.Parameters"): + super().__init__(OperationIdResult) + self._parameters = parameters + self._raft_unique_request_id = RaftIdGenerator.new_id() + + def is_read_request(self) -> bool: + return False + + def create_request(self, node: ServerNode) -> requests.Request: + url = f"{node.url}/databases/{node.database}/schema-validation/validate" + request = requests.Request("POST", url) + request.data = json.dumps(self._parameters.to_json()) + request.headers = {"Content-Type": "application/json"} + return request + + def set_response(self, response: Optional[str], from_cache: bool) -> None: + if response is None: + self._throw_invalid_response() + self.result = OperationIdResult.from_json(json.loads(response)) + + def get_raft_unique_request_id(self) -> str: + return self._raft_unique_request_id diff --git a/ravendb/documents/operations/statistics.py b/ravendb/documents/operations/statistics.py index 772a586d..e29c8bf5 100644 --- a/ravendb/documents/operations/statistics.py +++ b/ravendb/documents/operations/statistics.py @@ -261,6 +261,7 @@ def from_json(cls, json_dict: Dict) -> DetailedDatabaseStatistics: detailed_database_stats.count_of_identities = json_dict["CountOfIdentities"] detailed_database_stats.count_of_compare_exchange = json_dict["CountOfCompareExchange"] detailed_database_stats.count_of_compare_exchange_tombstones = json_dict["CountOfCompareExchangeTombstones"] + detailed_database_stats.count_of_remote_attachments = json_dict.get("CountOfRemoteAttachments", 0) return detailed_database_stats diff --git a/ravendb/documents/session/document_session.py b/ravendb/documents/session/document_session.py index f288b546..327da2ea 100644 --- a/ravendb/documents/session/document_session.py +++ b/ravendb/documents/session/document_session.py @@ -58,6 +58,7 @@ GetAttachmentOperation, AttachmentName, CloseableAttachmentResult, + StoreAttachmentParameters, ) from ravendb.documents.operations.batch import BatchOperation from ravendb.documents.operations.executor import OperationExecutor, SessionOperationExecutor @@ -1252,6 +1253,31 @@ def store( stream: bytes, content_type: str = None, change_vector: str = None, + ): + self._store_internal(entity_or_document_id, name, stream, content_type, change_vector) + + def store_with_parameters( + self, + entity_or_document_id: Union[object, str], + parameters: StoreAttachmentParameters, + ): + self._store_internal( + entity_or_document_id, + parameters.name, + parameters.stream, + parameters.content_type, + parameters.change_vector, + parameters.remote_parameters, + ) + + def _store_internal( + self, + entity_or_document_id: Union[object, str], + name: str, + stream: bytes, + content_type: str = None, + change_vector: str = None, + remote_parameters: Optional["RemoteAttachmentParameters"] = None, ): if not isinstance(entity_or_document_id, str): entity = self.__session._documents_by_entity.get(entity_or_document_id, None) @@ -1298,7 +1324,9 @@ def store( ) self.__session.defer( - PutAttachmentCommandData(entity_or_document_id, name, stream, content_type, change_vector) + PutAttachmentCommandData( + entity_or_document_id, name, stream, content_type, change_vector, remote_parameters + ) ) def delete(self, entity_or_document_id, name): diff --git a/ravendb/exceptions/documents/__init__.py b/ravendb/exceptions/documents/__init__.py index b2f8db0e..d67050f7 100644 --- a/ravendb/exceptions/documents/__init__.py +++ b/ravendb/exceptions/documents/__init__.py @@ -2,7 +2,11 @@ import json -from ravendb.exceptions.raven_exceptions import ConflictException, BadResponseException +from ravendb.exceptions.raven_exceptions import ConflictException, BadResponseException, RavenException + + +class DocumentDoesNotExistException(RavenException): + pass class DocumentConflictException(ConflictException): diff --git a/ravendb/exceptions/exception_dispatcher.py b/ravendb/exceptions/exception_dispatcher.py index b43aeee0..f1be2ae7 100644 --- a/ravendb/exceptions/exception_dispatcher.py +++ b/ravendb/exceptions/exception_dispatcher.py @@ -2,9 +2,60 @@ import http import os +from datetime import timedelta -from ravendb.exceptions.documents import DocumentConflictException -from ravendb.exceptions.raven_exceptions import ConcurrencyException, RavenException +from ravendb.exceptions.cluster import NodeIsPassiveException, NoLoaderException +from ravendb.exceptions.documents import DocumentConflictException, DocumentDoesNotExistException +from ravendb.exceptions.documents.bulkinsert import BulkInsertAbortedException, BulkInsertProtocolViolationException +from ravendb.exceptions.documents.indexes import IndexDoesNotExistException +from ravendb.exceptions.raven_exceptions import ( + AiException, + BadResponseException, + ClientVersionMismatchException, + ConcurrencyException, + IndexCompactionInProgressException, + InsufficientQuotaException, + PortInUseException, + RateLimitException, + RavenException, + RefusedToAnswerException, + SchemaValidationException, + TooManyRequestsException, + TooManyTokensException, + UnsuccessfulAiRequestException, +) + +# Maps the simple C# class name to the Python exception class. +# C# type strings look like "Raven.Client.Exceptions.Documents.DocumentConflictException" +# — we match on the last segment only. +_EXCEPTION_MAP: dict = { + # raven_exceptions.py + "RavenException": RavenException, + "BadResponseException": BadResponseException, + "ConcurrencyException": ConcurrencyException, + "ClientVersionMismatchException": ClientVersionMismatchException, + "PortInUseException": PortInUseException, + "IndexCompactionInProgressException": IndexCompactionInProgressException, + # AI exceptions + "AiException": AiException, + "RefusedToAnswerException": RefusedToAnswerException, + "UnsuccessfulAiRequestException": UnsuccessfulAiRequestException, + "TooManyRequestsException": TooManyRequestsException, + "RateLimitException": RateLimitException, + "InsufficientQuotaException": InsufficientQuotaException, + "TooManyTokensException": TooManyTokensException, + # documents + "DocumentConflictException": DocumentConflictException, + "DocumentDoesNotExistException": DocumentDoesNotExistException, + "IndexDoesNotExistException": IndexDoesNotExistException, + "BulkInsertAbortedException": BulkInsertAbortedException, + "BulkInsertProtocolViolationException": BulkInsertProtocolViolationException, + # schema validation + "SchemaValidationException": SchemaValidationException, + # cluster + "NodeIsPassiveException": NodeIsPassiveException, + "NoLoaderException": NoLoaderException, +} class ExceptionDispatcher: @@ -16,7 +67,9 @@ def __init__(self, url: str = None, object_type: str = None, message: str = None self.error = error @staticmethod - def get(schema: ExceptionDispatcher.ExceptionSchema, code: int, inner: Exception = None) -> RavenException: + def get( + schema: ExceptionDispatcher.ExceptionSchema, code: int, inner: Exception = None, json_body: dict = None + ) -> RavenException: message = schema.message type_as_string = schema.type @@ -39,22 +92,48 @@ def get(schema: ExceptionDispatcher.ExceptionSchema, code: int, inner: Exception if not issubclass(error_type, RavenException): return RavenException(error, exception) + if json_body: + ExceptionDispatcher.__fill_exception(exception, json_body) + return exception + @staticmethod + def __fill_exception(exception: RavenException, data: dict) -> None: + if isinstance(exception, RateLimitException): + # StatusCode is always 429 for RateLimitException — set it directly (mirrors C# FillException) + exception.status_code = 429 + retry_after_raw = data.get("RetryAfter") + if retry_after_raw: + try: + # RetryAfter is a C# TimeSpan serialized as "hh:mm:ss[.fffffff]" + parts = retry_after_raw.split(":") + if len(parts) == 3: + h, m, s = int(parts[0]), int(parts[1]), float(parts[2]) + exception.retry_after = timedelta(hours=h, minutes=m, seconds=s) + else: + exception.retry_after = timedelta(seconds=float(retry_after_raw)) + except Exception: + pass + elif isinstance(exception, UnsuccessfulAiRequestException): + status_code = data.get("StatusCode") + if status_code is not None: + try: + exception.status_code = int(status_code) + except Exception: + pass + elif isinstance(exception, RefusedToAnswerException): + exception.refusal = data.get("Refusal") + exception.finish_reason = data.get("FinishReason") + + if isinstance(exception, AiException): + exception.request_id = data.get("RequestId") + @staticmethod def __get_type(type_as_string: str) -> type: - if "System.TimeoutException" == type_as_string: + if type_as_string == "System.TimeoutException": return TimeoutError - prefix = "Raven.Client.Exceptions." - if type_as_string.startswith(prefix): - exception_name = type_as_string[len(prefix) : :] - if "." in exception_name: - exception_name = ".".join(list(map(str.lower, exception_name.split(".")))) - - try: - return __import__(f"pyravendb.exceptions.{exception_name}") # todo: fix, doesn't work - except Exception as e: - return None - else: - return None + # C# type strings: "Raven.Client.Exceptions[.Namespace].ClassName" + # Match on the simple class name regardless of namespace depth. + simple_name = type_as_string.split(".")[-1] + return _EXCEPTION_MAP.get(simple_name) diff --git a/ravendb/exceptions/raven_exceptions.py b/ravendb/exceptions/raven_exceptions.py index b05d6c4a..1f9c823c 100644 --- a/ravendb/exceptions/raven_exceptions.py +++ b/ravendb/exceptions/raven_exceptions.py @@ -1,10 +1,13 @@ from abc import abstractmethod +from datetime import timedelta +from enum import Enum from typing import Optional class RavenException(RuntimeError): def __init__(self, message: str = None, cause: BaseException = None): - super(RavenException, self).__init__((message, cause) or message) + super(RavenException, self).__init__(message) + self.cause = cause self.reached_leader = None @classmethod @@ -39,3 +42,46 @@ class PortInUseException(RavenException): class IndexCompactionInProgressException(RavenException): pass + + +class AiException(RavenException): + def __init__(self, message: str = None, cause: BaseException = None): + super().__init__(message, cause) + self.request_id: Optional[str] = None + + +class RefusedToAnswerException(AiException): + def __init__(self, message: str = None): + super().__init__(message) + self.refusal: Optional[str] = None + self.finish_reason: Optional[str] = None + + +class UnsuccessfulAiRequestException(AiException): + def __init__(self, message: str = None, status_code: int = 0): + super().__init__(message) + self.status_code = status_code + + +class TooManyRequestsException(UnsuccessfulAiRequestException): + def __init__(self, message: str = None): + super().__init__(message, status_code=429) + + +class RateLimitException(TooManyRequestsException): + def __init__(self, message: str = None): + super().__init__(message) + self.retry_after: Optional[timedelta] = None + + +class InsufficientQuotaException(TooManyRequestsException): + pass + + +class TooManyTokensException(TooManyRequestsException): + pass + + +class SchemaValidationException(RavenException): + def __init__(self, message: str = None): + super().__init__(message) diff --git a/ravendb/http/request_executor.py b/ravendb/http/request_executor.py index 6e93487b..45a51333 100644 --- a/ravendb/http/request_executor.py +++ b/ravendb/http/request_executor.py @@ -48,7 +48,7 @@ class RequestExecutor: __INITIAL_TOPOLOGY_ETAG = -2 __GLOBAL_APPLICATION_IDENTIFIER = uuid.uuid4() - CLIENT_VERSION = "7.1.5" + CLIENT_VERSION = "7.2.0" logger = logging.getLogger("request_executor") # todo: initializer should take also cryptography certificates @@ -1085,10 +1085,13 @@ def _handle_unsuccessful_response( elif response.status_code == HTTPStatus.CONFLICT: data = json.loads(response.text) - message = data.get("Message", None) - err_type = data.get("Type", None) - - raise RuntimeError(f"{err_type}: {message}") # todo: handle conflict (exception dispatcher involved) + schema = ExceptionDispatcher.ExceptionSchema( + url=self.url, + object_type=data.get("Type", ""), + message=data.get("Message", ""), + error=data.get("Error", ""), + ) + raise ExceptionDispatcher.get(schema, response.status_code, json_body=data) elif response.status_code == 425: # too early if not should_retry: @@ -1114,8 +1117,15 @@ def _handle_unsuccessful_response( return True else: command.on_response_failure(response) - try: # todo: exception dispatcher - raise RuntimeError(json.loads(response.text).get("Message", "Missing message")) + try: + data = json.loads(response.text) + schema = ExceptionDispatcher.ExceptionSchema( + url=self.url, + object_type=data.get("Type", ""), + message=data.get("Message", ""), + error=data.get("Error", ""), + ) + raise ExceptionDispatcher.get(schema, response.status_code, json_body=data) except JSONDecodeError as e: raise RuntimeError(f"Failed to parse response: {response.text}") from e @@ -1220,13 +1230,14 @@ def __read_exception_from_server( raw = None try: raw = response.content.decode("utf-8") - - def _decode(d: dict) -> ExceptionDispatcher.ExceptionSchema: - return ExceptionDispatcher.ExceptionSchema( - d.get("url"), d.get("class"), d.get("message"), d.get("error") - ) - - return ExceptionDispatcher.get(json.loads(raw, object_hook=_decode), response.status_code, e) + d = json.loads(raw) + schema = ExceptionDispatcher.ExceptionSchema( + d.get("Url") or d.get("url"), + d.get("Type") or d.get("type") or d.get("class"), + d.get("Message") or d.get("message"), + d.get("Error") or d.get("error"), + ) + return ExceptionDispatcher.get(schema, response.status_code, e, d) except Exception: schema = ExceptionDispatcher.ExceptionSchema( request.url if request else "", diff --git a/ravendb/primitives/constants.py b/ravendb/primitives/constants.py index 30285280..7932e005 100644 --- a/ravendb/primitives/constants.py +++ b/ravendb/primitives/constants.py @@ -51,6 +51,10 @@ class Headers: INCREMENTAL_TIME_SERIES_PREFIX = "INC:" SHARDED = "Sharded" ATTACHMENT_HASH = "Attachment-Hash" + ATTACHMENT_SIZE = "Attachment-Size" + ATTACHMENT_REMOTE_PARAMETERS_AT = "Attachment-RemoteParameters-At" + ATTACHMENT_REMOTE_PARAMETERS_FLAGS = "Attachment-RemoteParameters-Flags" + ATTACHMENT_REMOTE_PARAMETERS_IDENTIFIER = "Attachment-RemoteParameters-Identifier" DATABASE_MISSING = "Database-Missing" class Encodings: @@ -222,6 +226,8 @@ class JavaScript: BOOST_PROPERTY_NAME = "$boost" VECTOR_PROPERTY_NAME = "$vector" LOAD_VECTOR_PROPERTY_NAME = "$loadvector" + LOAD_VECTOR_EMBEDDING_SOURCE_DOCUMENT_ID = "$embeddingSourceDocumentId" + LOAD_VECTOR_EMBEDDING_SOURCE_DOCUMENT_COLLECTION_NAME = "$embeddingSourceDocumentCollectionName" class Spatial: DEFAULT_DISTANCE_ERROR_PCT = 0.025 diff --git a/ravendb/tests/jvm_migrated_tests/attachments_tests/test_bulk_insert_attachments.py b/ravendb/tests/jvm_migrated_tests/attachments_tests/test_bulk_insert_attachments.py index 1be9a601..bfc9cea1 100644 --- a/ravendb/tests/jvm_migrated_tests/attachments_tests/test_bulk_insert_attachments.py +++ b/ravendb/tests/jvm_migrated_tests/attachments_tests/test_bulk_insert_attachments.py @@ -1,3 +1,11 @@ +import datetime + +from ravendb.documents.operations.attachments import ( + RemoteAttachmentFlags, + RemoteAttachmentParameters, + StoreAttachmentParameters, +) +from ravendb.infrastructure.entities import User from ravendb.tests.test_base import TestBase @@ -11,3 +19,102 @@ def callback(): bulk_insert.attachments_for(None) self.assertRaisesWithMessage(callback, ValueError, "Document id cannot be None or empty.") + + def test_store_attachment(self): + with self.store.bulk_insert() as bulk_insert: + bulk_insert.store_as(User(name="John"), "users/1") + attachments = bulk_insert.attachments_for("users/1") + attachments.store("file.txt", bytes([1, 2, 3]), "text/plain") + + with self.store.open_session() as session: + names = session.advanced.attachments.get_names(session.load("users/1", User)) + self.assertEqual(1, len(names)) + self.assertEqual("file.txt", names[0].name) + self.assertEqual("text/plain", names[0].content_type) + self.assertEqual(3, names[0].size) + + def test_store_attachment_with_parameters(self): + attachment_bytes = bytes([10, 20, 30, 40, 50]) + + with self.store.bulk_insert() as bulk_insert: + bulk_insert.store_as(User(name="Jane"), "users/2") + attachments = bulk_insert.attachments_for("users/2") + params = StoreAttachmentParameters("photo.png", attachment_bytes, content_type="image/png") + params.remote_parameters = RemoteAttachmentParameters( + "destination", datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=3) + ) + attachments.store_with_parameters(params) + + with self.store.open_session() as session: + names = session.advanced.attachments.get_names(session.load("users/2", User)) + self.assertEqual(1, len(names)) + self.assertEqual("photo.png", names[0].name) + self.assertEqual("image/png", names[0].content_type) + self.assertEqual(5, names[0].size) + + def test_store_remote_attachment_persists_remote_parameters(self): + """remote_parameters (identifier, at, flags=None) survive the bulk insert round-trip.""" + at = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=5) + + with self.store.bulk_insert() as bulk_insert: + bulk_insert.store_as(User(name="Bob"), "users/remote-1") + attachments = bulk_insert.attachments_for("users/remote-1") + params = StoreAttachmentParameters("report.pdf", bytes([1, 2, 3, 4]), content_type="application/pdf") + params.remote_parameters = RemoteAttachmentParameters("dest-1", at) + attachments.store_with_parameters(params) + + with self.store.open_session() as session: + attachment = session.advanced.attachments.get("users/remote-1", "report.pdf") + self.assertIsNotNone(attachment) + self.assertEqual("report.pdf", attachment.details.name) + self.assertEqual("application/pdf", attachment.details.content_type) + self.assertIsNotNone(attachment.details.remote_parameters) + self.assertEqual("dest-1", attachment.details.remote_parameters.identifier) + self.assertEqual(RemoteAttachmentFlags.NONE, attachment.details.remote_parameters.flags) + self.assertEqual( + at.replace(microsecond=0, tzinfo=None), attachment.details.remote_parameters.at.replace(microsecond=0) + ) + + def test_store_multiple_remote_attachments_on_same_document(self): + """Two remote attachments on the same document, each with a different identifier.""" + at = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=5) + + with self.store.bulk_insert() as bulk_insert: + bulk_insert.store_as(User(name="Carol"), "users/remote-2") + attachments = bulk_insert.attachments_for("users/remote-2") + + p1 = StoreAttachmentParameters("photo.jpg", bytes([10, 20, 30]), content_type="image/jpeg") + p1.remote_parameters = RemoteAttachmentParameters("dest-1", at) + attachments.store_with_parameters(p1) + + p2 = StoreAttachmentParameters("thumb.jpg", bytes([1, 2]), content_type="image/jpeg") + p2.remote_parameters = RemoteAttachmentParameters("dest-2", at) + attachments.store_with_parameters(p2) + + with self.store.open_session() as session: + user = session.load("users/remote-2", User) + names = session.advanced.attachments.get_names(user) + self.assertEqual(2, len(names)) + name_set = {a.name for a in names} + self.assertIn("photo.jpg", name_set) + self.assertIn("thumb.jpg", name_set) + + a1 = session.advanced.attachments.get("users/remote-2", "photo.jpg") + self.assertEqual("dest-1", a1.details.remote_parameters.identifier) + + a2 = session.advanced.attachments.get("users/remote-2", "thumb.jpg") + self.assertEqual("dest-2", a2.details.remote_parameters.identifier) + + def test_store_multiple_attachments(self): + with self.store.bulk_insert() as bulk_insert: + bulk_insert.store_as(User(name="Alice"), "users/3") + attachments = bulk_insert.attachments_for("users/3") + attachments.store("file1.txt", bytes([1, 2, 3])) + attachments.store("file2.bin", bytes([4, 5, 6, 7]), "application/octet-stream") + + with self.store.open_session() as session: + names = session.advanced.attachments.get_names(session.load("users/3", User)) + self.assertEqual(2, len(names)) + attachment_names = {a.name for a in names} + self.assertIn("file1.txt", attachment_names) + self.assertIn("file2.bin", attachment_names) diff --git a/ravendb/tests/jvm_migrated_tests/attachments_tests/test_document_session_remote_attachments.py b/ravendb/tests/jvm_migrated_tests/attachments_tests/test_document_session_remote_attachments.py new file mode 100644 index 00000000..c6a618c8 --- /dev/null +++ b/ravendb/tests/jvm_migrated_tests/attachments_tests/test_document_session_remote_attachments.py @@ -0,0 +1,204 @@ +""" +Tests for storing attachments with RemoteAttachmentParameters via the document session. + +Migrated from: + test/SlowTests/Server/Documents/Attachments/DocumentSessionRemoteAttachmentsAsyncTests.cs + +Scope: client-side session API only. +Tests that require the server-side RemoteAttachmentsSender (ProcessRemoteAttachments) to +actually push bytes to S3/Azure — and therefore need real cloud credentials — are omitted +here. There are separate mock tests for that purpose. + +What IS tested: + - store_with_parameters stores the attachment and persists RemoteAttachmentParameters + (identifier, at, flags=None) as returned by session.advanced.attachments.get() + - exists() returns True after storing with remote params + - delete() removes the attachment + - get() by entity object (not just document id) + - get_names() reflects the attachment + - overwriting an attachment with new remote params replaces the old params + - storing with remote_parameters=None produces a plain attachment (no remote_parameters) + - storing to multiple destinations (two different identifiers) works independently +""" + +import datetime +import unittest + +from ravendb.documents.operations.attachments import ( + RemoteAttachmentFlags, + RemoteAttachmentParameters, + StoreAttachmentParameters, +) +from ravendb.tests.test_base import TestBase + + +class User: + def __init__(self, name: str): + self.name = name + + +_IDENTIFIER = "Conf-identifier-s3-1" +_IDENTIFIER_2 = "Conf-identifier-s3-2" +_AT = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=3) + + +class TestDocumentSessionRemoteAttachments(TestBase): + def setUp(self): + super().setUp() + + # ── helpers ────────────────────────────────────────────────────────────── + + def _store_doc(self, doc_id: str) -> None: + with self.store.open_session() as session: + session.store(User("Alice"), doc_id) + session.save_changes() + + def _store_attachment( + self, doc_id: str, name: str, data: bytes, identifier: str, at=None, content_type="image/png" + ): + params = StoreAttachmentParameters(name, data, content_type=content_type) + params.remote_parameters = RemoteAttachmentParameters(identifier, at or _AT) + with self.store.open_session() as session: + session.advanced.attachments.store_with_parameters(doc_id, params) + session.save_changes() + + # ── tests ───────────────────────────────────────────────────────────────── + + def test_can_store_and_get_remote_attachment_metadata(self): + """store_with_parameters persists identifier, at, and flags=None.""" + doc_id = "orders/1" + self._store_doc(doc_id) + at = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=3) + self._store_attachment(doc_id, "test.png", bytes([1, 2, 3]), _IDENTIFIER, at) + + with self.store.open_session() as session: + attachment = session.advanced.attachments.get(doc_id, "test.png") + self.assertIsNotNone(attachment) + self.assertEqual("test.png", attachment.details.name) + self.assertEqual("image/png", attachment.details.content_type) + self.assertIsNotNone(attachment.details.remote_parameters) + self.assertEqual(RemoteAttachmentFlags.NONE, attachment.details.remote_parameters.flags) + self.assertEqual(_IDENTIFIER, attachment.details.remote_parameters.identifier) + # compare truncated to seconds to avoid sub-second drift + self.assertEqual( + at.replace(microsecond=0, tzinfo=None), attachment.details.remote_parameters.at.replace(microsecond=0) + ) + + def test_can_check_if_remote_attachment_exists(self): + doc_id = "orders/2" + self._store_doc(doc_id) + self._store_attachment(doc_id, "test.png", bytes([1, 2, 3]), _IDENTIFIER) + + with self.store.open_session() as session: + self.assertTrue(session.advanced.attachments.exists(doc_id, "test.png")) + + def test_can_get_remote_attachment_by_entity(self): + doc_id = "orders/3" + self._store_doc(doc_id) + self._store_attachment(doc_id, "test.png", bytes([1, 2, 3]), _IDENTIFIER) + + with self.store.open_session() as session: + order = session.load(doc_id, User) + attachment = session.advanced.attachments.get(order, "test.png") + self.assertIsNotNone(attachment) + self.assertEqual("test.png", attachment.details.name) + self.assertIsNotNone(attachment.details.remote_parameters) + self.assertEqual(RemoteAttachmentFlags.NONE, attachment.details.remote_parameters.flags) + + def test_can_delete_remote_attachment(self): + doc_id = "orders/4" + self._store_doc(doc_id) + self._store_attachment(doc_id, "test.png", bytes([1, 2, 3]), _IDENTIFIER) + + with self.store.open_session() as session: + session.advanced.attachments.delete(doc_id, "test.png") + session.save_changes() + + with self.store.open_session() as session: + self.assertFalse(session.advanced.attachments.exists(doc_id, "test.png")) + + def test_can_delete_remote_attachment_by_entity(self): + doc_id = "orders/5" + self._store_doc(doc_id) + self._store_attachment(doc_id, "test.png", bytes([1, 2, 3]), _IDENTIFIER) + + with self.store.open_session() as session: + order = session.load(doc_id, User) + session.advanced.attachments.delete(order, "test.png") + session.save_changes() + + with self.store.open_session() as session: + self.assertFalse(session.advanced.attachments.exists(doc_id, "test.png")) + + def test_get_names_reflects_remote_attachment(self): + doc_id = "orders/6" + self._store_doc(doc_id) + self._store_attachment(doc_id, "test.png", bytes([1, 2, 3]), _IDENTIFIER) + + with self.store.open_session() as session: + order = session.load(doc_id, User) + names = session.advanced.attachments.get_names(order) + self.assertEqual(1, len(names)) + self.assertEqual("test.png", names[0].name) + self.assertEqual("image/png", names[0].content_type) + + def test_can_overwrite_remote_attachment_with_new_params(self): + """Overwriting with new identifier/at replaces the old remote params.""" + doc_id = "orders/7" + self._store_doc(doc_id) + at1 = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=3) + self._store_attachment(doc_id, "test.png", bytes([1, 2, 3]), _IDENTIFIER, at1) + + at2 = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=15) + self._store_attachment(doc_id, "test.png", bytes([1, 2, 3]), _IDENTIFIER_2, at2) + + with self.store.open_session() as session: + attachment = session.advanced.attachments.get(doc_id, "test.png") + self.assertIsNotNone(attachment) + self.assertEqual(_IDENTIFIER_2, attachment.details.remote_parameters.identifier) + self.assertEqual( + at2.replace(microsecond=0, tzinfo=None), attachment.details.remote_parameters.at.replace(microsecond=0) + ) + + def test_can_store_plain_attachment_after_remote(self): + """Overwriting a remote attachment with remote_parameters=None clears remote params.""" + doc_id = "orders/8" + self._store_doc(doc_id) + self._store_attachment(doc_id, "test.png", bytes([1, 2, 3]), _IDENTIFIER) + + # overwrite with plain (no remote params) + with self.store.open_session() as session: + session.advanced.attachments.store(doc_id, "test.png", bytes([4, 5, 6]), content_type="image/png") + session.save_changes() + + with self.store.open_session() as session: + attachment = session.advanced.attachments.get(doc_id, "test.png") + self.assertIsNotNone(attachment) + self.assertIsNone(attachment.details.remote_parameters) + + def test_can_store_to_multiple_destinations(self): + """Two documents each get an attachment pointing to a different identifier.""" + id1, id2 = "orders/9", "orders/10" + self._store_doc(id1) + self._store_doc(id2) + at1 = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=3) + at2 = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=5) + self._store_attachment(id1, "test.png", bytes([1, 2, 3]), _IDENTIFIER, at1) + self._store_attachment(id2, "test.png", bytes([3, 2, 1]), _IDENTIFIER_2, at2) + + with self.store.open_session() as session: + a1 = session.advanced.attachments.get(id1, "test.png") + self.assertEqual(_IDENTIFIER, a1.details.remote_parameters.identifier) + self.assertEqual( + at1.replace(microsecond=0, tzinfo=None), a1.details.remote_parameters.at.replace(microsecond=0) + ) + + a2 = session.advanced.attachments.get(id2, "test.png") + self.assertEqual(_IDENTIFIER_2, a2.details.remote_parameters.identifier) + self.assertEqual( + at2.replace(microsecond=0, tzinfo=None), a2.details.remote_parameters.at.replace(microsecond=0) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/tests/jvm_migrated_tests/attachments_tests/test_mock_remote_attachments.py b/ravendb/tests/jvm_migrated_tests/attachments_tests/test_mock_remote_attachments.py new file mode 100644 index 00000000..545da11a --- /dev/null +++ b/ravendb/tests/jvm_migrated_tests/attachments_tests/test_mock_remote_attachments.py @@ -0,0 +1,378 @@ +""" +Mock tests for remote attachment session flows. + +CRUD operations (store doc, store/delete attachment, exists) run against the real DB via +TestBase. Only session.advanced.attachments.get() is mocked — it would normally return +flags=Remote only after the server-side background sender has pushed the blob to cloud, +which requires real S3/Azure credentials. + +Mirrors: test/SlowTests/Server/Documents/Attachments/DocumentSessionRemoteAttachmentsAsyncTests.cs +""" + +import datetime +import unittest +from unittest.mock import MagicMock, patch + +from ravendb.documents.operations.attachments import ( + AttachmentDetails, + CloseableAttachmentResult, + RemoteAttachmentFlags, + RemoteAttachmentParameters, + StoreAttachmentParameters, +) +from ravendb.tests.test_base import TestBase + +_IDENTIFIER = "dest-1" +_IDENTIFIER_2 = "dest-2" + + +class User: + def __init__(self, name: str): + self.name = name + + +def _make_attachment_result( + name: str, + content_type: str, + data: bytes, + identifier: str, + at: datetime.datetime, + flags: RemoteAttachmentFlags = RemoteAttachmentFlags.REMOTE, +) -> CloseableAttachmentResult: + """Build a CloseableAttachmentResult as the server would return after cloud upload.""" + details = AttachmentDetails(name, "hash-mock", content_type, len(data)) + remote_params = RemoteAttachmentParameters.__new__(RemoteAttachmentParameters) + remote_params.identifier = identifier + remote_params.at = at + remote_params.flags = flags + details.remote_parameters = remote_params + + response = MagicMock() + response.content = data + response.close = MagicMock() + return CloseableAttachmentResult(response, details) + + +class TestMockRemoteAttachments(TestBase): + def setUp(self): + super().setUp() + + def test_store_and_get_flags_remote_after_upload(self): + """After cloud upload, get() returns flags=Remote.""" + with self.store.open_session() as session: + session.store(User("Alice"), "orders/1") + session.save_changes() + + at = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=3) + params = StoreAttachmentParameters("test.png", bytes([1, 2, 3]), content_type="image/png") + params.remote_parameters = RemoteAttachmentParameters(_IDENTIFIER, at) + with self.store.open_session() as session: + session.advanced.attachments.store_with_parameters("orders/1", params) + session.save_changes() + + mock_result = _make_attachment_result("test.png", "image/png", bytes([1, 2, 3]), _IDENTIFIER, at) + with self.store.open_session() as session: + with patch.object(session.advanced.attachments, "get", return_value=mock_result): + attachment = session.advanced.attachments.get("orders/1", "test.png") + self.assertIsNotNone(attachment) + self.assertEqual("test.png", attachment.details.name) + self.assertEqual("image/png", attachment.details.content_type) + self.assertIsNotNone(attachment.details.remote_parameters) + self.assertEqual(RemoteAttachmentFlags.REMOTE, attachment.details.remote_parameters.flags) + self.assertEqual(_IDENTIFIER, attachment.details.remote_parameters.identifier) + self.assertEqual(bytes([1, 2, 3]), attachment.data) + + def test_get_by_entity_flags_remote(self): + with self.store.open_session() as session: + session.store(User("Alice"), "orders/2") + session.save_changes() + + at = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=3) + params = StoreAttachmentParameters("test.png", bytes([1, 2, 3]), content_type="image/png") + params.remote_parameters = RemoteAttachmentParameters(_IDENTIFIER, at) + with self.store.open_session() as session: + session.advanced.attachments.store_with_parameters("orders/2", params) + session.save_changes() + + mock_result = _make_attachment_result("test.png", "image/png", bytes([1, 2, 3]), _IDENTIFIER, at) + with self.store.open_session() as session: + order = session.load("orders/2", User) + with patch.object(session.advanced.attachments, "get", return_value=mock_result): + attachment = session.advanced.attachments.get(order, "test.png") + self.assertIsNotNone(attachment) + self.assertEqual(RemoteAttachmentFlags.REMOTE, attachment.details.remote_parameters.flags) + self.assertEqual(_IDENTIFIER, attachment.details.remote_parameters.identifier) + + def test_delete_after_upload(self): + """After cloud upload, delete() removes the local ref; exists() returns False.""" + with self.store.open_session() as session: + session.store(User("Alice"), "orders/3") + session.save_changes() + + at = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=3) + params = StoreAttachmentParameters("test.png", bytes([1, 2, 3]), content_type="image/png") + params.remote_parameters = RemoteAttachmentParameters(_IDENTIFIER, at) + with self.store.open_session() as session: + session.advanced.attachments.store_with_parameters("orders/3", params) + session.save_changes() + + with self.store.open_session() as session: + session.advanced.attachments.delete("orders/3", "test.png") + session.save_changes() + + with self.store.open_session() as session: + self.assertFalse(session.advanced.attachments.exists("orders/3", "test.png")) + + def test_overwrite_with_new_identifier_flags_remote(self): + with self.store.open_session() as session: + session.store(User("Alice"), "orders/4") + session.save_changes() + + at = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=3) + params = StoreAttachmentParameters("test.png", bytes([1, 2, 3]), content_type="image/png") + params.remote_parameters = RemoteAttachmentParameters(_IDENTIFIER, at) + with self.store.open_session() as session: + session.advanced.attachments.store_with_parameters("orders/4", params) + session.save_changes() + + params2 = StoreAttachmentParameters("test.png", bytes([1, 2, 3]), content_type="image/png") + params2.remote_parameters = RemoteAttachmentParameters(_IDENTIFIER_2, at) + with self.store.open_session() as session: + session.advanced.attachments.store_with_parameters("orders/4", params2) + session.save_changes() + + mock_result = _make_attachment_result("test.png", "image/png", bytes([1, 2, 3]), _IDENTIFIER_2, at) + with self.store.open_session() as session: + with patch.object(session.advanced.attachments, "get", return_value=mock_result): + attachment = session.advanced.attachments.get("orders/4", "test.png") + self.assertEqual(RemoteAttachmentFlags.REMOTE, attachment.details.remote_parameters.flags) + self.assertEqual(_IDENTIFIER_2, attachment.details.remote_parameters.identifier) + + def test_overwrite_with_new_at_flags_remote(self): + with self.store.open_session() as session: + session.store(User("Alice"), "orders/5") + session.save_changes() + + at = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=3) + params = StoreAttachmentParameters("test.png", bytes([1, 2, 3]), content_type="image/png") + params.remote_parameters = RemoteAttachmentParameters(_IDENTIFIER, at) + with self.store.open_session() as session: + session.advanced.attachments.store_with_parameters("orders/5", params) + session.save_changes() + + new_at = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=15) + params2 = StoreAttachmentParameters("test.png", bytes([1, 2, 3]), content_type="image/png") + params2.remote_parameters = RemoteAttachmentParameters(_IDENTIFIER, new_at) + with self.store.open_session() as session: + session.advanced.attachments.store_with_parameters("orders/5", params2) + session.save_changes() + + mock_result = _make_attachment_result("test.png", "image/png", bytes([1, 2, 3]), _IDENTIFIER, new_at) + with self.store.open_session() as session: + with patch.object(session.advanced.attachments, "get", return_value=mock_result): + attachment = session.advanced.attachments.get("orders/5", "test.png") + self.assertEqual(RemoteAttachmentFlags.REMOTE, attachment.details.remote_parameters.flags) + self.assertEqual(_IDENTIFIER, attachment.details.remote_parameters.identifier) + self.assertEqual( + new_at.replace(microsecond=0, tzinfo=None), + attachment.details.remote_parameters.at.replace(microsecond=0, tzinfo=None), + ) + + def test_overwrite_with_new_identifier_and_at_flags_remote(self): + with self.store.open_session() as session: + session.store(User("Alice"), "orders/6") + session.save_changes() + + at = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=3) + params = StoreAttachmentParameters("test.png", bytes([1, 2, 3]), content_type="image/png") + params.remote_parameters = RemoteAttachmentParameters(_IDENTIFIER, at) + with self.store.open_session() as session: + session.advanced.attachments.store_with_parameters("orders/6", params) + session.save_changes() + + new_at = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=15) + params2 = StoreAttachmentParameters("test.png", bytes([1, 2, 3]), content_type="image/png") + params2.remote_parameters = RemoteAttachmentParameters(_IDENTIFIER_2, new_at) + with self.store.open_session() as session: + session.advanced.attachments.store_with_parameters("orders/6", params2) + session.save_changes() + + mock_result = _make_attachment_result("test.png", "image/png", bytes([1, 2, 3]), _IDENTIFIER_2, new_at) + with self.store.open_session() as session: + with patch.object(session.advanced.attachments, "get", return_value=mock_result): + attachment = session.advanced.attachments.get("orders/6", "test.png") + self.assertEqual(RemoteAttachmentFlags.REMOTE, attachment.details.remote_parameters.flags) + self.assertEqual(_IDENTIFIER_2, attachment.details.remote_parameters.identifier) + self.assertEqual( + new_at.replace(microsecond=0, tzinfo=None), + attachment.details.remote_parameters.at.replace(microsecond=0, tzinfo=None), + ) + + def test_overwrite_with_no_remote_params_demotes_to_plain(self): + """After demoting to plain, get() returns no remote_parameters.""" + with self.store.open_session() as session: + session.store(User("Alice"), "orders/7") + session.save_changes() + + at = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=3) + params = StoreAttachmentParameters("test.png", bytes([1, 2, 3]), content_type="image/png") + params.remote_parameters = RemoteAttachmentParameters(_IDENTIFIER, at) + with self.store.open_session() as session: + session.advanced.attachments.store_with_parameters("orders/7", params) + session.save_changes() + + with self.store.open_session() as session: + session.advanced.attachments.store("orders/7", "test.png", bytes([4, 5, 6]), content_type="image/png") + session.save_changes() + + with self.store.open_session() as session: + attachment = session.advanced.attachments.get("orders/7", "test.png") + self.assertIsNotNone(attachment) + self.assertIsNone(attachment.details.remote_parameters) + + def test_store_to_multiple_destinations_flags_remote(self): + with self.store.open_session() as session: + session.store(User("Alice"), "orders/8") + session.store(User("Bob"), "orders/9") + session.save_changes() + + at1 = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=3) + params1 = StoreAttachmentParameters("test.png", bytes([1, 2, 3]), content_type="image/png") + params1.remote_parameters = RemoteAttachmentParameters(_IDENTIFIER, at1) + with self.store.open_session() as session: + session.advanced.attachments.store_with_parameters("orders/8", params1) + session.save_changes() + + at2 = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=5) + params2 = StoreAttachmentParameters("test.png", bytes([3, 2, 1]), content_type="image/png") + params2.remote_parameters = RemoteAttachmentParameters(_IDENTIFIER_2, at2) + with self.store.open_session() as session: + session.advanced.attachments.store_with_parameters("orders/9", params2) + session.save_changes() + + mock1 = _make_attachment_result("test.png", "image/png", bytes([1, 2, 3]), _IDENTIFIER, at1) + mock2 = _make_attachment_result("test.png", "image/png", bytes([3, 2, 1]), _IDENTIFIER_2, at2) + + with self.store.open_session() as session: + with patch.object(session.advanced.attachments, "get", side_effect=[mock1, mock2]): + a1 = session.advanced.attachments.get("orders/8", "test.png") + self.assertEqual(RemoteAttachmentFlags.REMOTE, a1.details.remote_parameters.flags) + self.assertEqual(_IDENTIFIER, a1.details.remote_parameters.identifier) + + a2 = session.advanced.attachments.get("orders/9", "test.png") + self.assertEqual(RemoteAttachmentFlags.REMOTE, a2.details.remote_parameters.flags) + self.assertEqual(_IDENTIFIER_2, a2.details.remote_parameters.identifier) + + def test_azure_blob_store_and_get_flags_remote(self): + """Mocked Azure flow: after upload, get() returns flags=Remote with correct data.""" + with self.store.open_session() as session: + session.store(User("Alice"), "orders/azure-1") + session.save_changes() + + at = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=5) + params = StoreAttachmentParameters("photo.png", bytes([10, 20, 30, 40, 50]), content_type="image/png") + params.remote_parameters = RemoteAttachmentParameters("azure-dest-1", at) + with self.store.open_session() as session: + session.advanced.attachments.store_with_parameters("orders/azure-1", params) + session.save_changes() + + # Before upload: flags=None (real DB response) + with self.store.open_session() as session: + attachment = session.advanced.attachments.get("orders/azure-1", "photo.png") + self.assertIsNotNone(attachment) + self.assertEqual(RemoteAttachmentFlags.NONE, attachment.details.remote_parameters.flags) + + # After upload: mock flags=Remote + mock_result = _make_attachment_result("photo.png", "image/png", bytes([10, 20, 30, 40, 50]), "azure-dest-1", at) + with self.store.open_session() as session: + with patch.object(session.advanced.attachments, "get", return_value=mock_result): + attachment = session.advanced.attachments.get("orders/azure-1", "photo.png") + self.assertIsNotNone(attachment) + self.assertEqual("photo.png", attachment.details.name) + self.assertEqual("image/png", attachment.details.content_type) + self.assertEqual(RemoteAttachmentFlags.REMOTE, attachment.details.remote_parameters.flags) + self.assertEqual("azure-dest-1", attachment.details.remote_parameters.identifier) + self.assertEqual(bytes([10, 20, 30, 40, 50]), attachment.data) + + def test_bulk_insert_remote_attachment_persists_remote_parameters(self): + """Bulk insert: remote_parameters (identifier, at, flags=None) survive the round-trip.""" + at = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=5) + + with self.store.bulk_insert() as bulk_insert: + bulk_insert.store_as(User("Alice"), "orders/bulk-1") + attachments = bulk_insert.attachments_for("orders/bulk-1") + params = StoreAttachmentParameters("report.pdf", bytes([1, 2, 3, 4]), content_type="application/pdf") + params.remote_parameters = RemoteAttachmentParameters(_IDENTIFIER, at) + attachments.store_with_parameters(params) + + with self.store.open_session() as session: + attachment = session.advanced.attachments.get("orders/bulk-1", "report.pdf") + self.assertIsNotNone(attachment) + self.assertEqual("report.pdf", attachment.details.name) + self.assertEqual("application/pdf", attachment.details.content_type) + self.assertIsNotNone(attachment.details.remote_parameters) + self.assertEqual(_IDENTIFIER, attachment.details.remote_parameters.identifier) + self.assertEqual(RemoteAttachmentFlags.NONE, attachment.details.remote_parameters.flags) + self.assertEqual( + at.replace(microsecond=0, tzinfo=None), + attachment.details.remote_parameters.at.replace(microsecond=0), + ) + + # Mock flags=Remote to simulate post-upload state. + mock_result = _make_attachment_result("report.pdf", "application/pdf", bytes([1, 2, 3, 4]), _IDENTIFIER, at) + with self.store.open_session() as session: + with patch.object(session.advanced.attachments, "get", return_value=mock_result): + attachment = session.advanced.attachments.get("orders/bulk-1", "report.pdf") + self.assertEqual(RemoteAttachmentFlags.REMOTE, attachment.details.remote_parameters.flags) + self.assertEqual(_IDENTIFIER, attachment.details.remote_parameters.identifier) + self.assertEqual(bytes([1, 2, 3, 4]), attachment.data) + + def test_bulk_insert_100_remote_attachments_flags_remote(self): + """Bulk insert 10 docs × 10 attachments = 100 remote attachments; mock all as Remote.""" + at = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(minutes=5) + doc_ids = [f"orders/bulk100-mock-{i}" for i in range(10)] + + with self.store.bulk_insert() as bulk_insert: + for doc_id in doc_ids: + bulk_insert.store_as(User("Alice"), doc_id) + attachments = bulk_insert.attachments_for(doc_id) + for j in range(10): + params = StoreAttachmentParameters( + f"file-{j}.bin", + bytes([k % 256 for k in range(j + 1)]), + content_type="application/octet-stream", + ) + params.remote_parameters = RemoteAttachmentParameters(_IDENTIFIER, at) + attachments.store_with_parameters(params) + + # Before upload: all 100 must have flags=None (real DB). + with self.store.open_session() as session: + for doc_id in doc_ids: + user = session.load(doc_id, User) + names = session.advanced.attachments.get_names(user) + self.assertEqual(10, len(names)) + for attachment_name in names: + self.assertIsNotNone(attachment_name.remote_parameters) + self.assertEqual(RemoteAttachmentFlags.NONE, attachment_name.remote_parameters.flags) + self.assertEqual(_IDENTIFIER, attachment_name.remote_parameters.identifier) + + # After upload: mock all 100 as Remote. + with self.store.open_session() as session: + for doc_id in doc_ids: + user = session.load(doc_id, User) + names = session.advanced.attachments.get_names(user) + mock_results = [ + _make_attachment_result( + a.name, a.content_type, bytes([k % 256 for k in range(j + 1)]), _IDENTIFIER, at + ) + for j, a in enumerate(names) + ] + with patch.object(session.advanced.attachments, "get", side_effect=mock_results): + for j, attachment_name in enumerate(names): + attachment = session.advanced.attachments.get(doc_id, attachment_name.name) + self.assertEqual(RemoteAttachmentFlags.REMOTE, attachment.details.remote_parameters.flags) + self.assertEqual(_IDENTIFIER, attachment.details.remote_parameters.identifier) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/tests/jvm_migrated_tests/attachments_tests/test_remote_attachments_basic.py b/ravendb/tests/jvm_migrated_tests/attachments_tests/test_remote_attachments_basic.py new file mode 100644 index 00000000..60b7639a --- /dev/null +++ b/ravendb/tests/jvm_migrated_tests/attachments_tests/test_remote_attachments_basic.py @@ -0,0 +1,255 @@ +import os +import unittest + +from ravendb.documents.operations.attachments import ( + ConfigureRemoteAttachmentsOperation, + GetRemoteAttachmentsConfigurationOperation, + RemoteAttachmentsAzureSettings, + RemoteAttachmentsConfiguration, + RemoteAttachmentsDestinationConfiguration, + RemoteAttachmentsS3Settings, +) +from ravendb.tests.test_base import TestBase + + +class TestRemoteAttachmentsBasic(TestBase): + def setUp(self): + super().setUp() + + # ── CRUD tests ──────────────────────────────────────────────────────────── + + @unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") + def test_can_put_and_get_remote_attachments_configuration_with_case_insensitive_identifier(self): + self.store.maintenance.send( + ConfigureRemoteAttachmentsOperation( + RemoteAttachmentsConfiguration( + destinations={ + "S3-uSeRs": RemoteAttachmentsDestinationConfiguration( + s3_settings=RemoteAttachmentsS3Settings(bucket_name="testS3Bucket-Users"), + disabled=False, + ) + }, + max_items_to_process=1, + ) + ) + ) + + config = self.store.maintenance.send(GetRemoteAttachmentsConfigurationOperation()) + self.assertEqual(1, len(config.destinations)) + dest_key, dest_val = next(iter(config.destinations.items())) + self.assertEqual("S3-uSeRs", dest_key) + self.assertEqual("testS3Bucket-Users", dest_val.s3_settings.bucket_name) + self.assertFalse(dest_val.disabled) + self.assertIsNone(config.check_frequency_in_sec) + + @unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") + def test_can_put_and_get_remote_attachments_configuration_with_default_remote_frequency_in_sec(self): + self.store.maintenance.send( + ConfigureRemoteAttachmentsOperation( + RemoteAttachmentsConfiguration( + destinations={ + "S3-Users": RemoteAttachmentsDestinationConfiguration( + s3_settings=RemoteAttachmentsS3Settings(bucket_name="testS3Bucket-Users"), + disabled=False, + ) + }, + max_items_to_process=1, + ) + ) + ) + + config = self.store.maintenance.send(GetRemoteAttachmentsConfigurationOperation()) + self.assertEqual(1, len(config.destinations)) + dest_key, dest_val = next(iter(config.destinations.items())) + self.assertEqual("S3-Users", dest_key) + self.assertEqual("testS3Bucket-Users", dest_val.s3_settings.bucket_name) + self.assertFalse(dest_val.disabled) + self.assertIsNone(config.check_frequency_in_sec) + + @unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") + def test_can_put_and_get_remote_attachments_configuration(self): + c1 = RemoteAttachmentsConfiguration( + destinations={ + "S3-Users": RemoteAttachmentsDestinationConfiguration( + s3_settings=RemoteAttachmentsS3Settings(bucket_name="testS3Bucket-Users"), + disabled=False, + ) + }, + check_frequency_in_sec=1000, + ) + self.store.maintenance.send(ConfigureRemoteAttachmentsOperation(c1)) + + config = self.store.maintenance.send(GetRemoteAttachmentsConfigurationOperation()) + dest_key, dest_val = next(iter(config.destinations.items())) + self.assertEqual(1, len(config.destinations)) + self.assertEqual("S3-Users", dest_key) + self.assertEqual("testS3Bucket-Users", dest_val.s3_settings.bucket_name) + self.assertFalse(dest_val.disabled) + self.assertEqual(1000, config.check_frequency_in_sec) + + c2 = RemoteAttachmentsConfiguration( + destinations={ + "S3-Orders": RemoteAttachmentsDestinationConfiguration( + s3_settings=RemoteAttachmentsS3Settings(bucket_name="testS3Bucket-Orders"), + disabled=True, + ) + }, + check_frequency_in_sec=10000, + disabled=True, + ) + self.store.maintenance.send(ConfigureRemoteAttachmentsOperation(c2)) + + config2 = self.store.maintenance.send(GetRemoteAttachmentsConfigurationOperation()) + dest_key2, dest_val2 = next(iter(config2.destinations.items())) + self.assertEqual(1, len(config2.destinations)) + self.assertTrue(config2.disabled) + self.assertEqual("S3-Orders", dest_key2) + self.assertEqual("testS3Bucket-Orders", dest_val2.s3_settings.bucket_name) + self.assertTrue(dest_val2.disabled) + self.assertEqual(10000, config2.check_frequency_in_sec) + + @unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") + def test_can_put_and_update_remote_attachments_configuration(self): + c1 = RemoteAttachmentsConfiguration( + destinations={ + "S3-Users": RemoteAttachmentsDestinationConfiguration( + s3_settings=RemoteAttachmentsS3Settings(bucket_name="testS3Bucket-Users"), + disabled=False, + ) + }, + check_frequency_in_sec=1000, + ) + self.store.maintenance.send(ConfigureRemoteAttachmentsOperation(c1)) + + config = self.store.maintenance.send(GetRemoteAttachmentsConfigurationOperation()) + dest_key, dest_val = next(iter(config.destinations.items())) + self.assertEqual("S3-Users", dest_key) + self.assertEqual("testS3Bucket-Users", dest_val.s3_settings.bucket_name) + self.assertFalse(dest_val.disabled) + self.assertEqual(1000, config.check_frequency_in_sec) + + config.destinations["S3-Orders"] = RemoteAttachmentsDestinationConfiguration( + s3_settings=RemoteAttachmentsS3Settings(bucket_name="testS3Bucket-Orders"), + disabled=True, + ) + config.check_frequency_in_sec = 10000 + self.store.maintenance.send(ConfigureRemoteAttachmentsOperation(config)) + + config2 = self.store.maintenance.send(GetRemoteAttachmentsConfigurationOperation()) + self.assertEqual(2, len(config2.destinations)) + self.assertEqual(10000, config2.check_frequency_in_sec) + + dest_orders = config2.destinations.get("S3-Orders") + self.assertIsNotNone(dest_orders) + self.assertEqual("testS3Bucket-Orders", dest_orders.s3_settings.bucket_name) + self.assertTrue(dest_orders.disabled) + + config3 = self.store.maintenance.send(GetRemoteAttachmentsConfigurationOperation()) + dest_users = config3.destinations.get("S3-Users") + self.assertIsNotNone(dest_users) + self.assertEqual("testS3Bucket-Users", dest_users.s3_settings.bucket_name) + self.assertFalse(dest_users.disabled) + + # ── Validation tests (client-side assert_configuration) ─────────────────── + + def _make_op(self, config: RemoteAttachmentsConfiguration): + """Helper: call ConfigureRemoteAttachmentsOperation constructor (triggers assert_configuration).""" + ConfigureRemoteAttachmentsOperation(config) + + def test_assert_configuration_rejects_both_uploaders(self): + self.assertRaisesWithMessageContaining( + self._make_op, + ValueError, + "Only one uploader for RemoteAttachmentsConfiguration can be configured.", + RemoteAttachmentsConfiguration( + destinations={ + "test": RemoteAttachmentsDestinationConfiguration( + s3_settings=RemoteAttachmentsS3Settings(bucket_name="testS3Bucket"), + azure_settings=RemoteAttachmentsAzureSettings( + account_name="testAzureAccount", storage_container="testAzureContainer" + ), + ) + }, + check_frequency_in_sec=1000, + ), + ) + + def test_assert_configuration_rejects_zero_check_frequency(self): + self.assertRaisesWithMessageContaining( + self._make_op, + ValueError, + "Remote attachments check frequency must be greater than 0.", + RemoteAttachmentsConfiguration( + destinations={ + "test": RemoteAttachmentsDestinationConfiguration( + s3_settings=RemoteAttachmentsS3Settings(bucket_name="testS3Bucket"), + ) + }, + check_frequency_in_sec=0, + ), + ) + + def test_assert_configuration_rejects_zero_max_items(self): + self.assertRaisesWithMessageContaining( + self._make_op, + ValueError, + "Max items to process must be greater than 0.", + RemoteAttachmentsConfiguration( + destinations={ + "test": RemoteAttachmentsDestinationConfiguration( + s3_settings=RemoteAttachmentsS3Settings(bucket_name="testS3Bucket"), + ) + }, + check_frequency_in_sec=1, + max_items_to_process=0, + ), + ) + + def test_assert_configuration_rejects_no_uploader(self): + self.assertRaisesWithMessageContaining( + self._make_op, + ValueError, + "Exactly one uploader for RemoteAttachmentsConfiguration must be configured.", + RemoteAttachmentsConfiguration( + destinations={"test": RemoteAttachmentsDestinationConfiguration(disabled=False)}, + check_frequency_in_sec=1, + max_items_to_process=1, + ), + ) + + def test_assert_configuration_rejects_null_destination(self): + self.assertRaisesWithMessageContaining( + self._make_op, + ValueError, + "Destination configuration for key S3-Users is null", + RemoteAttachmentsConfiguration( + destinations={"S3-Users": None}, + check_frequency_in_sec=1000, + ), + ) + + def test_assert_configuration_rejects_duplicate_keys(self): + # Python dicts enforce unique keys natively, so we test via two separate + # destinations with keys that differ only in case. + self.assertRaisesWithMessageContaining( + self._make_op, + ValueError, + "Destination key 'TEST' is duplicate. Duplicate keys are not allowed in remote attachments configuration", + RemoteAttachmentsConfiguration( + destinations={ + "test": RemoteAttachmentsDestinationConfiguration( + s3_settings=RemoteAttachmentsS3Settings(bucket_name="testS3Bucket"), + ), + "TEST": RemoteAttachmentsDestinationConfiguration( + azure_settings=RemoteAttachmentsAzureSettings( + account_name="testAzureAccount", storage_container="testAzureContainer" + ), + ), + }, + check_frequency_in_sec=1, + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/ravendb/tests/jvm_migrated_tests/client_tests/time_series_tests/test_time_series_configuration.py b/ravendb/tests/jvm_migrated_tests/client_tests/time_series_tests/test_time_series_configuration.py index b4827592..edcd5d91 100644 --- a/ravendb/tests/jvm_migrated_tests/client_tests/time_series_tests/test_time_series_configuration.py +++ b/ravendb/tests/jvm_migrated_tests/client_tests/time_series_tests/test_time_series_configuration.py @@ -2,6 +2,7 @@ import unittest from ravendb import GetDatabaseRecordOperation +from ravendb.exceptions.raven_exceptions import RavenException from ravendb.documents.operations.time_series import ( ConfigureTimeSeriesOperation, TimeSeriesConfiguration, @@ -158,9 +159,9 @@ def test_not_valid_configure_should_throw(self): TimeSeriesPolicy("By30DaysFor5Years", TimeValue.of_days(30), TimeValue.of_years(5)) ] - self.assertRaisesWithMessage( + self.assertRaisesWithMessageContaining( self.store.maintenance.send, - Exception, + RavenException, "Unable to compare 1 month with 30 days, since a month might have different number of days.", ConfigureTimeSeriesOperation(config), ) @@ -177,9 +178,9 @@ def test_not_valid_configure_should_throw(self): TimeSeriesPolicy("By365DaysFor5Years", TimeValue.of_seconds(365 * 24 * 3600), TimeValue.of_years(5)) ] - self.assertRaisesWithMessage( + self.assertRaisesWithMessageContaining( self.store.maintenance.send, - Exception, + RavenException, "Unable to compare 1 year with 365 days, since a month might have different number of days.", ConfigureTimeSeriesOperation(config2), ) @@ -197,9 +198,9 @@ def test_not_valid_configure_should_throw(self): TimeSeriesPolicy("By364daysFor5Years", TimeValue.of_days(364), TimeValue.of_years(5)), ] - self.assertRaisesWithMessage( + self.assertRaisesWithMessageContaining( self.store.maintenance.send, - Exception, + RavenException, "The aggregation time of the policy 'By364daysFor5Years' (364 days) " "must be divided by the aggregation time of 'By27DaysFor1Year' (27 days) without a remainder.", ConfigureTimeSeriesOperation(config3), diff --git a/ravendb/tests/jvm_migrated_tests/client_tests/time_series_tests/test_time_series_operations.py b/ravendb/tests/jvm_migrated_tests/client_tests/time_series_tests/test_time_series_operations.py index e9b236df..3e95dd1f 100644 --- a/ravendb/tests/jvm_migrated_tests/client_tests/time_series_tests/test_time_series_operations.py +++ b/ravendb/tests/jvm_migrated_tests/client_tests/time_series_tests/test_time_series_operations.py @@ -1,6 +1,7 @@ from datetime import datetime, timedelta from ravendb import SessionOptions +from ravendb.exceptions.raven_exceptions import RavenException from ravendb.documents.operations.time_series import ( GetTimeSeriesOperation, TimeSeriesOperation, @@ -572,9 +573,9 @@ def test_should_throw_on_attempt_to_create_time_series_on_missing_document(self) ) time_series_batch = TimeSeriesBatchOperation("users/ayende", time_series_op) - self.assertRaisesWithMessage( + self.assertRaisesWithMessageContaining( self.store.operations.send, - RuntimeError, + RavenException, "Document 'users/ayende' does not exist. Cannot operate on time series of a missing document", time_series_batch, ) diff --git a/ravendb/tests/operations_tests/test_operations.py b/ravendb/tests/operations_tests/test_operations.py index 835b5396..ea612d19 100644 --- a/ravendb/tests/operations_tests/test_operations.py +++ b/ravendb/tests/operations_tests/test_operations.py @@ -1,8 +1,11 @@ -from ravendb.exceptions.exceptions import InvalidOperationException, ErrorResponseException +from ravendb.exceptions.documents.indexes import IndexDoesNotExistException +from ravendb.exceptions.exceptions import ErrorResponseException from ravendb.documents.indexes.definitions import IndexDefinition from ravendb.documents.operations.attachments import ( + AttachmentRequest, PutAttachmentOperation, DeleteAttachmentOperation, + DeleteAttachmentsOperation, ) from ravendb.documents.operations.indexes import PutIndexesOperation from ravendb.documents.operations.misc import QueryOperationOptions, DeleteByQueryOperation @@ -49,6 +52,35 @@ def test_delete_attachment(self): attachments = metadata.metadata.get(constants.Documents.Metadata.ATTACHMENTS, None) self.assertFalse(attachments) # 0 or None + def test_delete_attachments_bulk(self): + # store two attachments on the same document + self.store.operations.send(PutAttachmentOperation("users/1-A", "pic1.png", b"\x01\x02\x03", "image/png")) + self.store.operations.send(PutAttachmentOperation("users/1-A", "pic2.png", b"\x04\x05\x06", "image/png")) + + with self.store.open_session() as session: + user = session.load("users/1-A") + attachments = session.advanced.get_metadata_for(user).metadata.get( + constants.Documents.Metadata.ATTACHMENTS, [] + ) + self.assertEqual(2, len(attachments)) + + # bulk-delete both in one request + self.store.operations.send( + DeleteAttachmentsOperation( + [ + AttachmentRequest("users/1-A", "pic1.png"), + AttachmentRequest("users/1-A", "pic2.png"), + ] + ) + ) + + with self.store.open_session() as session: + user = session.load("users/1-A") + attachments = session.advanced.get_metadata_for(user).metadata.get( + constants.Documents.Metadata.ATTACHMENTS, None + ) + self.assertFalse(attachments) # both gone + def test_patch_by_index(self): index = IndexDefinition() index.name = "Patches" @@ -89,7 +121,6 @@ def test_patch_by_index(self): for v in values: self.assertTrue(v) - @unittest.skip("Exception dispatcher") def test_fail_patch_wrong_index_name(self): options = QueryOperationOptions(allow_stale=False, retrieve_details=True) query = IndexQuery("from index 'None' update {{this.name='NotExist'}}") @@ -98,7 +129,7 @@ def test_fail_patch_wrong_index_name(self): query_to_update=query, options=options, ) - with self.assertRaises(InvalidOperationException): + with self.assertRaises(IndexDoesNotExistException): response = self.store.operations.send(operation) if response: operation = NewOperation( diff --git a/ravendb/tests/operations_tests/test_schema_validation.py b/ravendb/tests/operations_tests/test_schema_validation.py new file mode 100644 index 00000000..1c694b50 --- /dev/null +++ b/ravendb/tests/operations_tests/test_schema_validation.py @@ -0,0 +1,448 @@ +import json +import os +import unittest + +from ravendb.documents.indexes.definitions import FieldStorage, IndexDefinition, IndexFieldOptions +from ravendb.documents.operations.indexes import PutIndexesOperation, ResetIndexOperation +from ravendb.documents.operations.schema_validation import ( + ConfigureSchemaValidationOperation, + GetSchemaValidationConfiguration, + SchemaDefinition, + SchemaValidationConfiguration, + StartSchemaValidationOperation, + ValidateSchemaResult, +) +from ravendb.exceptions.raven_exceptions import RavenException, SchemaValidationException +from ravendb.tests.test_base import TestBase + +# Minimal JSON Schema used across tests +_SCHEMA_REQUIRE_NAME = json.dumps( + { + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + } +) + +_SCHEMA_REQUIRE_AGE = json.dumps( + { + "type": "object", + "properties": {"age": {"type": "integer"}}, + "required": ["age"], + } +) + +# Schema used in indexing tests: Prop must be ≤ 10 chars +_SCHEMA_PROP_MAX_LENGTH_10 = json.dumps( + { + "properties": {"Prop": {"maxLength": 10}}, + } +) + +# Schema used in indexing tests: Prop ≤ 10 chars AND must match "^something", plus Prop1 required +_SCHEMA_PROP_MULTIPLE_RULES = json.dumps( + { + "properties": {"Prop": {"maxLength": 10, "pattern": "^something"}}, + "required": ["Prop1"], + } +) + +# Map that projects Schema.GetErrorsFor(doc) into an Errors field (LINQ syntax) +_MAP_VALIDATE_DOCUMENT = ( + "from doc in docs " + 'where MetadataFor(doc)["@collection"] != "@hilo" ' + "select new { Id = doc.Id, Errors = Schema.GetErrorsFor(doc) }" +) + + +class User: + def __init__(self, Id: str = None, name: str = None, age: int = None): + self.Id = Id + self.name = name + self.age = age + + +class TestSchemaValidation(TestBase): + def setUp(self): + super().setUp() + + def tearDown(self): + super().tearDown() + TestBase.delete_all_topology_files() + + # ------------------------------------------------------------------ + # Ported from SchemaValidationBasicTests.cs + # ------------------------------------------------------------------ + + @unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") + def test_can_configure_schema_validation(self): + """Configure a schema for Users collection and read it back.""" + schema_def = SchemaDefinition(schema=_SCHEMA_REQUIRE_NAME) + config = SchemaValidationConfiguration(validators_per_collection={"Users": schema_def}) + + self.store.maintenance.send(ConfigureSchemaValidationOperation(config)) + + result = self.store.maintenance.send(GetSchemaValidationConfiguration()) + self.assertIsNotNone(result) + self.assertIn("Users", result.validators_per_collection) + stored = result.validators_per_collection["Users"] + self.assertEqual(_SCHEMA_REQUIRE_NAME, stored.schema) + + @unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") + def test_schema_validation_blocks_invalid_document(self): + """Saving a document that violates the schema raises SchemaValidationException.""" + schema_def = SchemaDefinition(schema=_SCHEMA_REQUIRE_NAME) + config = SchemaValidationConfiguration(validators_per_collection={"Users": schema_def}) + self.store.maintenance.send(ConfigureSchemaValidationOperation(config)) + + with self.assertRaises(SchemaValidationException): + with self.store.open_session() as session: + # User without 'name' — violates the schema + user = User(age=30) + session.store(user, "users/1") + session.save_changes() + + @unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") + def test_schema_validation_allows_valid_document(self): + """Saving a document that satisfies the schema succeeds.""" + schema_def = SchemaDefinition(schema=_SCHEMA_REQUIRE_NAME) + config = SchemaValidationConfiguration(validators_per_collection={"Users": schema_def}) + self.store.maintenance.send(ConfigureSchemaValidationOperation(config)) + + with self.store.open_session() as session: + user = User(name="Alice") + session.store(user, "users/1") + session.save_changes() + + with self.store.open_session() as session: + loaded = session.load("users/1", User) + self.assertEqual("Alice", loaded.name) + + @unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") + def test_disabled_schema_allows_invalid_document(self): + """When the schema is disabled, invalid documents are accepted.""" + schema_def = SchemaDefinition(schema=_SCHEMA_REQUIRE_NAME, disabled=True) + config = SchemaValidationConfiguration(validators_per_collection={"Users": schema_def}) + self.store.maintenance.send(ConfigureSchemaValidationOperation(config)) + + with self.store.open_session() as session: + # No 'name' field — would fail if schema were active + user = User(age=99) + session.store(user, "users/1") + session.save_changes() + + with self.store.open_session() as session: + loaded = session.load("users/1", User) + self.assertEqual(99, loaded.age) + + # ------------------------------------------------------------------ + # Ported from SchemaValidationOperationTests.cs + # ------------------------------------------------------------------ + + def test_start_schema_validation_operation_returns_result(self): + """StartSchemaValidationOperation completes and returns a ValidateSchemaResult.""" + with self.store.open_session() as session: + session.store(User(name="Alice"), "users/1") + session.store(User(name="Bob"), "users/2") + session.save_changes() + + params = StartSchemaValidationOperation.Parameters( + schema_definition=_SCHEMA_REQUIRE_NAME, + collection="Users", + ) + op = self.store.maintenance.send_async(StartSchemaValidationOperation(params)) + op.wait_for_completion() + + status = op.fetch_operations_status() + result = ValidateSchemaResult.from_json(status["Result"]) + + self.assertIsNotNone(result) + self.assertEqual(2, result.validated_count) + self.assertEqual(0, result.error_count) + self.assertEqual({}, result.errors) + + def test_start_schema_validation_operation_reports_errors(self): + """StartSchemaValidationOperation reports documents that violate the schema.""" + # Insert documents without a schema active so they bypass write-time validation + with self.store.open_session() as session: + session.store(User(name="Alice"), "users/1") + session.store(User(age=30), "users/2") # missing 'name' — will fail audit + session.save_changes() + + params = StartSchemaValidationOperation.Parameters( + schema_definition=_SCHEMA_REQUIRE_NAME, + collection="Users", + ) + op = self.store.maintenance.send_async(StartSchemaValidationOperation(params)) + op.wait_for_completion() + + status = op.fetch_operations_status() + result = ValidateSchemaResult.from_json(status["Result"]) + + self.assertEqual(2, result.validated_count) + self.assertEqual(1, result.error_count) + self.assertIn("users/2", result.errors) + + def test_start_schema_validation_operation_with_max_error_messages(self): + """max_error_messages caps the number of error entries returned.""" + with self.store.open_session() as session: + for i in range(5): + session.store(User(age=i), f"users/{i + 1}") # all missing 'name' + session.save_changes() + + params = StartSchemaValidationOperation.Parameters( + schema_definition=_SCHEMA_REQUIRE_NAME, + collection="Users", + max_error_messages=2, + ) + op = self.store.maintenance.send_async(StartSchemaValidationOperation(params)) + op.wait_for_completion() + + status = op.fetch_operations_status() + result = ValidateSchemaResult.from_json(status["Result"]) + + self.assertEqual(5, result.validated_count) + self.assertEqual(5, result.error_count) + # Only 2 error messages should be stored despite 5 failures + self.assertLessEqual(len(result.errors), 2) + + # ------------------------------------------------------------------ + # Progress tests + # ------------------------------------------------------------------ + def test_validate_schema_start_etag_skips_earlier_docs(self): + """Using start_etag from a first run skips already-validated documents. + + Ported from ValidateSchemaOperation_WhenSettingEtagOnNonSharded_ShouldStartFromTheEtag. + """ + # Schema that requires 'name' to be a string — both docs violate it (age-only) + with self.store.open_session() as session: + session.store(User(age=1), "users/1") # inserted first → lower etag + session.store(User(age=2), "users/2") # inserted second → higher etag + session.save_changes() + + params1 = StartSchemaValidationOperation.Parameters( + schema_definition=_SCHEMA_REQUIRE_NAME, + collection="Users", + max_documents_to_validate=1, # only scan the first doc + ) + op1 = self.store.maintenance.send_async(StartSchemaValidationOperation(params1)) + op1.wait_for_completion() + result1 = ValidateSchemaResult.from_json(op1.fetch_operations_status()["Result"]) + + self.assertEqual(1, result1.validated_count) + self.assertEqual(1, result1.error_count) + self.assertGreater(result1.last_etag, 0) + + # Second run starts from where the first left off + params2 = StartSchemaValidationOperation.Parameters( + schema_definition=_SCHEMA_REQUIRE_NAME, + collection="Users", + start_etag=result1.last_etag + 1, + ) + op2 = self.store.maintenance.send_async(StartSchemaValidationOperation(params2)) + op2.wait_for_completion() + result2 = ValidateSchemaResult.from_json(op2.fetch_operations_status()["Result"]) + + # The second run should only see users/2 + self.assertEqual(1, result2.validated_count) + self.assertEqual(1, result2.error_count) + self.assertIn("users/2", result2.errors) + self.assertNotIn("users/1", result2.errors) + + def test_validate_schema_max_documents_to_validate_caps_scan(self): + """max_documents_to_validate limits how many documents are scanned.""" + with self.store.open_session() as session: + for i in range(6): + session.store(User(age=i), f"users/{i + 1}") # all missing 'name' + session.save_changes() + + params = StartSchemaValidationOperation.Parameters( + schema_definition=_SCHEMA_REQUIRE_NAME, + collection="Users", + max_documents_to_validate=3, + ) + op = self.store.maintenance.send_async(StartSchemaValidationOperation(params)) + op.wait_for_completion() + result = ValidateSchemaResult.from_json(op.fetch_operations_status()["Result"]) + + # Only 3 of the 6 documents should have been scanned + self.assertEqual(3, result.validated_count) + self.assertEqual(3, result.error_count) + # last_etag is set so a follow-up run can continue from here + self.assertGreater(result.last_etag, 0) + + +# --------------------------------------------------------------------------- +# Helper result class for indexing tests +# --------------------------------------------------------------------------- +class _IndexResult: + """Projection class for the Errors field produced by the schema-validation index.""" + + def __init__(self, Id: str = None, Errors: list = None): + self.Id = Id + self.Errors = Errors + + +class TestSchemaValidationIndexing(TestBase): + """ + Ported from SlowTests.Server.Documents.Indexing.SchemaValidationIndexingTests (C#). + + These tests verify that an index can project Schema.GetErrorsFor(doc) into an + Errors field, and that the server correctly populates it based on either an + index-level or database-level schema definition. + """ + + def setUp(self): + super().setUp() + + def tearDown(self): + super().tearDown() + TestBase.delete_all_topology_files() + + def _put_schema_index(self, index_name: str, schema_definitions: dict = None) -> IndexDefinition: + """Create and register an index that projects Schema.GetErrorsFor(doc) → Errors.""" + index_def = IndexDefinition( + name=index_name, + maps={_MAP_VALIDATE_DOCUMENT}, + fields={"Errors": IndexFieldOptions(storage=FieldStorage.YES)}, + schema_definitions=schema_definitions, + ) + self.store.maintenance.send(PutIndexesOperation(index_def)) + return index_def + + # ------------------------------------------------------------------ + # IndexingSchemaErrors_WhenFailsOneRule_ShouldGetTheError + # ------------------------------------------------------------------ + def test_indexing_schema_errors_when_fails_one_rule_should_get_the_error(self): + """ + An index with a schema_definition that limits Prop to 10 chars should + project a non-null Errors list for the violating document and null for the valid one. + """ + invalid_doc_id = "testobjs/invalid" + valid_doc_id = "testobjs/valid" + + index_def = self._put_schema_index( + "IndexWithSchemaValidation", + schema_definitions={"TestObjs": _SCHEMA_PROP_MAX_LENGTH_10}, + ) + + with self.store.open_session() as session: + session.store({"Prop": "0123456789a"}, invalid_doc_id) # 11 chars — violates maxLength:10 + session.store({"Prop": "01"}, valid_doc_id) + session.save_changes() + + self.wait_for_indexing(self.store) + + with self.store.open_session() as session: + results = list( + session.query_index(index_def.name, _IndexResult).select_fields(_IndexResult, "Id", "Errors") + ) + by_id = {r.Id: r for r in results} + + # Server returns None or [] for a valid document + self.assertFalse(by_id[valid_doc_id].Errors) + + errors = by_id[invalid_doc_id].Errors + self.assertTrue(errors) + self.assertEqual(1, len(errors)) + self.assertIn("Prop", errors[0]) + + # ------------------------------------------------------------------ + # IndexingSchemaErrors_WhenFailsMultipleRules_ShouldGetTheErrors + # ------------------------------------------------------------------ + def test_indexing_schema_errors_when_fails_multiple_rules_should_get_the_errors(self): + """ + When a document violates multiple schema rules (maxLength, pattern, required), + all error messages should appear in the projected Errors list. + """ + index_def = self._put_schema_index( + "IndexWithSchemaValidation", + schema_definitions={"TestObjs": _SCHEMA_PROP_MULTIPLE_RULES}, + ) + + with self.store.open_session() as session: + session.store({"Prop": "0123456789a"}, "testobjs/1") + session.store({"Prop": "0123456789a"}, "testobjs/2") + session.save_changes() + + self.wait_for_indexing(self.store) + + with self.store.open_session() as session: + results = list( + session.query_index(index_def.name, _IndexResult).select_fields(_IndexResult, "Id", "Errors") + ) + for result in results: + self.assertIsNotNone(result.Errors) + # Expect 3 violations: maxLength, pattern, required Prop1 + self.assertEqual(3, len(result.Errors)) + + # ------------------------------------------------------------------ + # IndexingSchemaErrors_WhenDefineSchemaOnMetadata_ShouldReject + # ------------------------------------------------------------------ + def test_indexing_schema_errors_when_define_schema_on_metadata_should_reject(self): + """ + Defining a schema rule on the @metadata key should be rejected by the server + with a RavenException containing 'Define a schema validation on metadata is not allowed'. + """ + schema_on_metadata = json.dumps({"properties": {"@metadata": {"maxLength": 10}}}) + + index_def = IndexDefinition( + name="IndexWithSchemaValidation", + maps={_MAP_VALIDATE_DOCUMENT}, + fields={"Errors": IndexFieldOptions(storage=FieldStorage.YES)}, + schema_definitions={"TestObjs": schema_on_metadata}, + ) + + self.assertRaisesWithMessageContaining( + self.store.maintenance.send, + RavenException, + "Define a schema validation on metadata is not allowed", + PutIndexesOperation(index_def), + ) + + # ------------------------------------------------------------------ + # IndexingSchemaErrors_WhenSchemaDefinedInDatabase_ShouldIndexErrors + # ------------------------------------------------------------------ + @unittest.skipIf(os.environ.get("RAVENDB_LICENSE") is None, "Insufficient license permissions. Skipping on CI/CD.") + def test_indexing_schema_errors_when_schema_defined_in_database_should_index_errors(self): + """ + When no schema_definitions are set on the index itself, but a database-level + schema is configured via ConfigureSchemaValidationOperation, resetting the index + should cause it to pick up the DB schema and project errors correctly. + """ + invalid_doc_id = "testobjs/invalid" + valid_doc_id = "testobjs/valid" + + # Index without schema_definitions — no validation yet + index_def = self._put_schema_index("IndexWithSchemaValidation") + + with self.store.open_session() as session: + session.store({"Prop": "0123456789a"}, invalid_doc_id) + session.store({"Prop": "01"}, valid_doc_id) + session.save_changes() + + self.wait_for_indexing(self.store) + + # Now configure a DB-level schema for the TestObjs collection + config = SchemaValidationConfiguration( + validators_per_collection={"TestObjs": SchemaDefinition(schema=_SCHEMA_PROP_MAX_LENGTH_10)} + ) + self.store.maintenance.send(ConfigureSchemaValidationOperation(config)) + + # Reset the index so it re-indexes all documents with the new schema + self.store.maintenance.send(ResetIndexOperation(index_def.name)) + self.wait_for_indexing(self.store) + + with self.store.open_session() as session: + results = list( + session.query_index(index_def.name, _IndexResult).select_fields(_IndexResult, "Id", "Errors") + ) + by_id = {r.Id: r for r in results} + + # Server returns None or [] for a valid document + self.assertFalse(by_id[valid_doc_id].Errors) + + errors = by_id[invalid_doc_id].Errors + self.assertTrue(errors) + self.assertEqual(1, len(errors)) + self.assertIn("Prop", errors[0]) diff --git a/setup.py b/setup.py index 734ae841..5495dc94 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="ravendb", packages=find_packages(exclude=["*.tests.*", "tests", "*.tests", "tests.*"]), - version="7.1.5", + version="7.2.0", long_description_content_type="text/markdown", long_description=open("README_pypi.md").read(), description="Python client for RavenDB NoSQL Database",