diff --git a/backend/drive/migrations/0014_private_quota_within_assigned.py b/backend/drive/migrations/0014_private_quota_within_assigned.py new file mode 100644 index 0000000..081d748 --- /dev/null +++ b/backend/drive/migrations/0014_private_quota_within_assigned.py @@ -0,0 +1,21 @@ +"""Count private files in their owner's assigned quota, without reservation. + +Only the SQL parent changes. Files remain under storage/privates/ (or at their +legacy path until move_private_storage is run). 50 Go is a cap, not a reserve. +""" +from django.db import migrations + + +def attach_privates(apps, schema_editor): + Space = apps.get_model('drive', 'DriveSpace') + quota = 50 * 1024 ** 3 + for private in Space.objects.filter(private_owner__isnull=False).iterator(): + assigned = Space.objects.filter(user_id=private.private_owner_id, private_owner__isnull=True).first() + if not assigned: + raise RuntimeError(f'Le compte {private.private_owner_id} ne possède pas d’espace attribué.') + Space.objects.filter(pk=private.pk).update(parent_id=assigned.pk, quota_bytes=quota) + + +class Migration(migrations.Migration): + dependencies = [('drive', '0013_private_space_quota')] + operations = [migrations.RunPython(attach_privates, migrations.RunPython.noop)] diff --git a/backend/drive/quotas.py b/backend/drive/quotas.py index c5aff4b..4279ba0 100644 --- a/backend/drive/quotas.py +++ b/backend/drive/quotas.py @@ -1,15 +1,15 @@ -"""Space quotas, including descendants, trash, and reserved resumable uploads. +"""Space quotas, including logical descendants, trash, and pending uploads. A child can be constrained by both its own allocation and its ancestors'. Call assert_space_capacity() inside transaction.atomic() before reserving/writing. """ from contextlib import contextmanager from django.db import transaction -from django.db.models import Q, Sum +from django.db.models import Sum from .models import DriveItem, DriveSpace, UploadSession MAX_QUOTA_BYTES = 9_007_199_254_740_991 # exact integer in JS too -PRIVATE_SPACE_QUOTA_BYTES = 50 * 1024 ** 3 # 50 Gio per private account (shown as 50 Go) +PRIVATE_SPACE_QUOTA_BYTES = 50 * 1024 ** 3 # Maximum privé, non réservé dans le quota attribué class QuotaExceeded(Exception): @@ -47,14 +47,24 @@ def ancestry(space): def subtree_space_ids(space): - """Use the stable DB path rather than a filesystem traversal.""" - # A shared root contains only shared spaces: private spaces have their - # own quota and must never consume/reserve shared root capacity. - if space.parent_id is None and space.relative_path == '': - return DriveSpace.objects.filter(private_owner__isnull=True).values_list('pk', flat=True) - return DriveSpace.objects.filter( - Q(relative_path=space.relative_path) | Q(relative_path__startswith=space.relative_path + '/') - ).values_list('pk', flat=True) + """Resolve the logical quota subtree, not the filesystem path. + + Private accounts store files below storage/privates/ but are children of + their assigned spaces *for quota accounting*. Physical path-prefix + queries would silently omit their real disk usage. + """ + by_parent = {} + for pk, parent_id in DriveSpace.objects.values_list('pk', 'parent_id'): + by_parent.setdefault(parent_id, []).append(pk) + result = set() + to_visit = [space.pk] + while to_visit: + pk = to_visit.pop() + if pk in result: + raise ValueError('Hiérarchie des espaces invalide.') + result.add(pk) + to_visit.extend(by_parent.get(pk, [])) + return result def occupancy(space): @@ -98,9 +108,9 @@ def trash_occupancy(space): def _budget_tree(space): """One DB snapshot of all descendant spaces and their direct physical usage. - A child's fixed quota reserves that much capacity for it; a child without - a quota reserves only its actual consumption (including its descendants). - Thus quotas of siblings cannot silently overcommit their common parent. + Shared descendants retain the existing reservation rules for assigned + quotas. A private space's 50-Go cap is NEVER preallocated: its parent counts + only its real files and pending uploads (including its own descendants). """ ids=list(subtree_space_ids(space)) nodes={node.pk:node for node in DriveSpace.objects.filter(pk__in=ids)} @@ -115,7 +125,11 @@ def _budget_tree(space): memo={} def committed(pk): if pk not in memo: - memo[pk]=direct[pk]+sum(max(nodes[c].quota_bytes or 0,committed(c)) for c in children[pk]) + memo[pk] = direct[pk] + sum( + committed(child) if nodes[child].private_owner_id + else max(nodes[child].quota_bytes or 0, committed(child)) + for child in children[pk] + ) return memo[pk] committed(space.pk) return memo @@ -129,8 +143,9 @@ def _committed(space): def available_for_growth(space): """Bytes that can be added directly to space without breaking any ancestor allocation. - Capacity already reserved for this child does not count against its parent's - available budget a second time. A zero-byte quota correctly permits no growth. + Quotas of normal assigned children can absorb growth already reserved in + their ancestors. For private children, the 50-Go cap is NOT reserved: every + newly uploaded byte immediately reduces the parent's available budget. """ maximum=MAX_QUOTA_BYTES absorbed=0 @@ -140,7 +155,9 @@ def available_for_growth(space): committed=committed_tree[ancestor.pk] if ancestor.quota_bytes is not None: maximum=min(maximum,ancestor.quota_bytes-committed+absorbed) - absorbed+=max(0,ancestor.quota_bytes-committed) + # Private caps constrain uploads, but never absorb parent capacity. + if not ancestor.private_owner_id: + absorbed+=max(0,ancestor.quota_bytes-committed) return None if maximum==MAX_QUOTA_BYTES else max(0,int(maximum)) @@ -203,7 +220,7 @@ def locked_quota_tree(space): def assert_space_capacity(space, additional_bytes): - """Called with ancestor locks; include sibling allocations when checking uploads.""" + """Called with ancestor locks; private uploads count toward both caps.""" available=available_for_growth(space) if available is not None and additional_bytes>available: raise QuotaExceeded( diff --git a/backend/drive/spaces.py b/backend/drive/spaces.py index 42aa2d9..cb90244 100644 --- a/backend/drive/spaces.py +++ b/backend/drive/spaces.py @@ -1,4 +1,4 @@ -"""Physical numbered directories and strict permissions between personal/shared spaces.""" +"""Private files stored separately but counted against the assigned-space quota.""" import hashlib import secrets from pathlib import Path @@ -34,10 +34,13 @@ def ensure_space(user): def _make_numbered_space(parent, **fields): - # Private directories live in a separate physical and quota hierarchy. + # Private directories are physically separate, but logically child spaces + # of the account's assigned space so their *actual bytes* are included. private = fields.get('private_owner') is not None - if private != (parent is None): - raise ValueError('Hiérarchie de stockage privée invalide.') + if parent is None or parent.private_owner_id: + raise ValueError('Un espace doit avoir un parent partagé.') + if private and parent.user_id != fields['private_owner'].pk: + raise ValueError('L’espace privé doit être rattaché au quota attribué à son propriétaire.') base = 'privates' if private else parent.relative_path if len(base) + 1 + 24 > 700: raise ValueError('Hiérarchie trop profonde.') @@ -95,8 +98,9 @@ def ensure_private_space(user): obj = DriveSpace.objects.select_for_update().filter(private_owner=user).first() if obj: return obj - ensure_space(user) # Account must still have its assigned/shared space. - return _make_numbered_space(None, private_owner=user, display_name='Mes fichiers') + assigned = ensure_space(user) + # The 50-Go cap is only a limit on private uploads, not a reservation. + return _make_numbered_space(assigned, private_owner=user, display_name='Mes fichiers') def my_space(user): diff --git a/backend/drive/test_private_layout.py b/backend/drive/test_private_layout.py index 8941659..fe8920a 100644 --- a/backend/drive/test_private_layout.py +++ b/backend/drive/test_private_layout.py @@ -1,4 +1,4 @@ -"""Focused regressions for separate private storage and its 50 Go quota. +"""Private disk layout, 50-Go private cap, and real-byte pooled accounting. Run with: USE_SQLITE=true python manage.py test drive.test_private_layout """ @@ -15,7 +15,8 @@ from django.test import TestCase, override_settings from rest_framework.test import APIClient from .models import DriveItem, DriveSpace -from .quotas import PRIVATE_SPACE_QUOTA_BYTES, QuotaExceeded, assert_space_capacity, locked_quota_tree, occupancy, quota_state +from .quotas import (PRIVATE_SPACE_QUOTA_BYTES, QuotaExceeded, assert_space_capacity, + assert_updated_quota, locked_quota_tree, occupancy, quota_state) from .spaces import ensure_private_space, ensure_space, create_space, storage_path @@ -38,7 +39,7 @@ class PrivateStorageTests(TestCase): self.client.force_authenticate(user=self.admin) def test_private_directories_are_separate_from_shared_spaces(self): - self.assertIsNone(self.private.parent_id) + self.assertEqual(self.private.parent_id, self.shared.pk) self.assertEqual(self.private.relative_path, f'privates/{self.private.directory_name}') self.assertEqual(self.private.quota_bytes, PRIVATE_SPACE_QUOTA_BYTES) self.assertTrue((self.rootdir / self.private.relative_path).is_dir()) @@ -47,12 +48,15 @@ class PrivateStorageTests(TestCase): private = ensure_private_space(person) self.assertTrue(shared.relative_path.startswith(f'{shared.directory_name}')) self.assertEqual(private.relative_path, f'privates/{private.directory_name}') - self.assertIsNone(private.parent_id) + self.assertEqual(private.parent_id, shared.pk) self.assertNotEqual(shared.directory_name, private.directory_name) - def test_private_quota_does_not_consume_shared_parent(self): - self.shared.quota_bytes = 1 + def test_private_usage_counts_toward_assigned_space_without_reserving(self): + self.shared.quota_bytes = 100 self.shared.save(update_fields=['quota_bytes']) + # Private can be 50 Go even when the assigned budget is just 100 bytes. + self.assertEqual(quota_state(self.shared)['available_bytes'], 100) + self.assertEqual(quota_state(self.private)['available_bytes'], 100) created = self.client.post('/api/drive/uploads/', { 'files': [SimpleUploadedFile('photo.jpg', b'abcdef')] }, format='multipart') @@ -60,7 +64,9 @@ class PrivateStorageTests(TestCase): node = DriveItem.objects.get(pk=created.data['items'][0]['id']) self.assertEqual(node.storage_location, f'{self.private.relative_path}/photo.jpg') self.assertEqual(storage_path(node).read_bytes(), b'abcdef') - self.assertEqual(occupancy(self.shared)[0], 0) + self.assertEqual(occupancy(self.shared)[0], 6) + self.assertEqual(quota_state(self.shared)['available_bytes'], 94) + self.assertEqual(quota_state(self.private)['available_bytes'], 94) self.assertEqual(quota_state(self.private)['quota_bytes'], PRIVATE_SPACE_QUOTA_BYTES) self.assertEqual(quota_state(self.private)['used_bytes'], 6) rejected = self.client.patch(f'/api/drive/spaces/{self.private.pk}/quota/', { @@ -68,6 +74,78 @@ class PrivateStorageTests(TestCase): }, format='json') self.assertEqual(rejected.status_code, 403) + def test_actual_private_use_is_pooled_in_100_go_assigned_space(self): + person = get_user_model().objects.create_user('quota-user', password='TestPassword123!') + assigned = create_space(person, self.shared, '100 Go', quota_bytes=100 * 1024 ** 3) + private = ensure_private_space(person) + gib = 1024 ** 3 + self.assertEqual(private.parent_id, assigned.pk) + self.assertEqual(private.relative_path, f'privates/{private.directory_name}') + self.assertEqual(quota_state(assigned)['available_bytes'], 100 * gib) + self.assertEqual(quota_state(private)['available_bytes'], 50 * gib) + # Existing 20 Go in shared space. This leaves both limits at their max. + DriveItem.objects.create( + owner=person, space=assigned, name='shared.bin', + storage_key=str(uuid.uuid4()), storage_location=f'{assigned.relative_path}/shared.bin', + size=20 * gib, + ) + self.assertEqual(quota_state(assigned)['used_bytes'], 20 * gib) + self.assertEqual(quota_state(private)['used_bytes'], 0) + DriveItem.objects.create( + owner=person, space=private, name='private.bin', + storage_key=str(uuid.uuid4()), storage_location=f'{private.relative_path}/private.bin', + size=20 * gib, + ) + self.assertEqual(occupancy(assigned)[0], 40 * gib) + self.assertEqual(quota_state(assigned)['used_bytes'], 40 * gib) + self.assertEqual(quota_state(assigned)['available_bytes'], 60 * gib) + self.assertEqual(quota_state(private)['used_bytes'], 20 * gib) + self.assertEqual(quota_state(private)['available_bytes'], 30 * gib) + with transaction.atomic(): + with locked_quota_tree(assigned): + assert_space_capacity(assigned, 60 * gib) + with self.assertRaises(QuotaExceeded): + assert_space_capacity(assigned, 60 * gib + 1) + assert_updated_quota(assigned, 40 * gib) + with self.assertRaisesRegex(ValueError, 'fichiers'): + assert_updated_quota(assigned, 39 * gib) + + def test_parent_limit_blocks_private_even_before_50_go(self): + self.shared.quota_bytes = 20 + self.shared.save(update_fields=['quota_bytes']) + DriveItem.objects.create( + owner=self.admin, space=self.private, name='existing.bin', + storage_key=str(uuid.uuid4()), storage_location=f'{self.private.relative_path}/existing.bin', size=19, + ) + self.assertEqual(quota_state(self.private)['available_bytes'], 1) + with transaction.atomic(): + with locked_quota_tree(self.private): + assert_space_capacity(self.private, 1) + with self.assertRaises(QuotaExceeded): + assert_space_capacity(self.private, 2) + + def test_invitations_allow_less_than_50_go_no_reservation(self): + self.shared.quota_bytes = 15 + self.shared.save(update_fields=['quota_bytes']) + res = self.client.post('/api/drive/invitations/', {'quota_bytes': 10}, format='json') + self.assertEqual(res.status_code, 201, res.data) + self.assertEqual(quota_state(self.shared)['available_bytes'], 15) + member = get_user_model().objects.create_user('tiny-private', password='TestPassword123!') + assigned = create_space(member, self.shared, '10 octets', quota_bytes=10) + private = ensure_private_space(member) + self.assertEqual(private.quota_bytes, PRIVATE_SPACE_QUOTA_BYTES) + self.assertEqual(quota_state(assigned)['available_bytes'], 10) + self.assertEqual(quota_state(private)['available_bytes'], 10) + + def test_private_pending_uploads_count_in_parent(self): + from .models import UploadSession + UploadSession.objects.create( + owner=self.admin, space=self.private, name='movie.bin', total_size=1234 + ) + self.assertEqual(quota_state(self.shared)['reserved_bytes'], 1234) + self.assertEqual(quota_state(self.shared)['used_bytes'], 0) + self.assertEqual(quota_state(self.private)['available_bytes'], PRIVATE_SPACE_QUOTA_BYTES - 1234) + def test_private_50_go_upload_limit_is_enforced(self): DriveItem.objects.create( owner=self.admin, space=self.private, name='large.bin', @@ -114,7 +192,7 @@ class PrivateStorageTests(TestCase): self.private.refresh_from_db() item.refresh_from_db() self.assertEqual(self.private.relative_path, f'privates/{self.private.directory_name}') - self.assertIsNone(self.private.parent_id) + self.assertEqual(self.private.parent_id, self.shared.pk) self.assertEqual(self.private.quota_bytes, PRIVATE_SPACE_QUOTA_BYTES) self.assertEqual(item.storage_location, f'{self.private.relative_path}/Documents/facture.pdf') self.assertEqual(storage_path(item).read_bytes(), b'PDF CONTENT') diff --git a/backend/drive/views_spaces.py b/backend/drive/views_spaces.py index 2e3f7f1..671062b 100644 --- a/backend/drive/views_spaces.py +++ b/backend/drive/views_spaces.py @@ -18,7 +18,8 @@ from rest_framework.response import Response from .models import DriveSpace, SpaceInvitation, SpaceGrant, DriveItem, UploadSession from .services import error, valid_name from .spaces import ensure_space, ensure_private_space, my_space, create_space, _make_numbered_space, is_admin, hash_token, space_info, can_read_space, can_manage_space -from .quotas import parse_quota, locked_quota_tree, assert_new_child_quota, assert_updated_quota +from .quotas import (parse_quota, locked_quota_tree, + assert_new_child_quota, assert_updated_quota) def permission_flags(data):