test
This commit is contained in:
Louis Mazin
parent
9c0c7b65dc
commit
ac4c0f4d6a
@ -0,0 +1,21 @@
|
||||
"""Count private files in their owner's assigned quota, without reservation.
|
||||
|
||||
Only the SQL parent changes. Files remain under storage/privates/ (or at their
|
||||
legacy path until move_private_storage is run). 50 Go is a cap, not a reserve.
|
||||
"""
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
def attach_privates(apps, schema_editor):
|
||||
Space = apps.get_model('drive', 'DriveSpace')
|
||||
quota = 50 * 1024 ** 3
|
||||
for private in Space.objects.filter(private_owner__isnull=False).iterator():
|
||||
assigned = Space.objects.filter(user_id=private.private_owner_id, private_owner__isnull=True).first()
|
||||
if not assigned:
|
||||
raise RuntimeError(f'Le compte {private.private_owner_id} ne possède pas d’espace attribué.')
|
||||
Space.objects.filter(pk=private.pk).update(parent_id=assigned.pk, quota_bytes=quota)
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [('drive', '0013_private_space_quota')]
|
||||
operations = [migrations.RunPython(attach_privates, migrations.RunPython.noop)]
|
||||
@ -1,15 +1,15 @@
|
||||
"""Space quotas, including descendants, trash, and reserved resumable uploads.
|
||||
"""Space quotas, including logical descendants, trash, and pending uploads.
|
||||
|
||||
A child can be constrained by both its own allocation and its ancestors'.
|
||||
Call assert_space_capacity() inside transaction.atomic() before reserving/writing.
|
||||
"""
|
||||
from contextlib import contextmanager
|
||||
from django.db import transaction
|
||||
from django.db.models import Q, Sum
|
||||
from django.db.models import Sum
|
||||
from .models import DriveItem, DriveSpace, UploadSession
|
||||
|
||||
MAX_QUOTA_BYTES = 9_007_199_254_740_991 # exact integer in JS too
|
||||
PRIVATE_SPACE_QUOTA_BYTES = 50 * 1024 ** 3 # 50 Gio per private account (shown as 50 Go)
|
||||
PRIVATE_SPACE_QUOTA_BYTES = 50 * 1024 ** 3 # Maximum privé, non réservé dans le quota attribué
|
||||
|
||||
|
||||
class QuotaExceeded(Exception):
|
||||
@ -47,14 +47,24 @@ def ancestry(space):
|
||||
|
||||
|
||||
def subtree_space_ids(space):
|
||||
"""Use the stable DB path rather than a filesystem traversal."""
|
||||
# A shared root contains only shared spaces: private spaces have their
|
||||
# own quota and must never consume/reserve shared root capacity.
|
||||
if space.parent_id is None and space.relative_path == '':
|
||||
return DriveSpace.objects.filter(private_owner__isnull=True).values_list('pk', flat=True)
|
||||
return DriveSpace.objects.filter(
|
||||
Q(relative_path=space.relative_path) | Q(relative_path__startswith=space.relative_path + '/')
|
||||
).values_list('pk', flat=True)
|
||||
"""Resolve the logical quota subtree, not the filesystem path.
|
||||
|
||||
Private accounts store files below storage/privates/ but are children of
|
||||
their assigned spaces *for quota accounting*. Physical path-prefix
|
||||
queries would silently omit their real disk usage.
|
||||
"""
|
||||
by_parent = {}
|
||||
for pk, parent_id in DriveSpace.objects.values_list('pk', 'parent_id'):
|
||||
by_parent.setdefault(parent_id, []).append(pk)
|
||||
result = set()
|
||||
to_visit = [space.pk]
|
||||
while to_visit:
|
||||
pk = to_visit.pop()
|
||||
if pk in result:
|
||||
raise ValueError('Hiérarchie des espaces invalide.')
|
||||
result.add(pk)
|
||||
to_visit.extend(by_parent.get(pk, []))
|
||||
return result
|
||||
|
||||
|
||||
def occupancy(space):
|
||||
@ -98,9 +108,9 @@ def trash_occupancy(space):
|
||||
def _budget_tree(space):
|
||||
"""One DB snapshot of all descendant spaces and their direct physical usage.
|
||||
|
||||
A child's fixed quota reserves that much capacity for it; a child without
|
||||
a quota reserves only its actual consumption (including its descendants).
|
||||
Thus quotas of siblings cannot silently overcommit their common parent.
|
||||
Shared descendants retain the existing reservation rules for assigned
|
||||
quotas. A private space's 50-Go cap is NEVER preallocated: its parent counts
|
||||
only its real files and pending uploads (including its own descendants).
|
||||
"""
|
||||
ids=list(subtree_space_ids(space))
|
||||
nodes={node.pk:node for node in DriveSpace.objects.filter(pk__in=ids)}
|
||||
@ -115,7 +125,11 @@ def _budget_tree(space):
|
||||
memo={}
|
||||
def committed(pk):
|
||||
if pk not in memo:
|
||||
memo[pk]=direct[pk]+sum(max(nodes[c].quota_bytes or 0,committed(c)) for c in children[pk])
|
||||
memo[pk] = direct[pk] + sum(
|
||||
committed(child) if nodes[child].private_owner_id
|
||||
else max(nodes[child].quota_bytes or 0, committed(child))
|
||||
for child in children[pk]
|
||||
)
|
||||
return memo[pk]
|
||||
committed(space.pk)
|
||||
return memo
|
||||
@ -129,8 +143,9 @@ def _committed(space):
|
||||
def available_for_growth(space):
|
||||
"""Bytes that can be added directly to space without breaking any ancestor allocation.
|
||||
|
||||
Capacity already reserved for this child does not count against its parent's
|
||||
available budget a second time. A zero-byte quota correctly permits no growth.
|
||||
Quotas of normal assigned children can absorb growth already reserved in
|
||||
their ancestors. For private children, the 50-Go cap is NOT reserved: every
|
||||
newly uploaded byte immediately reduces the parent's available budget.
|
||||
"""
|
||||
maximum=MAX_QUOTA_BYTES
|
||||
absorbed=0
|
||||
@ -140,7 +155,9 @@ def available_for_growth(space):
|
||||
committed=committed_tree[ancestor.pk]
|
||||
if ancestor.quota_bytes is not None:
|
||||
maximum=min(maximum,ancestor.quota_bytes-committed+absorbed)
|
||||
absorbed+=max(0,ancestor.quota_bytes-committed)
|
||||
# Private caps constrain uploads, but never absorb parent capacity.
|
||||
if not ancestor.private_owner_id:
|
||||
absorbed+=max(0,ancestor.quota_bytes-committed)
|
||||
return None if maximum==MAX_QUOTA_BYTES else max(0,int(maximum))
|
||||
|
||||
|
||||
@ -203,7 +220,7 @@ def locked_quota_tree(space):
|
||||
|
||||
|
||||
def assert_space_capacity(space, additional_bytes):
|
||||
"""Called with ancestor locks; include sibling allocations when checking uploads."""
|
||||
"""Called with ancestor locks; private uploads count toward both caps."""
|
||||
available=available_for_growth(space)
|
||||
if available is not None and additional_bytes>available:
|
||||
raise QuotaExceeded(
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
"""Physical numbered directories and strict permissions between personal/shared spaces."""
|
||||
"""Private files stored separately but counted against the assigned-space quota."""
|
||||
import hashlib
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
@ -34,10 +34,13 @@ def ensure_space(user):
|
||||
|
||||
|
||||
def _make_numbered_space(parent, **fields):
|
||||
# Private directories live in a separate physical and quota hierarchy.
|
||||
# Private directories are physically separate, but logically child spaces
|
||||
# of the account's assigned space so their *actual bytes* are included.
|
||||
private = fields.get('private_owner') is not None
|
||||
if private != (parent is None):
|
||||
raise ValueError('Hiérarchie de stockage privée invalide.')
|
||||
if parent is None or parent.private_owner_id:
|
||||
raise ValueError('Un espace doit avoir un parent partagé.')
|
||||
if private and parent.user_id != fields['private_owner'].pk:
|
||||
raise ValueError('L’espace privé doit être rattaché au quota attribué à son propriétaire.')
|
||||
base = 'privates' if private else parent.relative_path
|
||||
if len(base) + 1 + 24 > 700:
|
||||
raise ValueError('Hiérarchie trop profonde.')
|
||||
@ -95,8 +98,9 @@ def ensure_private_space(user):
|
||||
obj = DriveSpace.objects.select_for_update().filter(private_owner=user).first()
|
||||
if obj:
|
||||
return obj
|
||||
ensure_space(user) # Account must still have its assigned/shared space.
|
||||
return _make_numbered_space(None, private_owner=user, display_name='Mes fichiers')
|
||||
assigned = ensure_space(user)
|
||||
# The 50-Go cap is only a limit on private uploads, not a reservation.
|
||||
return _make_numbered_space(assigned, private_owner=user, display_name='Mes fichiers')
|
||||
|
||||
|
||||
def my_space(user):
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
"""Focused regressions for separate private storage and its 50 Go quota.
|
||||
"""Private disk layout, 50-Go private cap, and real-byte pooled accounting.
|
||||
|
||||
Run with: USE_SQLITE=true python manage.py test drive.test_private_layout
|
||||
"""
|
||||
@ -15,7 +15,8 @@ from django.test import TestCase, override_settings
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from .models import DriveItem, DriveSpace
|
||||
from .quotas import PRIVATE_SPACE_QUOTA_BYTES, QuotaExceeded, assert_space_capacity, locked_quota_tree, occupancy, quota_state
|
||||
from .quotas import (PRIVATE_SPACE_QUOTA_BYTES, QuotaExceeded, assert_space_capacity,
|
||||
assert_updated_quota, locked_quota_tree, occupancy, quota_state)
|
||||
from .spaces import ensure_private_space, ensure_space, create_space, storage_path
|
||||
|
||||
|
||||
@ -38,7 +39,7 @@ class PrivateStorageTests(TestCase):
|
||||
self.client.force_authenticate(user=self.admin)
|
||||
|
||||
def test_private_directories_are_separate_from_shared_spaces(self):
|
||||
self.assertIsNone(self.private.parent_id)
|
||||
self.assertEqual(self.private.parent_id, self.shared.pk)
|
||||
self.assertEqual(self.private.relative_path, f'privates/{self.private.directory_name}')
|
||||
self.assertEqual(self.private.quota_bytes, PRIVATE_SPACE_QUOTA_BYTES)
|
||||
self.assertTrue((self.rootdir / self.private.relative_path).is_dir())
|
||||
@ -47,12 +48,15 @@ class PrivateStorageTests(TestCase):
|
||||
private = ensure_private_space(person)
|
||||
self.assertTrue(shared.relative_path.startswith(f'{shared.directory_name}'))
|
||||
self.assertEqual(private.relative_path, f'privates/{private.directory_name}')
|
||||
self.assertIsNone(private.parent_id)
|
||||
self.assertEqual(private.parent_id, shared.pk)
|
||||
self.assertNotEqual(shared.directory_name, private.directory_name)
|
||||
|
||||
def test_private_quota_does_not_consume_shared_parent(self):
|
||||
self.shared.quota_bytes = 1
|
||||
def test_private_usage_counts_toward_assigned_space_without_reserving(self):
|
||||
self.shared.quota_bytes = 100
|
||||
self.shared.save(update_fields=['quota_bytes'])
|
||||
# Private can be 50 Go even when the assigned budget is just 100 bytes.
|
||||
self.assertEqual(quota_state(self.shared)['available_bytes'], 100)
|
||||
self.assertEqual(quota_state(self.private)['available_bytes'], 100)
|
||||
created = self.client.post('/api/drive/uploads/', {
|
||||
'files': [SimpleUploadedFile('photo.jpg', b'abcdef')]
|
||||
}, format='multipart')
|
||||
@ -60,7 +64,9 @@ class PrivateStorageTests(TestCase):
|
||||
node = DriveItem.objects.get(pk=created.data['items'][0]['id'])
|
||||
self.assertEqual(node.storage_location, f'{self.private.relative_path}/photo.jpg')
|
||||
self.assertEqual(storage_path(node).read_bytes(), b'abcdef')
|
||||
self.assertEqual(occupancy(self.shared)[0], 0)
|
||||
self.assertEqual(occupancy(self.shared)[0], 6)
|
||||
self.assertEqual(quota_state(self.shared)['available_bytes'], 94)
|
||||
self.assertEqual(quota_state(self.private)['available_bytes'], 94)
|
||||
self.assertEqual(quota_state(self.private)['quota_bytes'], PRIVATE_SPACE_QUOTA_BYTES)
|
||||
self.assertEqual(quota_state(self.private)['used_bytes'], 6)
|
||||
rejected = self.client.patch(f'/api/drive/spaces/{self.private.pk}/quota/', {
|
||||
@ -68,6 +74,78 @@ class PrivateStorageTests(TestCase):
|
||||
}, format='json')
|
||||
self.assertEqual(rejected.status_code, 403)
|
||||
|
||||
def test_actual_private_use_is_pooled_in_100_go_assigned_space(self):
|
||||
person = get_user_model().objects.create_user('quota-user', password='TestPassword123!')
|
||||
assigned = create_space(person, self.shared, '100 Go', quota_bytes=100 * 1024 ** 3)
|
||||
private = ensure_private_space(person)
|
||||
gib = 1024 ** 3
|
||||
self.assertEqual(private.parent_id, assigned.pk)
|
||||
self.assertEqual(private.relative_path, f'privates/{private.directory_name}')
|
||||
self.assertEqual(quota_state(assigned)['available_bytes'], 100 * gib)
|
||||
self.assertEqual(quota_state(private)['available_bytes'], 50 * gib)
|
||||
# Existing 20 Go in shared space. This leaves both limits at their max.
|
||||
DriveItem.objects.create(
|
||||
owner=person, space=assigned, name='shared.bin',
|
||||
storage_key=str(uuid.uuid4()), storage_location=f'{assigned.relative_path}/shared.bin',
|
||||
size=20 * gib,
|
||||
)
|
||||
self.assertEqual(quota_state(assigned)['used_bytes'], 20 * gib)
|
||||
self.assertEqual(quota_state(private)['used_bytes'], 0)
|
||||
DriveItem.objects.create(
|
||||
owner=person, space=private, name='private.bin',
|
||||
storage_key=str(uuid.uuid4()), storage_location=f'{private.relative_path}/private.bin',
|
||||
size=20 * gib,
|
||||
)
|
||||
self.assertEqual(occupancy(assigned)[0], 40 * gib)
|
||||
self.assertEqual(quota_state(assigned)['used_bytes'], 40 * gib)
|
||||
self.assertEqual(quota_state(assigned)['available_bytes'], 60 * gib)
|
||||
self.assertEqual(quota_state(private)['used_bytes'], 20 * gib)
|
||||
self.assertEqual(quota_state(private)['available_bytes'], 30 * gib)
|
||||
with transaction.atomic():
|
||||
with locked_quota_tree(assigned):
|
||||
assert_space_capacity(assigned, 60 * gib)
|
||||
with self.assertRaises(QuotaExceeded):
|
||||
assert_space_capacity(assigned, 60 * gib + 1)
|
||||
assert_updated_quota(assigned, 40 * gib)
|
||||
with self.assertRaisesRegex(ValueError, 'fichiers'):
|
||||
assert_updated_quota(assigned, 39 * gib)
|
||||
|
||||
def test_parent_limit_blocks_private_even_before_50_go(self):
|
||||
self.shared.quota_bytes = 20
|
||||
self.shared.save(update_fields=['quota_bytes'])
|
||||
DriveItem.objects.create(
|
||||
owner=self.admin, space=self.private, name='existing.bin',
|
||||
storage_key=str(uuid.uuid4()), storage_location=f'{self.private.relative_path}/existing.bin', size=19,
|
||||
)
|
||||
self.assertEqual(quota_state(self.private)['available_bytes'], 1)
|
||||
with transaction.atomic():
|
||||
with locked_quota_tree(self.private):
|
||||
assert_space_capacity(self.private, 1)
|
||||
with self.assertRaises(QuotaExceeded):
|
||||
assert_space_capacity(self.private, 2)
|
||||
|
||||
def test_invitations_allow_less_than_50_go_no_reservation(self):
|
||||
self.shared.quota_bytes = 15
|
||||
self.shared.save(update_fields=['quota_bytes'])
|
||||
res = self.client.post('/api/drive/invitations/', {'quota_bytes': 10}, format='json')
|
||||
self.assertEqual(res.status_code, 201, res.data)
|
||||
self.assertEqual(quota_state(self.shared)['available_bytes'], 15)
|
||||
member = get_user_model().objects.create_user('tiny-private', password='TestPassword123!')
|
||||
assigned = create_space(member, self.shared, '10 octets', quota_bytes=10)
|
||||
private = ensure_private_space(member)
|
||||
self.assertEqual(private.quota_bytes, PRIVATE_SPACE_QUOTA_BYTES)
|
||||
self.assertEqual(quota_state(assigned)['available_bytes'], 10)
|
||||
self.assertEqual(quota_state(private)['available_bytes'], 10)
|
||||
|
||||
def test_private_pending_uploads_count_in_parent(self):
|
||||
from .models import UploadSession
|
||||
UploadSession.objects.create(
|
||||
owner=self.admin, space=self.private, name='movie.bin', total_size=1234
|
||||
)
|
||||
self.assertEqual(quota_state(self.shared)['reserved_bytes'], 1234)
|
||||
self.assertEqual(quota_state(self.shared)['used_bytes'], 0)
|
||||
self.assertEqual(quota_state(self.private)['available_bytes'], PRIVATE_SPACE_QUOTA_BYTES - 1234)
|
||||
|
||||
def test_private_50_go_upload_limit_is_enforced(self):
|
||||
DriveItem.objects.create(
|
||||
owner=self.admin, space=self.private, name='large.bin',
|
||||
@ -114,7 +192,7 @@ class PrivateStorageTests(TestCase):
|
||||
self.private.refresh_from_db()
|
||||
item.refresh_from_db()
|
||||
self.assertEqual(self.private.relative_path, f'privates/{self.private.directory_name}')
|
||||
self.assertIsNone(self.private.parent_id)
|
||||
self.assertEqual(self.private.parent_id, self.shared.pk)
|
||||
self.assertEqual(self.private.quota_bytes, PRIVATE_SPACE_QUOTA_BYTES)
|
||||
self.assertEqual(item.storage_location, f'{self.private.relative_path}/Documents/facture.pdf')
|
||||
self.assertEqual(storage_path(item).read_bytes(), b'PDF CONTENT')
|
||||
|
||||
@ -18,7 +18,8 @@ from rest_framework.response import Response
|
||||
from .models import DriveSpace, SpaceInvitation, SpaceGrant, DriveItem, UploadSession
|
||||
from .services import error, valid_name
|
||||
from .spaces import ensure_space, ensure_private_space, my_space, create_space, _make_numbered_space, is_admin, hash_token, space_info, can_read_space, can_manage_space
|
||||
from .quotas import parse_quota, locked_quota_tree, assert_new_child_quota, assert_updated_quota
|
||||
from .quotas import (parse_quota, locked_quota_tree,
|
||||
assert_new_child_quota, assert_updated_quota)
|
||||
|
||||
|
||||
def permission_flags(data):
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user