Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use common logic for naming and deleting test buckets #168

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Use common logic for naming and deleting test buckets #168

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cf86c0a
Use ApiTestManager from b2-sdk in integration tests for common logic
kgwizdz-reef May 15, 2023
d18b264
Merge branch 'master' into use-common-logic-for-naming-and-deleting-t…
kgwizdz-reef May 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Infrastructure
* Autocomplete integration tests will now work properly even if tested package has not been installed
* Automatically set copyright date when generating the docs
* Use ApiTestManager from b2-sdk in integration tests for common logic

## [3.9.0] - 2023-04-28

Expand Down
2 changes: 1 addition & 1 deletion test/integration/cleanup_buckets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@ def test_cleanup_buckets(b2_api):
# this is not a test, but it is intended to be called
# via pytest because it reuses fixtures which have everything
# set up
b2_api.clean_buckets()
b2_api.clean_all_buckets()
50 changes: 14 additions & 36 deletions test/integration/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,20 @@
#
######################################################################

import base64
import contextlib
import sys
import uuid
from contextlib import suppress

from os import environ, path
from tempfile import TemporaryDirectory

import pytest

from b2sdk.v2 import B2_ACCOUNT_INFO_ENV_VAR, XDG_CONFIG_HOME_ENV_VAR
from b2sdk.exception import BadRequest, BucketIdNotFound
from b2sdk.v2 import B2_ACCOUNT_INFO_ENV_VAR, XDG_CONFIG_HOME_ENV_VAR, Bucket
from b2sdk.exception import BucketIdNotFound
from b2sdk.test.api_test_manager import ApiTestManager

from .helpers import Api, CommandLine, bucket_name_part

GENERAL_BUCKET_NAME_PREFIX = 'clitst'
from test.integration.helpers import CommandLine


@pytest.hookimpl
Expand Down Expand Up @@ -53,17 +52,10 @@ def realm() -> str:


@pytest.fixture(scope='function')
def bucket(b2_api) -> str:
try:
bucket = b2_api.create_bucket()
except BadRequest as e:
if e.code != 'too_many_buckets':
raise
num_buckets = b2_api.count_and_print_buckets()
print('current number of buckets:', num_buckets)
raise
def bucket(b2_api) -> Bucket:
bucket = b2_api.create_test_bucket()
yield bucket
with contextlib.suppress(BucketIdNotFound):
with suppress(BucketIdNotFound):
b2_api.clean_bucket(bucket)


Expand All @@ -74,7 +66,7 @@ def bucket_name(bucket) -> str:

@pytest.fixture(scope='function')
def file_name(bucket) -> str:
file_ = bucket.upload_bytes(b'test_file', f'{bucket_name_part(8)}.txt')
file_ = bucket.upload_bytes(b'test_file', f'{uuid.uuid4()}.txt')
yield file_.file_name


Expand All @@ -94,11 +86,6 @@ def debug_print_buckets(b2_api):
print('-' * 30)


@pytest.fixture(scope='session')
def this_run_bucket_name_prefix() -> str:
yield GENERAL_BUCKET_NAME_PREFIX + bucket_name_part(8)


@pytest.fixture(scope='module')
def monkey_patch():
""" Module-scope monkeypatching (original `monkeypatch` is function-scope) """
Expand Down Expand Up @@ -127,23 +114,14 @@ def auto_change_account_info_dir(monkey_patch) -> dir:


@pytest.fixture(scope='module')
def b2_api(application_key_id, application_key, realm, this_run_bucket_name_prefix) -> Api:
yield Api(
application_key_id, application_key, realm, GENERAL_BUCKET_NAME_PREFIX,
this_run_bucket_name_prefix
)
def b2_api(application_key_id, application_key, realm) -> ApiTestManager:
yield ApiTestManager(application_key_id, application_key, realm)


@pytest.fixture(scope='module')
def global_b2_tool(
request, application_key_id, application_key, realm, this_run_bucket_name_prefix
) -> CommandLine:
def global_b2_tool(request, application_key_id, application_key, realm) -> CommandLine:
tool = CommandLine(
request.config.getoption('--sut'),
application_key_id,
application_key,
realm,
this_run_bucket_name_prefix,
request.config.getoption('--sut'), application_key_id, application_key, realm
)
tool.reauthorize(check_key_capabilities=True) # reauthorize for the first time (with check)
return tool
Expand Down
189 changes: 5 additions & 184 deletions test/integration/helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,36 +15,28 @@
import random
import re
import shutil
import string
import subprocess
import sys
import threading
import time

from dataclasses import dataclass
from datetime import datetime
from os import environ, linesep, path
from pathlib import Path
from tempfile import gettempdir, mkdtemp
from typing import List, Optional, Union

import backoff
import pytest

from b2sdk._v3.exception import BucketIdNotFound as v3BucketIdNotFound
from b2sdk.v2 import ALL_CAPABILITIES, NO_RETENTION_FILE_SETTING, B2Api, Bucket, EncryptionAlgorithm, EncryptionKey, EncryptionMode, EncryptionSetting, InMemoryAccountInfo, InMemoryCache, LegalHold, RetentionMode, SqliteAccountInfo, fix_windows_path_limit
from b2sdk.v2.exception import BucketIdNotFound, DuplicateBucketName, FileNotPresent, TooManyRequests
from b2sdk.v2 import ALL_CAPABILITIES, EncryptionAlgorithm, EncryptionKey, EncryptionMode, EncryptionSetting, \
SqliteAccountInfo, fix_windows_path_limit

from b2.console_tool import Command, current_time_millis
from b2.console_tool import Command

logger = logging.getLogger(__name__)

BUCKET_CLEANUP_PERIOD_MILLIS = 0
ONE_HOUR_MILLIS = 60 * 60 * 1000
ONE_DAY_MILLIS = ONE_HOUR_MILLIS * 24

BUCKET_NAME_LENGTH = 50
BUCKET_NAME_CHARS = string.ascii_letters + string.digits + '-'
BUCKET_CREATED_AT_MILLIS = 'created_at_millis'

SSE_NONE = EncryptionSetting(mode=EncryptionMode.NONE,)
Expand All @@ -63,171 +55,6 @@
key=EncryptionKey(secret=os.urandom(32), key_id='another-user-generated-key-id')
)

RNG_SEED = '_'.join(
[
os.getenv('GITHUB_REPOSITORY', ''),
os.getenv('GITHUB_SHA', ''),
os.getenv('GITHUB_RUN_ID', ''),
os.getenv('GITHUB_RUN_ATTEMPT', ''),
os.getenv('GITHUB_JOB', ''),
os.getenv('GITHUB_ACTION', ''),
str(os.getpid()), # for local runs with xdist
str(time.time()),
]
)

RNG = random.Random(RNG_SEED)

RNG_COUNTER = 0


def bucket_name_part(length: int) -> str:
assert length >= 1
global RNG_COUNTER
RNG_COUNTER += 1
name_part = ''.join(RNG.choice(BUCKET_NAME_CHARS) for _ in range(length))
logger.info('RNG_SEED: %s', RNG_SEED)
logger.info('RNG_COUNTER: %i, length: %i', RNG_COUNTER, length)
logger.info('name_part: %s', name_part)
return name_part


@dataclass
class Api:
account_id: str
application_key: str
realm: str
general_bucket_name_prefix: str
this_run_bucket_name_prefix: str

api: B2Api = None

def __post_init__(self):
info = InMemoryAccountInfo()
cache = InMemoryCache()
self.api = B2Api(info, cache=cache)
self.api.authorize_account(self.realm, self.account_id, self.application_key)
assert BUCKET_NAME_LENGTH - len(
self.this_run_bucket_name_prefix
) > 5, self.this_run_bucket_name_prefix

def create_bucket(self) -> Bucket:
for _ in range(10):
bucket_name = self.this_run_bucket_name_prefix + bucket_name_part(
BUCKET_NAME_LENGTH - len(self.this_run_bucket_name_prefix)
)
print('Creating bucket:', bucket_name)
try:
return self.api.create_bucket(
bucket_name,
'allPublic',
bucket_info={BUCKET_CREATED_AT_MILLIS: str(current_time_millis())},
)
except DuplicateBucketName:
pass
print()

raise ValueError('Failed to create bucket due to name collision')

def _should_remove_bucket(self, bucket: Bucket):
if bucket.name.startswith(self.this_run_bucket_name_prefix):
return True, 'it is a bucket for this very run'
OLD_PATTERN = 'test-b2-cli-'
if bucket.name.startswith(self.general_bucket_name_prefix) or bucket.name.startswith(OLD_PATTERN): # yapf: disable
if BUCKET_CREATED_AT_MILLIS in bucket.bucket_info:
delete_older_than = current_time_millis() - BUCKET_CLEANUP_PERIOD_MILLIS
this_bucket_creation_time = bucket.bucket_info[BUCKET_CREATED_AT_MILLIS]
if int(this_bucket_creation_time) < delete_older_than:
return True, f"this_bucket_creation_time={this_bucket_creation_time} < delete_older_than={delete_older_than}"
else:
return True, 'undefined ' + BUCKET_CREATED_AT_MILLIS
return False, ''

def clean_buckets(self):
buckets = self.api.list_buckets()
print('Total bucket count:', len(buckets))
for bucket in buckets:
should_remove, why = self._should_remove_bucket(bucket)
if not should_remove:
print(f'Skipping bucket removal: "{bucket.name}"')
continue

print('Trying to remove bucket:', bucket.name, 'because', why)
try:
self.clean_bucket(bucket)
except (BucketIdNotFound, v3BucketIdNotFound):
print('It seems that bucket %s has already been removed' % (bucket.name,))
buckets = self.api.list_buckets()
print('Total bucket count after cleanup:', len(buckets))
for bucket in buckets:
print(bucket)

@backoff.on_exception(
backoff.expo,
TooManyRequests,
max_tries=8,
)
def clean_bucket(self, bucket: Union[Bucket, str]):
if isinstance(bucket, str):
bucket = self.api.get_bucket_by_name(bucket)

files_leftover = False
file_versions = bucket.ls(latest_only=False, recursive=True)

for file_version_info, _ in file_versions:
if file_version_info.file_retention:
if file_version_info.file_retention.mode == RetentionMode.GOVERNANCE:
print('Removing retention from file version:', file_version_info.id_)
self.api.update_file_retention(
file_version_info.id_, file_version_info.file_name,
NO_RETENTION_FILE_SETTING, True
)
elif file_version_info.file_retention.mode == RetentionMode.COMPLIANCE:
if file_version_info.file_retention.retain_until > current_time_millis(): # yapf: disable
print(
'File version: %s cannot be removed due to compliance mode retention' %
(file_version_info.id_,)
)
files_leftover = True
continue
elif file_version_info.file_retention.mode == RetentionMode.NONE:
pass
else:
raise ValueError(
'Unknown retention mode: %s' % (file_version_info.file_retention.mode,)
)
if file_version_info.legal_hold.is_on():
print('Removing legal hold from file version:', file_version_info.id_)
self.api.update_file_legal_hold(
file_version_info.id_, file_version_info.file_name, LegalHold.OFF
)
print('Removing file version:', file_version_info.id_)
try:
self.api.delete_file_version(file_version_info.id_, file_version_info.file_name)
except FileNotPresent:
print(
'It seems that file version %s has already been removed' %
(file_version_info.id_,)
)

if files_leftover:
print('Unable to remove bucket because some retained files remain')
else:
print('Removing bucket:', bucket.name)
try:
self.api.delete_bucket(bucket)
except BucketIdNotFound:
print('It seems that bucket %s has already been removed' % (bucket.name,))
print()

def count_and_print_buckets(self) -> int:
buckets = self.api.list_buckets()
count = len(buckets)
print(f'Total bucket count at {datetime.now()}: {count}')
for i, bucket in enumerate(buckets, start=1):
print(f'- {i}\t{bucket.name} [{bucket.id_}]')
return count


def print_text_indented(text):
"""
Expand Down Expand Up @@ -383,20 +210,14 @@ class CommandLine:
re.compile(r'Trying to print: .*'),
]

def __init__(self, command, account_id, application_key, realm, bucket_name_prefix):
def __init__(self, command, account_id, application_key, realm):
self.command = command
self.account_id = account_id
self.application_key = application_key
self.realm = realm
self.bucket_name_prefix = bucket_name_prefix
self.env_var_test_context = EnvVarTestContext(SqliteAccountInfo().filename)
self.account_info_file_name = SqliteAccountInfo().filename

def generate_bucket_name(self):
return self.bucket_name_prefix + bucket_name_part(
BUCKET_NAME_LENGTH - len(self.bucket_name_prefix)
)

def run_command(self, args, additional_env: Optional[dict] = None):
"""
Runs the command with the given arguments, returns a tuple in form of
Expand Down Expand Up @@ -426,7 +247,7 @@ def should_succeed(

if expected_pattern is not None:
assert re.search(expected_pattern, stdout), \
f'did not match pattern="{expected_pattern}", stdout="{stdout}"'
f'did not match pattern="{expected_pattern}", stdout="{stdout}"'

return stdout

Expand Down
Loading