diff --git a/cterasdk/asynchronous/core/buckets.py b/cterasdk/asynchronous/core/buckets.py new file mode 100644 index 00000000..9961338c --- /dev/null +++ b/cterasdk/asynchronous/core/buckets.py @@ -0,0 +1,26 @@ +import logging + +from ...common import union +from .base_command import BaseCommand +from ...core.query import QueryParamBuilder +from . import query + + +logger = logging.getLogger('cterasdk.core') + + +class Buckets(BaseCommand): + """ + Portal Storage Node APIs + """ + default = ['name'] + + def list_buckets(self, include=None): + """ + List Buckets. + + :param list[str],optional include: List of fields to retrieve, defaults to ``['name']`` + """ + include = union(include or [], Buckets.default) + param = QueryParamBuilder().include(include).build() + return query.iterator(self._core, '/locations', param) diff --git a/cterasdk/asynchronous/core/cloudfs.py b/cterasdk/asynchronous/core/cloudfs.py index 802878a8..72467e8c 100644 --- a/cterasdk/asynchronous/core/cloudfs.py +++ b/cterasdk/asynchronous/core/cloudfs.py @@ -14,8 +14,31 @@ class CloudFS(BaseCommand): def __init__(self, core): super().__init__(core) + self.groups = FolderGroups(self._core) self.drives = CloudDrives(self._core) self.zones = Zones(self._core) + self.exports = Exports(self._core) + + +class FolderGroups(BaseCommand): + """ Cloud Drive Folder APIs """ + + default = ['name'] + + async def all(self, include=None, namespaces=None): + """ + List folder groups + + :param str,optional include: List of fields to retrieve, defaults to ['name'] + :param list[str],optional namespaces: List of namespaces to query + :returns: Iterator for all folder groups + """ + include = union(include or [], FolderGroups.default) + + for resource in [f'/portals/{namespace}' for namespace in namespaces] if namespaces is not None else ['']: + param = QueryParamBuilder().include(include).build() + async for group in query.iterator(self._core, f'{resource}/foldersGroups', param): + yield group class CloudDrives(BaseCommand): @@ -23,6 +46,21 @@ class CloudDrives(BaseCommand): default = ['name'] + async def all(self, include=None, namespaces=None): + """ + List Cloud Drive folders. + + :param str,optional include: List of fields to retrieve, defaults to ['name'] + :param list[str],optional namespaces: List of namespaces to query + :returns: Iterator for all Cloud Drive folders + """ + include = union(include or [], CloudDrives.default) + + for resource in [f'/portals/{namespace}' for namespace in namespaces] if namespaces is not None else ['']: + param = QueryParamBuilder().include(include).put('includeDeleted', True).build() + async for drive in query.iterator(self._core, f'{resource}/cloudDrives', param): + yield drive + async def find(self, name, owner, include=None): """ Find a Cloud Drive Folder @@ -46,46 +84,48 @@ class Zones(BaseCommand): Portal Zones APIs """ - async def all(self, filters=None): + async def all(self, expand_zone=False, namespaces=None): """ List Zones - :param list[],optional filters: List of additional filters, defaults to None + + :param list[str],optional namespaces: List of namespaces to query :return: Iterator for all Zones - :rtype: cterasdk.lib.iterator.QueryIterator + :rtype: cterasdk.asynchronous.core.iterator.QueryAsyncIterator """ - builder = QueryParamBuilder().include_classname().startFrom(0).countLimit(25) - filters = filters or [] - for query_filter in filters: - builder.addFilter(query_filter) - builder.orFilter((len(filters) > 1)) - param = builder.build() - async for zone in query.iterator(self._core, '', param, 'getZonesDisplayInfo'): - yield zone + for resource in [f'/portals/{namespace}' for namespace in namespaces] if namespaces is not None else ['']: + param = QueryParamBuilder().include_classname().build() + async for zone in query.iterator(self._core, resource, param, 'getZonesDisplayInfo'): + if expand_zone: + info = await self._core.v1.api.execute(resource, 'getZoneBasicInfo', zone.zoneId) + zone.policyType = info.policyType + zone.devices = [device async for device in query.iterator(self._core, resource, + ZoneQueryParams(info.zoneId, DevicesDelta()), + 'getZoneDevices')] + if zone.policyType == 'selectedFolders': + zone.cloudfolders = [ + volume + async for volume in query.iterator( + self._core, + resource, + ZoneQueryParams(info.zoneId, FoldersDelta()), + 'getZoneFolders', + ) + ] + yield zone + + +class Exports(BaseCommand): + """ Fusion Gateway S3 APIs """ - async def list_zones(self, filters=None, expand_zone=False): + async def all(self, namespaces=None): """ - List Zones - :param list[],optional filters: List of additional filters, defaults to None - :param bool,optional expand_zone: Include Cloud Drive folders and devices + List Fusion Gateway S3 Exports - :return: Iterator for all Zones - :rtype: cterasdk.lib.iterator.QueryIterator + :param str,optional include: List of fields to retrieve, defaults to ['name'] + :param list[str],optional namespaces: List of namespaces to query + :returns: Iterator for all Fusion Gateway exports """ - async for zone in self.all(filters): - if expand_zone: - info = await self._core.v1.api.execute('', 'getZoneBasicInfo', zone.zoneId) - zone.devices = [device async for device in query.iterator(self._core, '', - ZoneQueryParams(info.zoneId, DevicesDelta()), 'getZoneDevices')] - if info.policyType == 'selectedFolders': - zone.cloudfolders = [ - volume - async for volume in query.iterator( - self._core, - '', - ZoneQueryParams(info.zoneId, FoldersDelta()), - 'getZoneFolders', - ) - ] - yield zone - yield zone + for resource in [f'/portals/{namespace}' for namespace in namespaces] if namespaces is not None else ['']: + for export in await self._core.v1.api.get(f'{resource}/buckets'): + yield export diff --git a/cterasdk/asynchronous/core/plans.py b/cterasdk/asynchronous/core/plans.py new file mode 100644 index 00000000..b7dfba93 --- /dev/null +++ b/cterasdk/asynchronous/core/plans.py @@ -0,0 +1,36 @@ +import logging +from .base_command import BaseCommand +from ...common import union +from ...core.query import QueryParamBuilder +from . import query + + +logger = logging.getLogger('cterasdk.core') + + +class Plans(BaseCommand): + """ + Portal Plan APIs + + :ivar cterasdk.core.plans.PlanAutoAssignPolicy auto_assign: Object holding the Portal subscription plan auto assignment rules APIs + """ + default = ['name'] + + def list_plans(self, include=None, filters=None): + """ + List Plans + + :param list[str],optional include: List of fields to retrieve, defaults to ['name'] + :param list[],optional filters: List of additional filters, defaults to None + + :return: Iterator for all matching Plans + :rtype: cterasdk.asynchronous.core.iterator.QueryAsyncIterator + """ + include = union(include or [], Plans.default) + builder = QueryParamBuilder().include(include) + filters = filters or [] + for query_filter in filters: + builder.addFilter(query_filter) + builder.orFilter((len(filters) > 1)) + param = builder.build() + return query.iterator(self._core, '/plans', param) diff --git a/cterasdk/asynchronous/core/portals.py b/cterasdk/asynchronous/core/portals.py index 13984c16..2a3b5fb6 100644 --- a/cterasdk/asynchronous/core/portals.py +++ b/cterasdk/asynchronous/core/portals.py @@ -1,5 +1,7 @@ from .base_command import BaseCommand from ...core import decorator +from ...core.query import QueryParamBuilder +from . import query class Portals(BaseCommand): @@ -18,3 +20,12 @@ async def browse_global_admin(self): Browse the Global Admin """ await self.browse('') + + def tenants(self, include_deleted=False): + """ + Get all tenants + + :param bool,optional include_deleted: Include deleted tenants, defaults to False + """ + param = QueryParamBuilder().include_classname().put('isTrashcan', include_deleted).build() + return query.iterator(self._core, '', param, 'getPortalsDisplayInfo') diff --git a/cterasdk/asynchronous/core/servers.py b/cterasdk/asynchronous/core/servers.py new file mode 100644 index 00000000..94a5e047 --- /dev/null +++ b/cterasdk/asynchronous/core/servers.py @@ -0,0 +1,43 @@ +import logging +import asyncio + +from ...common import union +from .base_command import BaseCommand +from ...core.query import QueryParamBuilder +from . import query + + +logger = logging.getLogger('cterasdk.core') + + +class Servers(BaseCommand): + """ + Global Admin Servers APIs + """ + + default = ['name'] + + async def system_info(self, server): + return await self._core.v1.api.get(f'/servers/{server.name}/systemInfo') + + async def list_servers(self, include=None): + """ + Retrieve the servers that comprise CTERA Portal. + Restricted to the Global Administration Portal. Browse it using :py:func:`cterasdk.core.portals.browse_global_admin`. + + :param list[str],optional include: List of fields to retrieve, defaults to ['name'] + """ + include = union(include or [], Servers.default) + param = QueryParamBuilder().include(include).build() + + servers = [server async for server in query.iterator(self._core, '/servers', param)] + + if 'systemInfo' in include: + tasks = [self.system_info(server) for server in servers] + systems = await asyncio.gather(*tasks) + + for server, sys_info in zip(servers, systems): + server.systemInfo = sys_info + + for server in servers: + yield server diff --git a/cterasdk/cli/export.py b/cterasdk/cli/export.py new file mode 100644 index 00000000..07fafa89 --- /dev/null +++ b/cterasdk/cli/export.py @@ -0,0 +1,877 @@ +# pylint: disable=too-many-lines +import argparse +import asyncio +import logging +import sys +from datetime import datetime + +from .. import settings +from ..objects.asynchronous.core import AsyncGlobalAdmin +from ..common import Object, parse_base_object_ref +from ..convert.deserializers import fromjsonstr +from ..exceptions.auth import AuthenticationError +from ..exceptions.transport import HTTPError, InternalServerError, BadGateway, GatewayTimeout + + +logger = logging.getLogger('cterasdk.export') + + +ATTRIBUTE_PATHS = { + + '.': [ + 'insight.globalStatus.status', + 'insight.settings.enabled', + 'mcp.globalStatus.status', + 'mediaConnector.globalStatus.status', + 'mediaConnector.settings.enabled', + 'messaging.globalStatus.status', + 'syslog.status.globalStatus.status', + 'varonis.globalStatus.status', + 'samlCertificate' + ], + + '.settings.': [ + 'ca.expirationDate', + 'cloudFSSettings.loadBlocksMaxThreads', + 'cloudFSSettings.mapFileInDB', + 'cloudFSSettings.maxThreadsForMigration', + 'cloudFSSettings.storeBlocksMaxThreads', + 'cteraZonesEnabled', + 'cteraZonesTaskMode', + 'cteraZonesTaskRecalcIntervalInSeconds', + 'dnsSuffix', + 'officeOnlineSettings.enabled' + ], + + '.portalLicenses[].': [ + 'antivirus', + 'appliances', + 'archiveStorage', + 'cloudDrives', + 'cloudDrivesLite', + 'comment', + 'dlp', + 'expirationDate', + 'expired', + 'globalFileLock', + 'key', + 'keyManager', + 'originalKey', + 'portalLicense', + 'serverAgents', + 'storage', + 'vGateways128', + 'vGateways256', + 'vGateways32', + 'vGateways4', + 'vGateways64', + 'vGateways8', + 'valid', + 'varonis', + 'workstationAgents', + ], + + '.firmwares[].': [ + 'binaryDataMD5', + 'firmwareType', + 'guid', + 'isDefault', + 'uid', + 'version', + ], + + '.servers[].': [ + 'backupToBucket.details.endPoint', + 'backupToBucket.details.storage', + 'backupToBucket.details.trustAllCertificates', + 'backupToBucket.details.useHttps', + 'backupToBucket.details.usePathStyleAddressing', + 'backupToBucket.enabled', + 'backupToBucket.exportSchedulePeriod', + 'backupToBucket.status', + 'connected', + 'createDate', + 'isAVBGServer', + 'isApplicationServer', + 'isMessagingServer', + 'isS3Endpoint', + 'isThumbnailsServer', + 'mainDB', + 'modifiedDate', + 'name', + 'previewStatus', + 'renderingServer', + 'replicationSettings.replicationOf', + 'runningVersion', + 'systemInfo', + ], + + '.locations[].': [ + 'bucket', + 'connected', + 'createDate', + 'dedicated', + 'dedicatedPortal', + 'directUpload', + 'doFsync', + 'endPoint', + 'folderSize', + 'httpsOnly', + 'modifiedDate', + 'name', + 'objectLock', + 'readOnly', + 's3Endpoint', + 'status', + 'storage', + 'storageClass', + 'trustAllCertificates', + 'useHttps', + 'usePathStyleAddressing', + ], + + '.portals[].': [ + 'activationStatus', + 'externalPortalId', + 'isDefault', + 'name', + 'numberOfAddons', + 'numberOfConnectedDevices', + 'numberOfUsers', + 'plan.archiveStorage.amount', + 'plan.cloudDrives.amount', + 'plan.cloudDrivesLite.amount', + 'plan.retentionPolicy', + 'plan.serverAgents.amount', + 'plan.services[].serviceName', + 'plan.services[].serviceState', + 'plan.storage.amount', + 'plan.vGateways.amount', + 'plan.vGateways128.amount', + 'plan.vGateways256.amount', + 'plan.vGateways32.amount', + 'plan.vGateways4.amount', + 'plan.vGateways64.amount', + 'plan.vGateways8.amount', + 'plan.workstationAgents.amount', + 'portalTrashcanInfo.isTrashcan', + 'portalType', + 'resourcesQuotas[].resourceType', + 'resourcesQuotas[].totalQuota', + 'resourcesQuotas[].usedQuota', + 'totalArchiveStorageQuota', + 'totalStorageQuota', + 'uid', + 'usedArchiveStorageQuota', + 'usedStorageQuota', + ], + + '.portals[].foldersGroups[].': [ + 'averageBlockSizeKb', + 'averageMapFileSizeKb', + 'compressionMethod', + 'createDate', + 'deduplicationMethodType', + 'encryptionMode', + 'fixedBlockSizeKb', + 'mapFileInDB', + 'modifiedDate', + 'name', + 'storageClass', + 'uid', + ], + + '.portals[].cloudDrives[].': [ + 'archiveSettings.archive', + 'archiveSettings.deleteData', + 'archiveSettings.gracePeriod.amount', + 'archiveSettings.gracePeriod.type', + 'archiveSettings.retentionMode', + 'archiveSettings.retentionPeriod.amount', + 'archiveSettings.retentionPeriod.type', + 'archiveSettings.seal', + 'createDate', + 'enableSyncWinNtExtendedAttributes', + 'extendedAttributes.enable', + 'folderQuota', + 'folderStats.cloudFolderSize', + 'folderStats.cloudFolderSize.totalFiles', + 'globalFileLockSettings.enabled', + 'globalFileLockSettings.globalFileLockExtensions', + 'group', + 'isDeleted', + 'modifiedDate', + 'owner', + 'teamProject', + 'uid', + 'wormSettings.gracePeriod.amount', + 'wormSettings.gracePeriod.type', + 'wormSettings.retentionMode', + 'wormSettings.retentionPeriod.amount', + 'wormSettings.retentionPeriod.type', + 'wormSettings.worm', + ], + + '.portals[].zones[].': [ + 'cloudfolders[].isIncluded', + 'cloudfolders[].owner.uid', + 'cloudfolders[].totalFiles', + 'cloudfolders[].totalSize', + 'cloudfolders[].uid', + 'devices[].deviceType', + 'devices[].uid', + 'devicesCount', + 'isDefault', + 'name', + 'policyType', + 'zoneId', + 'zoneStatistics.totalFiles', + 'zoneStatistics.totalFolders', + 'zoneStatistics.totalSize', + ], + + '.portals[].buckets[].': [ + 'cloudDrive', + 'createDate', + 'modifiedDate', + 'uid', + ], + + '.portals[].devices[].': [ + 'backup.backupStatus.backupHistory.lastSuccessfulSync', + 'backup.backupStatus.backupHistory.totalFiles', + 'backup.backupStatus.backupHistory.totalSize', + 'backup.backupStatus.deviceTime.LocalTime', + 'backup.backupStatus.deviceTime.TimeGMT', + 'backup.backupStatus.deviceTime.uptime', + 'backup.backupStatus.serviceStatus.desc', + 'config.device.location', + 'config.services.remoteAccess.adminRemoteAccess', + 'createDate', + 'deviceConnectionStatus.connected', + 'deviceConnectionStatus.updateTime', + 'deviceReportedStatus.status.device.deviceReportedStatus', + 'deviceReportedStatus.status.device.installedFirmware.md5', + 'deviceReportedStatus.status.device.installedFirmware.version', + 'deviceReportedStatus.status.device.platform', + 'deviceReportedStatus.status.device.portalFirmware.guid', + 'deviceReportedStatus.status.device.portalFirmware.md5', + 'deviceReportedStatus.status.device.SerialNumber', + 'deviceType', + 'metadata.config.av.realtime.mode', + 'metadata.cloudsync.cloudExtender.operationMode', + 'metadata.cloudsync.cloudExtender.selectedFolders', + 'metadata.config.fileservices.cifs.SMBEncryption', + 'metadata.config.fileservices.cifs.mode', + 'metadata.config.fileservices.cifs.packetSigning', + 'metadata.config.fileservices.cifs.passwordServer', + 'metadata.config.fileservices.cifs.type', + 'metadata.config.fileservices.ftp.RequireSSL', + 'metadata.config.fileservices.ftp.mode', + 'metadata.config.fileservices.nfs.aggregateWrites', + 'metadata.config.fileservices.nfs.async', + 'metadata.config.fileservices.nfs.krb5', + 'metadata.config.fileservices.nfs.mode', + 'metadata.config.fileservices.nfs.nfsv4enabled', + 'metadata.config.fileservices.share[].access', + 'metadata.config.fileservices.share[].acl', + 'metadata.config.fileservices.share[].clientSideCaching', + 'metadata.config.fileservices.share[].exportToFTP', + 'metadata.config.fileservices.share[].exportToNFS', + 'metadata.config.fileservices.share[].exportToWebdav', + 'metadata.config.fileservices.share[].screenedFileTypesEnabled', + 'metadata.config.fileservices.share[].trustedNFSClients', + 'metadata.config.dedup.useLocalMapFileDedup', + 'metadata.config.ransomProtect.enableHoneypot', + 'metadata.config.ransomProtect.enabled', + 'metadata.config.snmp.mode', + 'metadata.config.snmp.snmpV3.mode', + 'metadata.status.storage.arrays[].activeDevices', + 'metadata.status.storage.arrays[].allocatedCapacity', + 'metadata.status.storage.arrays[].availableCapacity', + 'metadata.status.storage.arrays[].failedDevices', + 'metadata.status.storage.arrays[].logicalCapacity', + 'metadata.status.storage.arrays[].name', + 'metadata.status.storage.arrays[].spareDevices', + 'metadata.status.storage.arrays[].state', + 'metadata.status.storage.arrays[].workingDevices', + 'metadata.status.storage.disks[].allocatedCapacity', + 'metadata.status.storage.disks[].availableCapacity', + 'metadata.status.storage.disks[].bus', + 'metadata.status.storage.disks[].capacity', + 'metadata.status.storage.disks[].logicalCapacity', + 'metadata.status.storage.disks[].name', + 'metadata.status.storage.disks[].status', + 'metadata.status.storage.summary.allocatedDriveSpace', + 'metadata.status.storage.summary.availableDriveSpace', + 'metadata.status.storage.summary.encryptedVolumeCount', + 'metadata.status.storage.summary.logicalDriveSpace', + 'metadata.status.storage.summary.physicalDriveSpace', + 'metadata.status.storage.summary.spareDriveCount', + 'metadata.status.storage.summary.state', + 'metadata.status.storage.summary.totalDriveCount', + 'metadata.status.storage.summary.totalVolumeCount', + 'metadata.status.storage.summary.unusedDriveCount', + 'metadata.status.storage.volumes[].fileSystemType', + 'metadata.status.storage.volumes[].name', + 'metadata.status.storage.volumes[].status', + 'modifiedDate', + 'owner', + 'proc.storage.summary.freeVolumeSpace', + 'proc.storage.summary.totalVolumeSpace', + 'proc.storage.summary.usedVolumeSpace', + 'storage.status.summary.allocatedDriveSpace', + 'storage.status.summary.availableDriveSpace', + 'storage.status.summary.encryptedVolumeCount', + 'storage.status.summary.logicalDriveSpace', + 'storage.status.summary.physicalDriveSpace', + 'storage.status.summary.spareDriveCount', + 'storage.status.summary.state', + 'storage.status.summary.totalDriveCount', + 'storage.status.summary.totalVolumeCount', + 'storage.status.summary.unusedDriveCount', + 'uid', + 'version', + 'name' + ], +} + + +ANONYMIZE_ATTRIBUTES = [ + '.portals[].cloudDrives[].owner', + '.portals[].cloudDrives[].group', + '.portals[].devices[].owner', + '.poratls[].buckets[].cloudDrive', + '.locations[].storageClass', + '.locations[].dedicatedPortal', + '.portals[].foldersGroups[].storageClass' +] + + +COUNT_ATTRIBUTES = [ + '.portals[].devices[].metadata.config.fileservices.share[].acl', + '.portals[].devices[].metadata.config.fileservices.share[].trustedNFSClients' +] + + +def filter_object(o, attribute_paths): + tree = {} + + for prefix, attributes in attribute_paths.items(): + for attribute in attributes: + current = tree + for part in f'{prefix}{attribute}'.lstrip('.').split('.'): + is_list = part.endswith('[]') + name = part.removesuffix('[]') + current = current.setdefault(name, (is_list, {}))[1] + + return _filter_object(o, tree) + + +def _filter_object(o, tree): + result = Object() + + for name, (is_list, children) in tree.items(): + if not hasattr(o, name): + continue + + value = getattr(o, name) + + if value is None or not children: + setattr(result, name, value) + elif is_list: + setattr(result, name, [ + _filter_object(item, children) if item is not None else None + for item in value + ]) + else: + setattr(result, name, _filter_object(value, children)) + + return result if vars(result) else None + + +def transform_attributes(o, attributes, transform): + for attribute in attributes: + _transform_attribute(o, attribute.lstrip('.').split('.'), transform) + return o + + +def _transform_attribute(o, parts, transform): + if o is None: + return + + part = parts[0] + + if part.endswith('[]'): + for item in getattr(o, part[:-2], None) or []: + _transform_attribute(item, parts[1:], transform) + return + + value = getattr(o, part, None) + + if value is None: + return + + if len(parts) == 1: + setattr(o, part, transform(value)) + else: + _transform_attribute(value, parts[1:], transform) + + +def anonymize_uids(o, attributes): + return transform_attributes( + o, + attributes, + lambda value: int(parse_base_object_ref(value).uid), + ) + + +def count_attributes(o, attributes): + return transform_attributes( + o, + attributes, + len, + ) + + +async def retry(coro, retries, return_on_error=None): + for attempt in range(retries): + try: + return await coro() + except (GatewayTimeout, BadGateway, InternalServerError) as e: + if attempt == retries - 1: + logger.error("Operation failed after %d attempts: %s", retries, e) + if return_on_error: + return return_on_error + raise + + logger.warning("Operation failed (attempt %d/%d), retrying: %s", attempt + 1, retries, e) + await asyncio.sleep(1) + + +def expand_portal_schema(array, portals, attribute, identifier='portal'): + for element in array: + value = getattr(element, identifier) + if value: + portal = portals[int(parse_base_object_ref(value).uid)] + getattr(portal, attribute).append(element) + + +async def enumerate_devices(admin, inspect=True, max_workers=5): + """ + Enumerate devices, including general metadata captured by CTERA Portal. + + Args: + admin: Global administrator session. + inspect (bool, optional): Inspect the configuration of connected devices. Defaults to True. + max_workers (int, optional): Max number of concurrent requests for inspection. Defaults to 5. + + Returns: + dict: A dictionary mapping an Edge Filer's unique numeric ID to either its + raw device object (if inspect=False) or its configuration object (if inspect=True). + """ + devices = [device async for device in admin.devices.devices([ + 'uid', + 'name', + 'portal', + 'version', + 'owner', + 'createDate', + 'modifiedDate', + 'deviceType', + 'deviceConnectionStatus', + 'deviceReportedStatus' + ], allPortals=True)] + + def serialize(device): + return fromjsonstr(str(device)) + + devices = await inspect_devices(devices, max_workers) if inspect else devices + + return [serialize(device) for device in devices] + + +async def inspect_devices(devices, max_workers): + """ + Inspect the configuration of connected devices. + + Args: + devices (dict): A dictionary mapping device UIDs to device objects. + max_workers (int): Max number of concurrent requests. + + Returns: + dict: A dictionary mapping an Edge Filer's unique numeric ID to its configuration object. + """ + + def inspectable(device): + return ( + device.deviceType in ['vGateway', 'C200', 'C400', 'C800', 'C800P', 'VBox', 'CloudPlug'] + and device.deviceConnectionStatus.connected + ) + + async def inspect_device(device, semaphore): + async with semaphore: + device.metadata = await device.api.get_multi('/', [ + '/config/fileservices/nfs', + '/config/fileservices/ftp', + '/config/fileservices/cifs', + '/config/fileservices/share', + '/config/av/realtime', + '/config/ransomProtect', + '/config/snmp', + '/config/dedup/useLocalMapFileDedup', + '/status/storage', + '/config/cloudsync/cloudExtender/selectedFolders', + '/config/cloudsync/metadataPinning', + '/config/cloudsync/cloudExtender/operationMode' + ]) + return device + + tasks = [] + semaphore = asyncio.Semaphore(max_workers) + + for device in devices: + if inspectable(device): + tasks.append( + retry( + lambda device=device: inspect_device(device, semaphore), + 3, + device + ) + ) + + return await asyncio.gather(*tasks) + + +async def enumerate_portals(admin): + """ + Enumerate Virtual Portals. + + Args: + admin: Global administrator session. + + Returns: + dict: A dictionary mapping an Virtual Portal's unique numeric ID to Virtual Portal objects. + """ + return {portal.uid: portal async for portal in admin.portals.tenants()} + + +async def enumerate_cloudfolders(admin, portals): + """ + Enumerate Cloud Drive Folders. + + Args: + admin: Global administrator session. + portals (list(str)): A list of Virtual Portal names. + """ + include = [ + "uid", + "createDate", + "modifiedDate", + "enableSyncWinNtExtendedAttributes", + "extendedAttributes.enable", + "folderStats.cloudFolderSize", + "folderStats.totalFiles", + "globalFileLockSettings", + "group", + "owner", + "portal", + "teamProject", + "folderQuota", + "isDeleted", + "wormSettings", + "archiveSettings", + "openFabricSettings.storageMode", + "openFabricSettings.dataStorage.storage", + "openFabricStorageStatus", + "openStorageEnabled" + ] + + return [drive async for drive in admin.cloudfs.drives.all(include, portals)] + + +async def enumerate_zones(admin, portals): + """ + Enumerate Zones. + + Args: + admin: Global administrator session. + portals (list(str)): A list of Virtual Portal names. + """ + return {portal: [zone async for zone in admin.cloudfs.zones.all(True, [portal])] for portal in portals} + + +async def enumerate_folder_groups(admin, portals): + """ + Enumerate Folder Groups. + + Args: + admin: Global administrator session. + portals (list(str)): A list of Virtual Portal names. + """ + include = [ + "uid", + "portal", + "createDate", + "modifiedDate", + "deduplicationMethodType", + "fixedBlockSizeKb", + "averageBlockSizeKb", + "averageMapFileSizeKb", + "mapFileInDB", + "storageClass", + "encryptionMode", + "compressionMethod", + ] + + return [group async for group in admin.cloudfs.groups.all(include, portals)] + + +async def enumerate_exports(admin, portals): + """ + Enumerate Fusion Direct S3 Exports. + + Args: + admin: Global administrator session. + portals (list(str)): A list of Virtual Portal names. + """ + return [export async for export in admin.cloudfs.exports.all(portals)] + + +async def enumerate_subscription_plans(admin): + """ + Expand subscription plans. + + Args: + admin: Global administrator session. + """ + include = [ + 'uid', + 'services', + 'archiveStorage', + 'vGateways', + 'vGateways4', + 'vGateways8', + 'vGateways32', + 'vGateways64', + 'vGateways128', + 'vGateways256', + 'storage', + 'cloudDrives', + 'cloudDrivesLite', + 'serverAgents', + 'workstationAgents', + 'createDate', + 'modifiedDate', + 'retentionPolicy' + ] + return {plan.uid: plan async for plan in admin.plans.list_plans(include)} + + +async def enumerate_servers(admin): + """ + Enumerate servers. + + Args: + admin: Global administrator session. + """ + include = [ + 'createDate', + 'modifiedDate', + 'connected', + 'name', + 'mainDB', + 'isAVBGServer', + 'isApplicationServer', + 'isMessagingServer', + 'isS3Endpoint', + 'isThumbnailsServer', + 'renderingServer', + 'previewStatus', + 'runningVersion', + 'replicationSettings.replicationOf', + 'systemInfo', + 'backupToBucket' + ] + return [server async for server in admin.servers.list_servers(include)] + + +async def enumerate_locations(admin): + """ + Enumerate Storage Nodes. + + Args: + admin: Global administrator session. + """ + include = [ + "connected", + "name", + "storage", + "bucket", + "readOnly", + "status", + "dedicated", + "dedicatedPortal", + "directUpload", + "httpsOnly", + "useHttps", + "trustAllCertificates", + "objectLock", + "usePathStyleAddressing", + "s3Endpoint", + "endPoint", + "createDate", + "modifiedDate", + "storageClass", + "doFsync", + "folderSize" + ] + + return [node async for node in admin.buckets.list_buckets(include)] + + +async def inspect_environment(admin): + """ + Enumerate global licenses, firmwares, settings and microservices. + + Args: + admin: Global administrator session. + """ + root = await admin.v1.api.get_multi('', [ + '/portalLicenses', + '/firmwares', + '/settings' + ]) + root.microservices = await admin.v1.api.get('/microservices') + return root + + +async def export_objects(admin): + """ + Enumerate every object CTERA Portal exposes, chain it into a single object, then + filter it down to the attributes in ATTRIBUTE_PATHS and anonymize/count as configured. + + Args: + admin: Global administrator session. + """ + + logger.info('Retrieving global settings, portals, servers and storage nodes...') + root, portals, servers, locations = await asyncio.gather( + inspect_environment(admin), + enumerate_portals(admin), + enumerate_servers(admin), + enumerate_locations(admin) + ) + + names = [portal.name for portal in portals.values()] + + logger.info('Retrieving plans, folder groups, cloud drives, zones, buckets, and devices ' + '(device inspection may take a few minutes)...') + plans, folder_groups, cloudfolders, zones, buckets, devices = await asyncio.gather( + enumerate_subscription_plans(admin), + enumerate_folder_groups(admin, names), + enumerate_cloudfolders(admin, names), + enumerate_zones(admin, names), + enumerate_exports(admin, names), + enumerate_devices(admin, True) + ) + + for portal in portals.values(): + portal.plan = plans[int(parse_base_object_ref(portal.plan).uid)] + portal.locations = [] + portal.foldersGroups = [] + portal.cloudDrives = [] + portal.zones = zones[portal.name] + portal.buckets = [] + portal.devices = [] + + expand_portal_schema(locations, portals, 'locations', 'dedicatedPortal') + expand_portal_schema(folder_groups, portals, 'foldersGroups') + expand_portal_schema(cloudfolders, portals, 'cloudDrives') + expand_portal_schema(buckets, portals, 'buckets') + expand_portal_schema(devices, portals, 'devices') + + root.servers = servers + root.locations = locations + root.portals = list(portals.values()) + + logger.info('Filtering, anonymizing and counting collected data...') + result = filter_object(root, ATTRIBUTE_PATHS) + result = anonymize_uids(result, ANONYMIZE_ATTRIBUTES) + result = count_attributes(result, COUNT_ATTRIBUTES) + + return result + + +async def main(args): + async with AsyncGlobalAdmin(args.address) as admin: + try: + await admin.login(args.user, args.password) + await admin.portals.browse_global_admin() + except AuthenticationError as error: + logger.error("Login to %s as '%s' failed: %s", args.address, args.user, error) + return None + + try: + return await export_objects(admin) + except HTTPError as error: + logger.error('Failed to export objects from %s: %s', args.address, error) + logger.error('Reason: %s', error.error) + return None + finally: + await admin.logout() + + +def configure_logging(debug): + logging.basicConfig( + level=logging.DEBUG if debug else logging.INFO, + format='%(asctime)s %(levelname)-8s %(name)s: %(message)s', + ) + + +def configure_transport_layer_security(no_verify): + settings.core.asyn.settings.connector.ssl = not no_verify + + +def configure_timeout(): + settings.core.asyn.settings.timeout.sock_connect = 180 + settings.core.asyn.settings.timeout.sock_read = 180 + settings.edge.asyn.settings.timeout.sock_connect = 180 + settings.edge.asyn.settings.timeout.sock_read = 180 + + +def parse_args(): + parser = argparse.ArgumentParser(description='Generate a configuration export for import into User Center.') + parser.add_argument('-a', dest='address', required=True, help='CTERA Portal address') + parser.add_argument('-u', dest='user', required=True, help='Support or read-only admin username') + parser.add_argument('-p', dest='password', required=True, help='Support or read-only admin password') + parser.add_argument('-o', '--output', default=None, + help='Path to write the export file to (default: .
.cterasdk.export.json)') + parser.add_argument('--no-verify', action='store_true', help='Disable TLS verification') + parser.add_argument('--debug', action='store_true', help='Enable verbose (debug) logging') + parser.add_argument('--shared', action='store_true', help='Enable if this Portal serves multiple distinct organizations.') + args = parser.parse_args() + + if not args.output: + args.output = f'{datetime.now():%Y%m%d_%H%M%S}.{args.address}.cterasdk.export.json' + + return args + + +def run(): + args = parse_args() + configure_logging(args.debug) + configure_transport_layer_security(args.no_verify) + configure_timeout() + + result = asyncio.run(main(args)) + if result is None: + sys.exit(1) + + result.site = Object(**{ + 'type': 'shared' if args.shared else 'private' + }) + + with open(args.output, 'w', encoding='utf-8') as f: + f.write(str(result)) + logger.info('Export written to %s', args.output) diff --git a/cterasdk/clients/clients.py b/cterasdk/clients/clients.py index 4f2957fe..7f730eb9 100644 --- a/cterasdk/clients/clients.py +++ b/cterasdk/clients/clients.py @@ -246,7 +246,8 @@ def multipart(self, path, form, *, on_response=None, on_error=None, **kwargs): @decorators.authenticated def delete(self, path, data=None, *, data_serializer=None, on_response=None, on_error=None, **kwargs): - request = async_requests.DeleteRequest(self._builder(path), data=data_serializer(data), **kwargs) + data = data_serializer(data) if data is not None else data + request = async_requests.DeleteRequest(self._builder(path), data=data, **kwargs) return self.request(request, on_response=on_response, on_error=on_error) def _request(self, request, *, on_response=None, on_error=None): diff --git a/cterasdk/core/cloudfs.py b/cterasdk/core/cloudfs.py index 6dc91b83..7aacbfe1 100644 --- a/cterasdk/core/cloudfs.py +++ b/cterasdk/core/cloudfs.py @@ -759,7 +759,7 @@ def _zone_param(name, policy_type, description=None, zid=None): class Exports(BaseCommand): - """ S3 Exports APIs """ + """ Fusion Gateway S3 APIs """ def get(self, name): """ diff --git a/cterasdk/objects/asynchronous/core.py b/cterasdk/objects/asynchronous/core.py index 8a1564a1..69d0f0ad 100644 --- a/cterasdk/objects/asynchronous/core.py +++ b/cterasdk/objects/asynchronous/core.py @@ -4,7 +4,8 @@ from ...clients import clients from .. import authenticators from ...lib.session.core import Session -from ...asynchronous.core import files, login, cloudfs, devices, notifications, portals, roles, settings, tasks, users +from ...asynchronous.core import buckets, files, login, cloudfs, devices, notifications, plans, portals, roles, servers, \ + settings, tasks, users class Clients: @@ -90,8 +91,11 @@ class AsyncGlobalAdmin(AsyncPortal): def __init__(self, host, port=None, https=True): super().__init__(host, port, https) + self.buckets = buckets.Buckets(self) + self.plans = plans.Plans(self) self.portals = portals.Portals(self) self.devices = devices.Devices(self) + self.servers = servers.Servers(self) @property def context(self): diff --git a/docs/source/UserGuides/CLI/Index.rst b/docs/source/UserGuides/CLI/Index.rst index 2b6aeaff..b1ceec18 100644 --- a/docs/source/UserGuides/CLI/Index.rst +++ b/docs/source/UserGuides/CLI/Index.rst @@ -100,3 +100,64 @@ Uploading files ^^^^^^^^^^^^^^^ ``cterasdk.io.dav upload -e ENDPOINT [-d DEST] files [files ...]`` + + +User Center +----------- + +The ``cterasdk.platform.export`` command generates a snapshot of the CTERA +Global File System configuration for import into CTERA User Center. + +Usage +^^^^^ + +.. code-block:: console + + cterasdk.platform.export -a
-u -p [options] + +Arguments +^^^^^^^^^ + +``-a ADDRESS`` + CTERA Portal address. + +``-u USER`` + Support or read-only administrator username. + +``-p PASSWORD`` + Support or read-only administrator password. + +``-o, --output OUTPUT`` + Path to write the export file to. By default, the file is written as + ``.
.cterasdk.export.json``. + +``--no-verify`` + Disable TLS certificate verification. + +``--debug`` + Enable verbose debug logging. + +``--shared`` + Enable this option if the Portal serves multiple distinct organizations. + +Examples +^^^^^^^^ + +Generate a configuration export using the default output filename: + +.. code-block:: console + + cterasdk.platform.export -a portal.example.com -u admin -p password + +Specify an output file: + +.. code-block:: console + + cterasdk.platform.export -a portal.example.com -u admin -p password \ + -o platform-export.json + +Generate an export for a Portal serving multiple distinct organizations: + +.. code-block:: console + + cterasdk.platform.export -a portal.example.com -u admin -p password --shared diff --git a/docs/source/UserGuides/Miscellaneous/Changelog.rst b/docs/source/UserGuides/Miscellaneous/Changelog.rst index cb4e154a..ed1f66c4 100644 --- a/docs/source/UserGuides/Miscellaneous/Changelog.rst +++ b/docs/source/UserGuides/Miscellaneous/Changelog.rst @@ -1,18 +1,38 @@ Changelog ========= +2.20.45 +------- + +Improvements +^^^^^^^^^^^^ + +* Added a CLI command for exporting the Global File System for User Center. + +Related issues and pull requests on GitHub: `#368 `_ + + 2.20.44 -======= +------- + +Improvements +^^^^^^^^^^^^ + +- Support path-style addressing when connecting to S3-compatible buckets +- Support listing zone cloud folders and devices +- Support remote access via CTTP using asynchronous I/O (`asyncio`) Bug Fixes ^^^^^^^^^ -- Support path style addressing when connecting to S3-Compatible buckets -- Support listing zone cloud folders and devices -- Support remote access via CTTP through asynchronous I/O (``asyncio``) -- Updated the Edge Filer shares module to accommodate the removal of the ``dirPermissions`` attribute. +- Resolve a regression in delete requests that do not contain data +- Update the Edge Filer shares module to accommodate the removal of the + ``dirPermissions`` attribute Related issues and pull requests on GitHub: `#365 `_ +`#366 `_ +`#367 `_ + 2.20.43 ------- diff --git a/pyproject.toml b/pyproject.toml index 73ea6955..538cc253 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ Documentation = "https://ctera-python-sdk.readthedocs.io/en/latest/" [project.scripts] "cterasdk.io.direct.download" = "cterasdk.cli.direct:download_object" "cterasdk.io.dav" = "cterasdk.cli.dav:webdav" +"cterasdk.platform.export" = "cterasdk.cli.export:run" [tool.setuptools.dynamic] dependencies = {file = ["requirements.txt"]}