Skip to content

signals#

LegacyFileUpdateFailed#

Bases: APIException

Source code in src/apps/files/signals.py 
26
27
class LegacyFileUpdateFailed(exceptions.APIException):
    status_code = status.HTTP_409_CONFLICT

get_v2_request_settings#

Source code in src/apps/files/signals.py 
30
31
32
33
34
35
36
37
def get_v2_request_settings():
    host = f"{settings.METAX_V2_HOST}/rest/v2"
    headers = urllib3.make_headers(
        basic_auth=f"{settings.METAX_V2_USER}:{settings.METAX_V2_PASSWORD}",
    )
    headers["Content-Type"] = "application/json"
    headers["Accept"] = "application/json"
    return host, headers

handle_sync_files#

Source code in src/apps/files/signals.py 
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
@receiver(sync_files)
def handle_sync_files(sender, actions: List[dict], **kwargs):
    if not settings.METAX_V2_INTEGRATION_ENABLED:
        return

    to_legacy = []
    files_without_legacy_ids = {}
    non_syncable_count = 0
    for file_action in actions:
        file: File = file_action["object"]
        if not file.is_legacy_syncable:
            non_syncable_count += 1
            continue
        if file.legacy_id is None:
            files_without_legacy_ids[(file.storage_service, file.storage_identifier)] = file
        to_legacy.append(file.to_legacy_sync())

    if non_syncable_count > 0:
        logger.info(f"{non_syncable_count} files are non-syncable and will not be synced to V2")

    host, headers = get_v2_request_settings()
    body = json.dumps(to_legacy, cls=DjangoJSONEncoder)
    res = requests.post(url=f"{host}/files/sync_from_v3", data=body, headers=headers)
    if res.status_code in {200, 201}:
        logger.info(f"Synced {len(to_legacy)} files to V2")
    else:
        logger.error(
            f"Syncing files to V2 failed: {res.status_code=}:\n  {res.content=}, \n  {res.headers=}"
        )
        raise LegacyFileUpdateFailed("Failed to sync files to Metax V2")

    # Fill in missing legacy_ids from response data
    for v2_file in res.json():
        v3_storage = settings.LEGACY_FILE_STORAGE_TO_V3_STORAGE_SERVICE[v2_file["file_storage"]]
        if file := files_without_legacy_ids.get((v3_storage, v2_file["identifier"])):
            file.legacy_id = v2_file["id"]

    for key, file in files_without_legacy_ids.items():
        if file.legacy_id is None:
            logger.warn(f"Sync error: file {key} did not get a legacy id")

    # Update legacy_id values for files that didn't have one yet
    files_with_new_legacy_ids = [f for f in files_without_legacy_ids.values() if f.legacy_id]
    File.all_objects.bulk_update(files_with_new_legacy_ids, fields=["legacy_id"], batch_size=2000)