From 3f6ac40fb4cff4b9b4cead24b898bd082526b033 Mon Sep 17 00:00:00 2001 From: Nutcake Date: Wed, 17 May 2023 16:32:23 +0200 Subject: [PATCH] Clean up asset upload code a litle --- lib/apis/record_api.dart | 137 +++++++++++---------------- lib/models/records/asset_digest.dart | 25 +++++ lib/models/records/record.dart | 49 ++++++++++ 3 files changed, 129 insertions(+), 82 deletions(-) create mode 100644 lib/models/records/asset_digest.dart diff --git a/lib/apis/record_api.dart b/lib/apis/record_api.dart index 1fb8e75..18d1cc2 100644 --- a/lib/apis/record_api.dart +++ b/lib/apis/record_api.dart @@ -2,10 +2,8 @@ import 'dart:convert'; import 'dart:io'; import 'dart:math'; import 'dart:typed_data'; -import 'dart:ui'; import 'package:collection/collection.dart'; -import 'package:contacts_plus_plus/auxiliary.dart'; -import 'package:contacts_plus_plus/models/message.dart'; +import 'package:contacts_plus_plus/models/records/asset_digest.dart'; import 'package:contacts_plus_plus/models/records/image_template.dart'; import 'package:http/http.dart' as http; import 'package:flutter/material.dart'; @@ -45,6 +43,19 @@ class RecordApi { return PreprocessStatus.fromMap(body); } + static Future tryPreprocessRecord(ApiClient client, {required Record record}) async { + var status = await preprocessRecord(client, record: record); + while (status.state == RecordPreprocessState.preprocessing) { + await Future.delayed(const Duration(seconds: 1)); + status = await getPreprocessStatus(client, preprocessStatus: status); + } + + if (status.state != RecordPreprocessState.success) { + throw "Record Preprocessing failed: ${status.failReason}"; + } + return status; + } + static Future beginUploadAsset(ApiClient client, {required NeosDBAsset asset}) async { final response = await client.post("/users/${client.userId}/assets/${asset.hash}/chunks"); ApiClient.checkResponse(response); @@ -60,14 +71,18 @@ class RecordApi { ApiClient.checkResponse(response); } - static Future uploadAsset(ApiClient client, {required AssetUploadData uploadData, required String filename, required NeosDBAsset asset, required Uint8List data}) async { + static Future uploadAsset(ApiClient client, + {required AssetUploadData uploadData, required String filename, required NeosDBAsset asset, required Uint8List data}) async { for (int i = 0; i < uploadData.totalChunks; i++) { - final offset = i*uploadData.chunkSize; - final end = (i+1)*uploadData.chunkSize; + final offset = i * uploadData.chunkSize; + final end = (i + 1) * uploadData.chunkSize; final request = http.MultipartRequest( "POST", ApiClient.buildFullUri("/users/${client.userId}/assets/${asset.hash}/chunks/$i"), - )..files.add(http.MultipartFile.fromBytes("file", data.getRange(offset, min(end, data.length)).toList(), filename: filename, contentType: MediaType.parse("multipart/form-data"))) + ) + ..files.add(http.MultipartFile.fromBytes( + "file", data.getRange(offset, min(end, data.length)).toList(), filename: filename, + contentType: MediaType.parse("multipart/form-data"))) ..headers.addAll(client.authorizationHeader); final response = await request.send(); final bodyBytes = await response.stream.toBytes(); @@ -80,87 +95,45 @@ class RecordApi { ApiClient.checkResponse(response); } - static Future uploadImage(ApiClient client, {required File image, required String machineId}) async { - final imageData = await image.readAsBytes(); - final imageImage = await decodeImageFromList(imageData); - final imageAsset = NeosDBAsset.fromData(imageData); - final imageNeosDbUri = "neosdb:///${imageAsset.hash}${extension(image.path)}"; - final objectJson = jsonEncode(ImageTemplate(imageUri: imageNeosDbUri, width: imageImage.width, height: imageImage.height).data); - final objectBytes = Uint8List.fromList(utf8.encode(objectJson)); - final objectAsset = NeosDBAsset.fromData(objectBytes); - final objectNeosDbUri = "neosdb:///${objectAsset.hash}.json"; - final combinedRecordId = RecordId(id: Record.generateId(), ownerId: client.userId, isValid: true); - final filename = basenameWithoutExtension(image.path); - final record = Record( - id: combinedRecordId.id.toString(), - combinedRecordId: combinedRecordId, - assetUri: objectNeosDbUri, - name: filename, - tags: [ - filename, - "message_item", - "message_id:${Message.generateId()}" - ], - recordType: RecordType.texture, - thumbnailUri: imageNeosDbUri, - isPublic: false, - isForPatreons: false, - isListed: false, - neosDBManifest: [ - imageAsset, - objectAsset, - ], - globalVersion: 0, - localVersion: 1, - lastModifyingUserId: client.userId, - lastModifyingMachineId: machineId, - lastModificationTime: DateTime.now().toUtc(), - creationTime: DateTime.now().toUtc(), - ownerId: client.userId, - isSynced: false, - fetchedOn: DateTimeX.one, - path: '', - description: '', - manifest: [ - imageNeosDbUri, - objectNeosDbUri - ], - url: "neosrec:///${client.userId}/${combinedRecordId.id}", - isValidOwnerId: true, - isValidRecordId: true, - visits: 0, - rating: 0, - randomOrder: 0, - ); - - var status = await preprocessRecord(client, record: record); - while (status.state == RecordPreprocessState.preprocessing) { - await Future.delayed(const Duration(seconds: 1)); - status = await getPreprocessStatus(client, preprocessStatus: status); - } - - if (status.state != RecordPreprocessState.success) { - throw "Record Preprocessing failed: ${status.failReason}"; - } - AssetUploadData uploadData; - if ((status.resultDiffs.firstWhereOrNull((element) => element.hash == imageAsset.hash)?.isUploaded ?? false) == false) { - uploadData = await beginUploadAsset(client, asset: imageAsset); + static Future uploadAssets(ApiClient client, {required List assets}) async { + for (final entry in assets) { + final uploadData = await beginUploadAsset(client, asset: entry.asset); if (uploadData.uploadState == UploadState.failed) { throw "Asset upload failed: ${uploadData.uploadState.name}"; } - - await uploadAsset(client, uploadData: uploadData, asset: imageAsset, data: imageData, filename: filename); - await finishUpload(client, asset: imageAsset); + await uploadAsset(client, uploadData: uploadData, asset: entry.asset, data: entry.data, filename: entry.name); + await finishUpload(client, asset: entry.asset); } + } - uploadData = await beginUploadAsset(client, asset: objectAsset); - if (uploadData.uploadState == UploadState.failed) { - throw "Asset upload failed: ${uploadData.uploadState.name}"; - } + static Future uploadImage(ApiClient client, {required File image, required String machineId}) async { + final imageDigest = await AssetDigest.fromData(await image.readAsBytes(), basename(image.path)); + final imageData = await decodeImageFromList(imageDigest.data); - await uploadAsset(client, uploadData: uploadData, asset: objectAsset, data: objectBytes, filename: filename); - await finishUpload(client, asset: objectAsset); + final objectJson = jsonEncode( + ImageTemplate(imageUri: imageDigest.dbUri, width: imageData.width, height: imageData.height).data); + final objectBytes = Uint8List.fromList(utf8.encode(objectJson)); + final objectDigest = await AssetDigest.fromData(objectBytes, "${basenameWithoutExtension(image.path)}.json"); + + final filename = basenameWithoutExtension(image.path); + final digests = [imageDigest, objectDigest]; + + final record = Record.fromRequiredData( + recordType: RecordType.texture, + userId: client.userId, + machineId: machineId, + assetUri: objectDigest.dbUri, + filename: filename, + thumbnailUri: imageDigest.dbUri, + digests: digests, + ); + + final status = await tryPreprocessRecord(client, record: record); + final toUpload = status.resultDiffs.whereNot((element) => element.isUploaded); + + await uploadAssets( + client, assets: digests.where((digest) => toUpload.any((diff) => digest.asset.hash == diff.hash)).toList()); return record; } -} \ No newline at end of file +} diff --git a/lib/models/records/asset_digest.dart b/lib/models/records/asset_digest.dart new file mode 100644 index 0000000..33b7332 --- /dev/null +++ b/lib/models/records/asset_digest.dart @@ -0,0 +1,25 @@ + +import 'dart:typed_data'; + +import 'package:contacts_plus_plus/models/records/neos_db_asset.dart'; +import 'package:path/path.dart'; + +class AssetDigest { + final Uint8List data; + final NeosDBAsset asset; + final String name; + final String dbUri; + + AssetDigest({required this.data, required this.asset, required this.name, required this.dbUri}); + + static Future fromData(Uint8List data, String filename) async { + final asset = NeosDBAsset.fromData(data); + + return AssetDigest( + data: data, + asset: asset, + name: basenameWithoutExtension(filename), + dbUri: "neosdb:///${asset.hash}${extension(filename)}", + ); + } +} \ No newline at end of file diff --git a/lib/models/records/record.dart b/lib/models/records/record.dart index 085754c..40e5bfd 100644 --- a/lib/models/records/record.dart +++ b/lib/models/records/record.dart @@ -1,4 +1,6 @@ import 'package:contacts_plus_plus/auxiliary.dart'; +import 'package:contacts_plus_plus/models/message.dart'; +import 'package:contacts_plus_plus/models/records/asset_digest.dart'; import 'package:contacts_plus_plus/models/records/neos_db_asset.dart'; import 'package:contacts_plus_plus/string_formatter.dart'; import 'package:flutter/material.dart'; @@ -101,6 +103,53 @@ class Record { required this.randomOrder, }) : formattedName = FormatNode.fromText(name); + factory Record.fromRequiredData({ + required RecordType recordType, + required String userId, + required String machineId, + required String assetUri, + required String filename, + required String thumbnailUri, + required List digests, + }) { + final combinedRecordId = RecordId(id: Record.generateId(), ownerId: userId, isValid: true); + return Record( + id: combinedRecordId.id.toString(), + combinedRecordId: combinedRecordId, + assetUri: assetUri, + name: filename, + tags: [ + filename, + "message_item", + "message_id:${Message.generateId()}" + ], + recordType: recordType, + thumbnailUri: thumbnailUri, + isPublic: false, + isForPatreons: false, + isListed: false, + neosDBManifest: digests.map((e) => e.asset).toList(), + globalVersion: 0, + localVersion: 1, + lastModifyingUserId: userId, + lastModifyingMachineId: machineId, + lastModificationTime: DateTime.now().toUtc(), + creationTime: DateTime.now().toUtc(), + ownerId: userId, + isSynced: false, + fetchedOn: DateTimeX.one, + path: '', + description: '', + manifest: digests.map((e) => e.dbUri).toList(), + url: "neosrec:///$userId/${combinedRecordId.id}", + isValidOwnerId: true, + isValidRecordId: true, + visits: 0, + rating: 0, + randomOrder: 0, + ); + } + factory Record.fromMap(Map map) { return Record( id: map["id"] ?? "0",