Clean up asset upload code a litle

This commit is contained in:
Nutcake 2023-05-17 16:32:23 +02:00
parent 76fcec05de
commit 3f6ac40fb4
3 changed files with 129 additions and 82 deletions

View file

@ -2,10 +2,8 @@ import 'dart:convert';
import 'dart:io';
import 'dart:math';
import 'dart:typed_data';
import 'dart:ui';
import 'package:collection/collection.dart';
import 'package:contacts_plus_plus/auxiliary.dart';
import 'package:contacts_plus_plus/models/message.dart';
import 'package:contacts_plus_plus/models/records/asset_digest.dart';
import 'package:contacts_plus_plus/models/records/image_template.dart';
import 'package:http/http.dart' as http;
import 'package:flutter/material.dart';
@ -45,6 +43,19 @@ class RecordApi {
return PreprocessStatus.fromMap(body);
}
static Future<PreprocessStatus> tryPreprocessRecord(ApiClient client, {required Record record}) async {
var status = await preprocessRecord(client, record: record);
while (status.state == RecordPreprocessState.preprocessing) {
await Future.delayed(const Duration(seconds: 1));
status = await getPreprocessStatus(client, preprocessStatus: status);
}
if (status.state != RecordPreprocessState.success) {
throw "Record Preprocessing failed: ${status.failReason}";
}
return status;
}
static Future<AssetUploadData> beginUploadAsset(ApiClient client, {required NeosDBAsset asset}) async {
final response = await client.post("/users/${client.userId}/assets/${asset.hash}/chunks");
ApiClient.checkResponse(response);
@ -60,14 +71,18 @@ class RecordApi {
ApiClient.checkResponse(response);
}
static Future<void> uploadAsset(ApiClient client, {required AssetUploadData uploadData, required String filename, required NeosDBAsset asset, required Uint8List data}) async {
static Future<void> uploadAsset(ApiClient client,
{required AssetUploadData uploadData, required String filename, required NeosDBAsset asset, required Uint8List data}) async {
for (int i = 0; i < uploadData.totalChunks; i++) {
final offset = i * uploadData.chunkSize;
final end = (i + 1) * uploadData.chunkSize;
final request = http.MultipartRequest(
"POST",
ApiClient.buildFullUri("/users/${client.userId}/assets/${asset.hash}/chunks/$i"),
)..files.add(http.MultipartFile.fromBytes("file", data.getRange(offset, min(end, data.length)).toList(), filename: filename, contentType: MediaType.parse("multipart/form-data")))
)
..files.add(http.MultipartFile.fromBytes(
"file", data.getRange(offset, min(end, data.length)).toList(), filename: filename,
contentType: MediaType.parse("multipart/form-data")))
..headers.addAll(client.authorizationHeader);
final response = await request.send();
final bodyBytes = await response.stream.toBytes();
@ -80,87 +95,45 @@ class RecordApi {
ApiClient.checkResponse(response);
}
static Future<void> uploadAssets(ApiClient client, {required List<AssetDigest> assets}) async {
for (final entry in assets) {
final uploadData = await beginUploadAsset(client, asset: entry.asset);
if (uploadData.uploadState == UploadState.failed) {
throw "Asset upload failed: ${uploadData.uploadState.name}";
}
await uploadAsset(client, uploadData: uploadData, asset: entry.asset, data: entry.data, filename: entry.name);
await finishUpload(client, asset: entry.asset);
}
}
static Future<Record> uploadImage(ApiClient client, {required File image, required String machineId}) async {
final imageData = await image.readAsBytes();
final imageImage = await decodeImageFromList(imageData);
final imageAsset = NeosDBAsset.fromData(imageData);
final imageNeosDbUri = "neosdb:///${imageAsset.hash}${extension(image.path)}";
final objectJson = jsonEncode(ImageTemplate(imageUri: imageNeosDbUri, width: imageImage.width, height: imageImage.height).data);
final imageDigest = await AssetDigest.fromData(await image.readAsBytes(), basename(image.path));
final imageData = await decodeImageFromList(imageDigest.data);
final objectJson = jsonEncode(
ImageTemplate(imageUri: imageDigest.dbUri, width: imageData.width, height: imageData.height).data);
final objectBytes = Uint8List.fromList(utf8.encode(objectJson));
final objectAsset = NeosDBAsset.fromData(objectBytes);
final objectNeosDbUri = "neosdb:///${objectAsset.hash}.json";
final combinedRecordId = RecordId(id: Record.generateId(), ownerId: client.userId, isValid: true);
final objectDigest = await AssetDigest.fromData(objectBytes, "${basenameWithoutExtension(image.path)}.json");
final filename = basenameWithoutExtension(image.path);
final record = Record(
id: combinedRecordId.id.toString(),
combinedRecordId: combinedRecordId,
assetUri: objectNeosDbUri,
name: filename,
tags: [
filename,
"message_item",
"message_id:${Message.generateId()}"
],
final digests = [imageDigest, objectDigest];
final record = Record.fromRequiredData(
recordType: RecordType.texture,
thumbnailUri: imageNeosDbUri,
isPublic: false,
isForPatreons: false,
isListed: false,
neosDBManifest: [
imageAsset,
objectAsset,
],
globalVersion: 0,
localVersion: 1,
lastModifyingUserId: client.userId,
lastModifyingMachineId: machineId,
lastModificationTime: DateTime.now().toUtc(),
creationTime: DateTime.now().toUtc(),
ownerId: client.userId,
isSynced: false,
fetchedOn: DateTimeX.one,
path: '',
description: '',
manifest: [
imageNeosDbUri,
objectNeosDbUri
],
url: "neosrec:///${client.userId}/${combinedRecordId.id}",
isValidOwnerId: true,
isValidRecordId: true,
visits: 0,
rating: 0,
randomOrder: 0,
userId: client.userId,
machineId: machineId,
assetUri: objectDigest.dbUri,
filename: filename,
thumbnailUri: imageDigest.dbUri,
digests: digests,
);
var status = await preprocessRecord(client, record: record);
while (status.state == RecordPreprocessState.preprocessing) {
await Future.delayed(const Duration(seconds: 1));
status = await getPreprocessStatus(client, preprocessStatus: status);
}
if (status.state != RecordPreprocessState.success) {
throw "Record Preprocessing failed: ${status.failReason}";
}
AssetUploadData uploadData;
if ((status.resultDiffs.firstWhereOrNull((element) => element.hash == imageAsset.hash)?.isUploaded ?? false) == false) {
uploadData = await beginUploadAsset(client, asset: imageAsset);
if (uploadData.uploadState == UploadState.failed) {
throw "Asset upload failed: ${uploadData.uploadState.name}";
}
await uploadAsset(client, uploadData: uploadData, asset: imageAsset, data: imageData, filename: filename);
await finishUpload(client, asset: imageAsset);
}
uploadData = await beginUploadAsset(client, asset: objectAsset);
if (uploadData.uploadState == UploadState.failed) {
throw "Asset upload failed: ${uploadData.uploadState.name}";
}
await uploadAsset(client, uploadData: uploadData, asset: objectAsset, data: objectBytes, filename: filename);
await finishUpload(client, asset: objectAsset);
final status = await tryPreprocessRecord(client, record: record);
final toUpload = status.resultDiffs.whereNot((element) => element.isUploaded);
await uploadAssets(
client, assets: digests.where((digest) => toUpload.any((diff) => digest.asset.hash == diff.hash)).toList());
return record;
}
}

View file

@ -0,0 +1,25 @@
import 'dart:typed_data';
import 'package:contacts_plus_plus/models/records/neos_db_asset.dart';
import 'package:path/path.dart';
class AssetDigest {
final Uint8List data;
final NeosDBAsset asset;
final String name;
final String dbUri;
AssetDigest({required this.data, required this.asset, required this.name, required this.dbUri});
static Future<AssetDigest> fromData(Uint8List data, String filename) async {
final asset = NeosDBAsset.fromData(data);
return AssetDigest(
data: data,
asset: asset,
name: basenameWithoutExtension(filename),
dbUri: "neosdb:///${asset.hash}${extension(filename)}",
);
}
}

View file

@ -1,4 +1,6 @@
import 'package:contacts_plus_plus/auxiliary.dart';
import 'package:contacts_plus_plus/models/message.dart';
import 'package:contacts_plus_plus/models/records/asset_digest.dart';
import 'package:contacts_plus_plus/models/records/neos_db_asset.dart';
import 'package:contacts_plus_plus/string_formatter.dart';
import 'package:flutter/material.dart';
@ -101,6 +103,53 @@ class Record {
required this.randomOrder,
}) : formattedName = FormatNode.fromText(name);
factory Record.fromRequiredData({
required RecordType recordType,
required String userId,
required String machineId,
required String assetUri,
required String filename,
required String thumbnailUri,
required List<AssetDigest> digests,
}) {
final combinedRecordId = RecordId(id: Record.generateId(), ownerId: userId, isValid: true);
return Record(
id: combinedRecordId.id.toString(),
combinedRecordId: combinedRecordId,
assetUri: assetUri,
name: filename,
tags: [
filename,
"message_item",
"message_id:${Message.generateId()}"
],
recordType: recordType,
thumbnailUri: thumbnailUri,
isPublic: false,
isForPatreons: false,
isListed: false,
neosDBManifest: digests.map((e) => e.asset).toList(),
globalVersion: 0,
localVersion: 1,
lastModifyingUserId: userId,
lastModifyingMachineId: machineId,
lastModificationTime: DateTime.now().toUtc(),
creationTime: DateTime.now().toUtc(),
ownerId: userId,
isSynced: false,
fetchedOn: DateTimeX.one,
path: '',
description: '',
manifest: digests.map((e) => e.dbUri).toList(),
url: "neosrec:///$userId/${combinedRecordId.id}",
isValidOwnerId: true,
isValidRecordId: true,
visits: 0,
rating: 0,
randomOrder: 0,
);
}
factory Record.fromMap(Map map) {
return Record(
id: map["id"] ?? "0",