moris/misskey
1
0
Fork 0
forked from mirror/misskey
Code Pull Requests Activity

並列に処理するように

Browse Source
This commit is contained in:
syuilo 2018-10-11 21:14:20 +09:00
parent 07560a4fdd
commit 3a80b59986
No known key found for this signature in database
GPG Key ID: BDC4C49D06AB9D69
2 changed files with 63 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@ -169,6 +169,7 @@
"parse5": "5.1.0", "parse5": "5.1.0",
"portscanner": "2.2.0", "portscanner": "2.2.0",
"progress-bar-webpack-plugin": "1.11.0", "progress-bar-webpack-plugin": "1.11.0",
"promise-limit": "2.7.0",
"promise-sequential": "1.1.1", "promise-sequential": "1.1.1",
"pug": "2.0.3", "pug": "2.0.3",
"punycode": "2.1.1", "punycode": "2.1.1",

118
src/tools/move-drive-files.ts
View File

@ -1,10 +1,12 @@
import * as Minio from 'minio'; import * as Minio from 'minio';
import * as uuid from 'uuid'; import * as uuid from 'uuid';
const sequential = require('promise-sequential'); import * as promiseLimit from 'promise-limit';
import DriveFile, { DriveFileChunk, getDriveFileBucket } from '../models/drive-file'; import DriveFile, { DriveFileChunk, getDriveFileBucket, IDriveFile } from '../models/drive-file';
import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../models/drive-file-thumbnail'; import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../models/drive-file-thumbnail';
import config from '../config'; import config from '../config';
const limit = promiseLimit(16);
DriveFile.find({ DriveFile.find({
$or: [{ $or: [{
withoutChunks: { $exists: false } withoutChunks: { $exists: false }
@ -19,59 +21,63 @@ DriveFile.find({
}).then(async files => { }).then(async files => {
console.log(`there is ${files.length} files`); console.log(`there is ${files.length} files`);
await sequential(files.map(file => async () => { await Promise.all(files.map(file => limit(() => job(file))));
file = await DriveFile.findOne({ _id: file._id });
const minio = new Minio.Client(config.drive.config); console.log('ALL DONE');
const name = file.filename;
const keyDir = `${config.drive.prefix}/${uuid.v4()}`;
const key = `${keyDir}/${name}`;
const thumbnailKeyDir = `${config.drive.prefix}/${uuid.v4()}`;
const thumbnailKey = `${thumbnailKeyDir}/${name}.thumbnail.jpg`;
const baseUrl = config.drive.baseUrl
|| `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`;
const bucket = await getDriveFileBucket();
const readable = bucket.openDownloadStream(file._id);
await minio.putObject(config.drive.bucket, key, readable, file.length, {
'Content-Type': file.contentType,
'Cache-Control': 'max-age=31536000, immutable'
});
await DriveFile.findOneAndUpdate({ _id: file._id }, {
$set: {
'metadata.withoutChunks': true,
'metadata.storage': 'minio',
'metadata.storageProps': {
key: key,
thumbnailKey: thumbnailKey
},
'metadata.url': `${ baseUrl }/${ keyDir }/${ encodeURIComponent(name) }`,
}
});
// チャンクをすべて削除
await DriveFileChunk.remove({
files_id: file._id
});
//#region サムネイルもあれば削除
const thumbnail = await DriveFileThumbnail.findOne({
'metadata.originalId': file._id
});
if (thumbnail) {
await DriveFileThumbnailChunk.remove({
files_id: thumbnail._id
});
await DriveFileThumbnail.remove({ _id: thumbnail._id });
}
//#endregion
console.log('done', file._id);
}));
}); });
async function job(file: IDriveFile): Promise<any> {
file = await DriveFile.findOne({ _id: file._id });
const minio = new Minio.Client(config.drive.config);
const name = file.filename;
const keyDir = `${config.drive.prefix}/${uuid.v4()}`;
const key = `${keyDir}/${name}`;
const thumbnailKeyDir = `${config.drive.prefix}/${uuid.v4()}`;
const thumbnailKey = `${thumbnailKeyDir}/${name}.thumbnail.jpg`;
const baseUrl = config.drive.baseUrl
|| `${ config.drive.config.useSSL ? 'https' : 'http' }://${ config.drive.config.endPoint }${ config.drive.config.port ? `:${config.drive.config.port}` : '' }/${ config.drive.bucket }`;
const bucket = await getDriveFileBucket();
const readable = bucket.openDownloadStream(file._id);
await minio.putObject(config.drive.bucket, key, readable, file.length, {
'Content-Type': file.contentType,
'Cache-Control': 'max-age=31536000, immutable'
});
await DriveFile.findOneAndUpdate({ _id: file._id }, {
$set: {
'metadata.withoutChunks': true,
'metadata.storage': 'minio',
'metadata.storageProps': {
key: key,
thumbnailKey: thumbnailKey
},
'metadata.url': `${ baseUrl }/${ keyDir }/${ encodeURIComponent(name) }`,
}
});
// チャンクをすべて削除
await DriveFileChunk.remove({
files_id: file._id
});
//#region サムネイルもあれば削除
const thumbnail = await DriveFileThumbnail.findOne({
'metadata.originalId': file._id
});
if (thumbnail) {
await DriveFileThumbnailChunk.remove({
files_id: thumbnail._id
});
await DriveFileThumbnail.remove({ _id: thumbnail._id });
}
//#endregion
console.log('done', file._id);
}