I'm actually building a script to extract data from a MySQL database and then populating a MongoDB. In the process there are some asynchronous stuff like establishing a connection to MySQL (through Sequelize library) and MongoDB (through Mongoose library), and some synchronous stuff like fetching and converting data.
I read a lot about async/await and Promises and my script is globally doing what I want want, but still have some issues.
Here's the code :
Migration.class.mjs
import MigrationBase from './Base/MigrationBase.class.mjs';
export default class Migration extends MigrationBase
{
constructor(config) {
super(config);
this.mysqlData = {};
this.mongoData = {};
}
async run() {
await this.selectMySQLData();
let docs = await this.convertMySQLToMongo();
await this.checkConvertedData(docs);
await this.insertMongoData();
}
async selectMySQLData() {
return new Promise(async resolve => {
await this.runSequelize();
console.log('B - Grabbing MySQL data\n');
for(var key in this.mysqlModels) {
if (this.mysqlModels.hasOwnProperty(key)) {
let search = { raw: true };
this.mysqlData[key] = await this.mysqlModels[key].findAll(search);
}
}
await this.closeSequelize();
resolve();
});
};
convertMySQLToMongo() {
return new Promise(async resolve => {
console.log('D - Convert MySQL data to MongoDB\n');
let customersDocument = this.defaultDocuments.customers;
let personalInfosDocument = this.defaultDocuments.personal_infos;
let billingInfosDocument = this.defaultDocuments.billing_infos;
// ... etc ...
await Object.entries(this.mysqlData.customer).forEach(async keyRow => {
let [key, row] = keyRow;
await Object.entries(row).forEach(async keyValue => {
customersDocument = await this._processCustomersFields(customersDocument, 'Customer', keyValue);
personalInfosDocument = await this._processPersonalInfosFields(personalInfosDocument, 'PersonalInfo', keyValue);
billingInfosDocument = await this._processBillingInfosFields(billingInfosDocument, 'BillingInfo', keyValue);
// ... etc ...
});
resolve([
customersDocument,
personalInfosDocument,
billingInfosDocument,
// ... etc ...
]);
});
};
checkConvertedData([
customersDocument,
personalInfosDocument,
billingInfosDocument,
// ... etc ...
]) {
return new Promise(resolve => {
console.log('E - Checking converted data');
if (! this._isNull(customersDocument, 'Customers')) {
this.mongoData.customers = customersDocument;
}
if (! this._isNull(personalInfosDocument, 'PersonalInfos')) {
this.mongoData.personal_infos = personalInfosDocument;
}
if (! this._isNull(billingInfosDocument, 'BillingInfos')) {
} this.mongoData.billing_infos = billingInfosDocument;
// ... etc ...
resolve();
});
}
async insertMongoData() {
return new Promise(async resolve => {
await this.runMongoose();
console.log('G - Insert MongoDB data.');
await this.mongoModels.customers.create(this.mongoData.customers);
await this.mongoModels.personal_infos.create(this.mongoData.personal_infos);
await this.mongoModels.billing_infos.create(this.mongoData.billing_infos);
// ... etc ...
await this.closeMongoose();
resolve();
});
};
_processCustomersFields(defaultDoc, docName, [colName, val]) {
return new Promise(resolve => {
switch (colName) {
case 'id_customer':
console.log(`${docName}: ${colName} => ${val}`);
defaultDoc.id = val;
break;
case 'email_customer':
console.log(`${docName}: ${colName} => ${val}`);
defaultDoc.email = val;
break;
case 'password_customer':
console.log(`${docName}: ${colName} => ${val}`);
defaultDoc.password = val;
break;
// ... etc ...
}
resolve(defaultDoc);
});
}
_processPersonalInfosFields(defaultDoc, docName, [colName, val]) {
return new Promise(resolve => {
switch (colName) {
// ... Same kind of code as in _processCustomersFields() ...
}
resolve(defaultDoc);
});
}
_processBillingInfosFields(defaultDoc, docName, [colName, val]) {
return new Promise(resolve => {
switch (colName) {
// ... Same kind of code as in _processCustomersFields() ...
}
resolve(defaultDoc);
});
}
_isNull(document, mongoName) {
if (document !== null) {
console.log(`\n${mongoName}:\n`, JSON.stringify(document));
return false;
} else {
console.log(`Error processing \`${mongoName}\` data!`);
return true;
}
}
_valueExists(val) {
return (val !== null && val !== "" && typeof val !== "undefined")
? true
: false
;
}
}
MigrationBase.class.mjs
import Sequelize from 'sequelize';
import DataTypes from 'sequelize';
import Mongoose from 'mongoose';
import Crypto from 'crypto';
import Models from '../../../models.mjs';
import Schemas from '../../../schemas.mjs';
export default class MigrationBase
{
constructor(config) {
this.config = config;
this.sequelize = this.createSequelize();
this.mongoose = Mongoose;
this.defaultDocuments = this.initDefaultDocuments();
this.mysqlModels = this.initMysqlModels();
this.mongoModels = this.initMongoSchemas();
this.mysqlData = {};
this.mongoData = {};
}
createSequelize() {
return new Sequelize(
this.config.mysql.dbName,
this.config.mysql.dbUser,
this.config.mysql.dbPass,
this.config.sequelize
);
}
initDefaultDocuments() {
const defaultDocument = {
"deleted_at": 0 // Thu Jan 01 1970 01:00:00 GMT 0100
};
let defaultDocuments = {
"customers": Object.assign({}, defaultDocument),
"personal_infos": Object.assign({}, defaultDocument),
"billing_infos": Object.assign({}, defaultDocument)
// ... etc ...
};
return defaultDocuments;
}
initMysqlModels() {
return {
"customer": Models.Customer(this.sequelize, DataTypes),
"billing_address": Models.BillingAddress(this.sequelize, DataTypes),
// ... etc ...
};
}
initMongoSchemas() {
return {
"customers": this.mongoose.model('Customer', Schemas.Customers),
"personal_infos": this.mongoose.model('PersonalInfo', Schemas.PersonalInfos),
"billing_infos": this.mongoose.model('BillingInfo', Schemas.BillingInfos),
// ... etc ...
}
}
async runSequelize() {
console.log('A - Connection to MySQL');
try {
await this.sequelize.authenticate();
console.log('Connection to MySQL has been established successfully.\n');
} catch (err) {
console.error("Unable to connect to the MySQL database:", err '\n');
}
}
async closeSequelize() {
console.log('C - Closing MySQL connection.\n');
await this.sequelize.close();
};
runMongoose() {
return new Promise(async resolve => {
console.log('F - Connection to MongoDB');
try {
await this.mongoose.connect(
`mongodb://${this.config.mongo.dbHost}:${this.config.mongo.dbPort}/${this.config.mongo.dbName}`,
{ useNewUrlParser: true, useUnifiedTopology: true }
);
console.log('Connection to MongoDB has been established successfully.');
} catch (err) {
console.error('Unable to connect to the MongoDB database: ', err);
}
resolve();
});
}
async closeMongoose() {
console.log('H - Closing MongoDB connection.');
await this.mongoose.connection.close();
};
}
And here is the Logs output:
A - Connection to MySQL
Connection to MySQL has been established successfully.
B - Grabbing MySQL data
C - Closing MySQL connection.
D - Convert MySQL data to MongoDB
Customer: id_customer => 1
Customer: email_customer => contact@example.com
Customer: password_customer => 0a1b2c3d4e5f0a1b2c3d4e5f0a1b2c3d
// ... etc ...
PersonalInfo: id_customer => 1
PersonalInfo: lastname_customer => Doe
PersonalInfo: firstname_customer => John
// ... etc ...
E - Checking converted data
Customers:
{"deleted_at":0,"id":"000000000000000000000001","email":"[email protected]","password":"0a1b2c3d4e5f0a1b2c3d4e5f0a1b2c3d", ... etc ... }
PersonalInfos:
{"deleted_at":0,"customer_id":"000000000000000000000001","last_name":"Doe","first_name":"John", ... etc ... }
BillingInfos:
{"deleted_at":0}
BillingInfos: id_customer => 1
BillingInfo: company => ExampleCompany
F - Connection to MongoDB
BillingInfos: lastname => Doe
BillingInfo: firstname => John
Connection to MongoDB has been established successfully.
G - Insert MongoDB data.
/home/user/Workspaces/namespace/project-name/node_modules/mongoose/lib/document.js:2757
this.$__.validationError = new ValidationError(this);
^
ValidationError: BillingInfos validation failed: id_customer: Cast to ObjectId failed for value "1" (type number) at path "customer_id", values: Path `values` is required., id: Path `id` is required.
Here we can see in the right order:
A - Connection to MySQL
B - Grabbing MySQL data
C - Closing MySQL connection
D - Convert MySQL data to MongoDB
Then we can see E - Checking converted data but the conversion process is not finished, despite the await statement and the fact that it return a Promise.
After that we can also see BillingInfos: id_customer => 1 and BillingInfo: company => ExampleCompany meaning that the convertion process is still doing stuff in the loop.
Then F - Connection to MongoDB
Then another convertion logs BillingInfos: lastname => Doe and BillingInfo: firstname => John (convertion process is still doing stuff in the loop).
Then G - Insert MongoDB data.
And finally a Validation Error because some Mongo Documents are incomplete and so the rules are not fullfiled.
Question?
So the question is what am I doing wrong here ?
As I said I read a lot about async/await and Promises but still struglle to understand why it's not working.
Thanks in advance and let me know if you need more info.
CodePudding user response：
That's because await will not work inside forEach(), which you are trying to do in your convertMySQLToMongo() function.
There are many ways in which you can solve it, one of the ways is using for ... of instead of forEach()
for (const keyRow of Object.entries(this.mysqlData.customer)) {
let [key, row] = keyRow;
for (const keyValue of Object.entries(row)) {
customersDocument = await this._processCustomersFields(customersDocument, 'Customer', keyValue);
personalInfosDocument = await this._processPersonalInfosFields(personalInfosDocument, 'PersonalInfo', keyValue);
billingInfosDocument = await this._processBillingInfosFields(billingInfosDocument, 'BillingInfo', keyValue);
}
}