Skip to content

Commit d721656

Browse files
SysSn13SysSn13
committed
add: limit on async calls for fetching contests and users
1 parent 61aaff9 commit d721656

File tree

8 files changed

+182
-293
lines changed

8 files changed

+182
-293
lines changed

‎models/contest.js

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
const mongoose = require('mongoose');
2-
const Schema = mongoose.Schema
1+
const mongoose = require("mongoose");
2+
const Schema = mongoose.Schema;
33

44
const rankingSchema = new Schema({
55
_id: String,
@@ -8,28 +8,24 @@ const rankingSchema = new Schema({
88
country_name: String,
99
data_region: {
1010
type: String,
11-
default: "US"
11+
default: "US",
1212
},
1313
rank: Number,
1414
current_rating: {
1515
type: Number,
16-
default: null
16+
default: null,
1717
},
1818
delta: {
1919
type: Number,
20-
default: null
21-
}
22-
})
20+
default: null,
21+
},
22+
});
2323
const ContestRankingsSchema = new Schema({
2424
_id: String,
2525
startTime: Date,
2626
endTime: Date,
2727
contest_id: Number,
28-
num_user: Number,
29-
pages_fetched: {
30-
type: Number,
31-
default:0,
32-
},
28+
user_num: Number,
3329
rankings_fetched: {
3430
type: Boolean,
3531
default: false,
@@ -42,7 +38,6 @@ const ContestRankingsSchema = new Schema({
4238
lastUpdated: {
4339
type: Date,
4440
},
45-
})
46-
41+
});
4742

48-
module.exports = mongoose.model('Contest', ContestRankingsSchema)
43+
module.exports = mongoose.model("Contest", ContestRankingsSchema);

‎package-lock.json

Lines changed: 0 additions & 34 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@
77
},
88
"main": "server.js",
99
"scripts": {
10-
"start": "node main.js",
10+
"start": "node main.js --max_old_space_size=512",
1111
"dev": "nodemon main.js --max_old_space_size=1024",
12-
"buildAddon":"npm run build --prefix ./services/predict-addon/"
12+
"buildAddon": "npm run build --prefix ./services/predict-addon/"
1313
},
1414
"keywords": [],
1515
"author": "Sudesh Chaudhary <sudeshchaudhary1122@gmail.com> (https://github.com/SysSn13)",
@@ -22,8 +22,7 @@
2222
"express-ejs-layouts": "^2.5.0",
2323
"mongoose": "^5.12.10",
2424
"node-fetch": "^2.6.1",
25-
"node-gyp": "^8.1.0",
26-
"node-schedule": "^2.0.0"
25+
"node-gyp": "^8.1.0"
2726
},
2827
"_id": "leetcode-rating-predictor@1.0.0",
2928
"devDependencies": {

‎services/contests.js

Lines changed: 108 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -3,41 +3,39 @@ const fetch = require("node-fetch");
33
const Contest = require("../models/contest");
44
let halfHour = 1000 * 60 * 30;
55

6-
async function pushRankingsToContest(
7-
contest_id,
8-
rankings,
9-
pagesCnt,
10-
islastPage
11-
) {
12-
try {
13-
await Contest.findOneAndUpdate(
14-
{
15-
_id: contest_id,
16-
},
17-
{
18-
$push: {
19-
rankings: rankings,
20-
},
21-
$inc: {
22-
pages_fetched: pagesCnt,
23-
},
24-
$set: {
25-
rankings_fetched: islastPage,
26-
},
27-
}
28-
);
6+
// async function pushRankingsToContest(
7+
// contest_id,
8+
// rankings,
9+
// pagesCnt,
10+
// islastPage
11+
// ) {
12+
// try {
13+
// await Contest.findOneAndUpdate(
14+
// {
15+
// _id: contest_id,
16+
// },
17+
// {
18+
// $push: {
19+
// rankings: rankings,
20+
// },
21+
// $inc: {
22+
// pages_fetched: pagesCnt,
23+
// },
24+
// $set: {
25+
// rankings_fetched: islastPage,
26+
// },
27+
// }
28+
// );
2929

30-
return null;
31-
} catch (err) {
32-
console.log(err);
33-
return err;
34-
}
35-
}
30+
// return null;
31+
// } catch (err) {
32+
// console.log(err);
33+
// return err;
34+
// }
35+
// }
3636
const fetchContestRankings = async function (contestSlug) {
3737
try {
38-
let contest = await Contest.findById(contestSlug, {
39-
rankings: 0,
40-
});
38+
let contest = await Contest.findById(contestSlug, { rankings: 0 });
4139
if (!contest) {
4240
return Error(`Contest ${contestSlug} not found in the db`);
4341
}
@@ -47,54 +45,86 @@ const fetchContestRankings = async function (contestSlug) {
4745
`https://leetcode.com/contest/api/ranking/${contestSlug}/?region=global`
4846
);
4947
resp = await resp.json();
50-
// let num_user = resp.user_num
51-
let pages = Math.floor(resp.user_num / 25),
52-
page = contest.pages_fetched + 1;
53-
for (let i = page; i <= pages; i++) {
54-
console.log(`Fetching rankings (${contestSlug}): page: ${i}`);
55-
let res = await fetch(
56-
`https://leetcode.com/contest/api/ranking/${contestSlug}/?pagination=${i}&region=global`
57-
);
58-
res = await res.json();
59-
rankings = res.total_rank
60-
.filter(
61-
(ranks) =>
62-
!(
63-
ranks.score == 0 &&
64-
ranks.finish_time * 1000 ==
65-
contest.startTime.getTime()
66-
)
67-
)
68-
.map((ranks) => {
69-
let {
70-
username,
71-
user_slug,
72-
country_code,
73-
country_name,
74-
data_region,
75-
rank,
76-
} = ranks;
77-
let ranking = {
78-
username,
79-
user_slug,
80-
country_code,
81-
country_name,
82-
data_region,
83-
rank,
84-
};
85-
ranking["_id"] = username;
86-
return ranking;
87-
});
88-
let err = await pushRankingsToContest(
89-
contest._id,
90-
rankings,
91-
1,
92-
i === pages
93-
);
94-
if (err) {
95-
return err;
48+
let pages = Math.floor(resp.user_num / 25);
49+
let all_rankings = [];
50+
let failed = [];
51+
const fetchPageRankings = async (
52+
pageNo,
53+
retries,
54+
throwError = false
55+
) => {
56+
console.log(`Fetching rankings (${contestSlug}): page: ${pageNo}`);
57+
try {
58+
let res = await fetch(
59+
`https://leetcode.com/contest/api/ranking/${contestSlug}/?pagination=${pageNo}&region=global`
60+
);
61+
res = await res.json();
62+
rankings = res.total_rank
63+
.filter(
64+
(ranks) =>
65+
!(
66+
ranks.score == 0 &&
67+
ranks.finish_time * 1000 ==
68+
contest.startTime.getTime()
69+
)
70+
)
71+
.map((ranks) => {
72+
let {
73+
username,
74+
user_slug,
75+
country_code,
76+
country_name,
77+
data_region,
78+
rank,
79+
} = ranks;
80+
let ranking = {
81+
username,
82+
user_slug,
83+
country_code,
84+
country_name,
85+
data_region,
86+
rank,
87+
};
88+
ranking["_id"] = username;
89+
return ranking;
90+
});
91+
all_rankings = all_rankings.concat(rankings);
92+
console.log(
93+
`Fetched rankings (${contestSlug}): page: ${pageNo}`
94+
);
95+
} catch (err) {
96+
console.log(
97+
`Failed to fetch rankings (${contestSlug}, page: ${pageNo})`
98+
);
99+
console.log(err);
100+
if (retries > 0) {
101+
await fetchPageRankings(pageNo, retries - 1);
102+
} else if (throwError) {
103+
throw err;
104+
} else {
105+
failed.push(pageNo);
106+
}
107+
}
108+
};
109+
const limit = 5;
110+
const maxRetries = 5;
111+
for (let i = 0; i < pages; i += limit) {
112+
let promises = [];
113+
for (let j = 0; j < limit && i + j < pages; j++) {
114+
promises.push(fetchPageRankings(i + j + 1, maxRetries));
96115
}
116+
await Promise.all(promises);
117+
}
118+
119+
for (let i = 0; i < failed.length; i++) {
120+
await fetchPageRankings(failed[i], maxRetries, true);
97121
}
122+
123+
all_rankings.sort((a, b) => (a.rank > b.rank ? 1 : -1));
124+
contest.rankings = all_rankings;
125+
contest.rankings_fetched = true;
126+
contest.user_num = resp.user_num;
127+
await contest.save();
98128
console.log(`Updated Rankings in ${contestSlug}.`);
99129
} catch (err) {
100130
return err;

‎services/job-queues/contestPredictionQueue.js

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,14 @@ const Queue = require("bull");
22
const { predict } = require("../predict");
33
const opts = require("../redis");
44

5+
opts.lockDuration = 30 * 60 * 1000; // 30 minutes
6+
57
const predictQueue = new Queue("Predictions", opts);
68
predictQueue.process(async (job, done) => {
79
console.log("Processing job: ", job.id);
810
const err = await predict(job);
911
if (err) {
12+
console.error(err);
1013
done(err);
1114
}
1215
done();

‎services/job-queues/jobScheduler.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ scheduler.process(async (job, done) => {
1616
};
1717

1818
const isLatest = (endTime) => {
19-
return Date.now() - endTime >= 0;
19+
return Date.now() - endTime <= 2*24 * 60 * 60 * 1000;
2020
};
2121

2222
const withinAWeek = (endTime) => {

0 commit comments

Comments
 (0)